Author | Tokens | Token Proportion | Commits | Commit Proportion |
---|---|---|---|---|
Chris Wilson | 6163 | 89.88% | 51 | 57.95% |
Maarten Lankhorst | 388 | 5.66% | 3 | 3.41% |
Tvrtko A. Ursulin | 115 | 1.68% | 9 | 10.23% |
Eric Anholt | 38 | 0.55% | 3 | 3.41% |
Fei Yang | 27 | 0.39% | 2 | 2.27% |
Akash Goel | 27 | 0.39% | 1 | 1.14% |
Michał Winiarski | 27 | 0.39% | 3 | 3.41% |
Zou Nan hai | 16 | 0.23% | 1 | 1.14% |
Xiang, Haihao | 10 | 0.15% | 1 | 1.14% |
Alex Dai | 8 | 0.12% | 1 | 1.14% |
Lucas De Marchi | 7 | 0.10% | 2 | 2.27% |
Arun Siluvery | 6 | 0.09% | 1 | 1.14% |
Oscar Mateo | 5 | 0.07% | 2 | 2.27% |
Michel Thierry | 4 | 0.06% | 1 | 1.14% |
Thomas Daniel | 4 | 0.06% | 1 | 1.14% |
Michal Wajdeczko | 3 | 0.04% | 1 | 1.14% |
Mika Kuoppala | 3 | 0.04% | 1 | 1.14% |
Matt Roper | 3 | 0.04% | 1 | 1.14% |
John Harrison | 1 | 0.01% | 1 | 1.14% |
Jason Ekstrand | 1 | 0.01% | 1 | 1.14% |
Daniel Vetter | 1 | 0.01% | 1 | 1.14% |
Total | 6857 | 88 |
// SPDX-License-Identifier: MIT /* * Copyright © 2017-2018 Intel Corporation */ #include <linux/prime_numbers.h> #include <linux/string_helpers.h> #include "intel_context.h" #include "intel_engine_heartbeat.h" #include "intel_engine_pm.h" #include "intel_engine_regs.h" #include "intel_gpu_commands.h" #include "intel_gt.h" #include "intel_gt_requests.h" #include "intel_ring.h" #include "selftest_engine_heartbeat.h" #include "../selftests/i915_random.h" #include "../i915_selftest.h" #include "selftests/igt_flush_test.h" #include "selftests/lib_sw_fence.h" #include "selftests/mock_gem_device.h" #include "selftests/mock_timeline.h" static struct page *hwsp_page(struct intel_timeline *tl) { struct drm_i915_gem_object *obj = tl->hwsp_ggtt->obj; GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); return sg_page(obj->mm.pages->sgl); } static unsigned long hwsp_cacheline(struct intel_timeline *tl) { unsigned long address = (unsigned long)page_address(hwsp_page(tl)); return (address + offset_in_page(tl->hwsp_offset)) / TIMELINE_SEQNO_BYTES; } static int selftest_tl_pin(struct intel_timeline *tl) { struct i915_gem_ww_ctx ww; int err; i915_gem_ww_ctx_init(&ww, false); retry: err = i915_gem_object_lock(tl->hwsp_ggtt->obj, &ww); if (!err) err = intel_timeline_pin(tl, &ww); if (err == -EDEADLK) { err = i915_gem_ww_ctx_backoff(&ww); if (!err) goto retry; } i915_gem_ww_ctx_fini(&ww); return err; } /* Only half of seqno's are usable, see __intel_timeline_get_seqno() */ #define CACHELINES_PER_PAGE (PAGE_SIZE / TIMELINE_SEQNO_BYTES / 2) struct mock_hwsp_freelist { struct intel_gt *gt; struct radix_tree_root cachelines; struct intel_timeline **history; unsigned long count, max; struct rnd_state prng; }; enum { SHUFFLE = BIT(0), }; static void __mock_hwsp_record(struct mock_hwsp_freelist *state, unsigned int idx, struct intel_timeline *tl) { tl = xchg(&state->history[idx], tl); if (tl) { radix_tree_delete(&state->cachelines, hwsp_cacheline(tl)); intel_timeline_unpin(tl); intel_timeline_put(tl); } } static int __mock_hwsp_timeline(struct mock_hwsp_freelist *state, unsigned int count, unsigned int flags) { struct intel_timeline *tl; unsigned int idx; while (count--) { unsigned long cacheline; int err; tl = intel_timeline_create(state->gt); if (IS_ERR(tl)) return PTR_ERR(tl); err = selftest_tl_pin(tl); if (err) { intel_timeline_put(tl); return err; } cacheline = hwsp_cacheline(tl); err = radix_tree_insert(&state->cachelines, cacheline, tl); if (err) { if (err == -EEXIST) { pr_err("HWSP cacheline %lu already used; duplicate allocation!\n", cacheline); } intel_timeline_unpin(tl); intel_timeline_put(tl); return err; } idx = state->count++ % state->max; __mock_hwsp_record(state, idx, tl); } if (flags & SHUFFLE) i915_prandom_shuffle(state->history, sizeof(*state->history), min(state->count, state->max), &state->prng); count = i915_prandom_u32_max_state(min(state->count, state->max), &state->prng); while (count--) { idx = --state->count % state->max; __mock_hwsp_record(state, idx, NULL); } return 0; } static int mock_hwsp_freelist(void *arg) { struct mock_hwsp_freelist state; struct drm_i915_private *i915; const struct { const char *name; unsigned int flags; } phases[] = { { "linear", 0 }, { "shuffled", SHUFFLE }, { }, }, *p; unsigned int na; int err = 0; i915 = mock_gem_device(); if (!i915) return -ENOMEM; INIT_RADIX_TREE(&state.cachelines, GFP_KERNEL); state.prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed); state.gt = to_gt(i915); /* * Create a bunch of timelines and check that their HWSP do not overlap. * Free some, and try again. */ state.max = PAGE_SIZE / sizeof(*state.history); state.count = 0; state.history = kcalloc(state.max, sizeof(*state.history), GFP_KERNEL); if (!state.history) { err = -ENOMEM; goto err_put; } for (p = phases; p->name; p++) { pr_debug("%s(%s)\n", __func__, p->name); for_each_prime_number_from(na, 1, 2 * CACHELINES_PER_PAGE) { err = __mock_hwsp_timeline(&state, na, p->flags); if (err) goto out; } } out: for (na = 0; na < state.max; na++) __mock_hwsp_record(&state, na, NULL); kfree(state.history); err_put: mock_destroy_device(i915); return err; } struct __igt_sync { const char *name; u32 seqno; bool expected; bool set; }; static int __igt_sync(struct intel_timeline *tl, u64 ctx, const struct __igt_sync *p, const char *name) { int ret; if (__intel_timeline_sync_is_later(tl, ctx, p->seqno) != p->expected) { pr_err("%s: %s(ctx=%llu, seqno=%u) expected passed %s but failed\n", name, p->name, ctx, p->seqno, str_yes_no(p->expected)); return -EINVAL; } if (p->set) { ret = __intel_timeline_sync_set(tl, ctx, p->seqno); if (ret) return ret; } return 0; } static int igt_sync(void *arg) { const struct __igt_sync pass[] = { { "unset", 0, false, false }, { "new", 0, false, true }, { "0a", 0, true, true }, { "1a", 1, false, true }, { "1b", 1, true, true }, { "0b", 0, true, false }, { "2a", 2, false, true }, { "4", 4, false, true }, { "INT_MAX", INT_MAX, false, true }, { "INT_MAX-1", INT_MAX-1, true, false }, { "INT_MAX+1", (u32)INT_MAX+1, false, true }, { "INT_MAX", INT_MAX, true, false }, { "UINT_MAX", UINT_MAX, false, true }, { "wrap", 0, false, true }, { "unwrap", UINT_MAX, true, false }, {}, }, *p; struct intel_timeline tl; int order, offset; int ret = -ENODEV; mock_timeline_init(&tl, 0); for (p = pass; p->name; p++) { for (order = 1; order < 64; order++) { for (offset = -1; offset <= (order > 1); offset++) { u64 ctx = BIT_ULL(order) + offset; ret = __igt_sync(&tl, ctx, p, "1"); if (ret) goto out; } } } mock_timeline_fini(&tl); mock_timeline_init(&tl, 0); for (order = 1; order < 64; order++) { for (offset = -1; offset <= (order > 1); offset++) { u64 ctx = BIT_ULL(order) + offset; for (p = pass; p->name; p++) { ret = __igt_sync(&tl, ctx, p, "2"); if (ret) goto out; } } } out: mock_timeline_fini(&tl); return ret; } static unsigned int random_engine(struct rnd_state *rnd) { return i915_prandom_u32_max_state(I915_NUM_ENGINES, rnd); } static int bench_sync(void *arg) { struct rnd_state prng; struct intel_timeline tl; unsigned long end_time, count; u64 prng32_1M; ktime_t kt; int order, last_order; mock_timeline_init(&tl, 0); /* Lookups from cache are very fast and so the random number generation * and the loop itself becomes a significant factor in the per-iteration * timings. We try to compensate the results by measuring the overhead * of the prng and subtract it from the reported results. */ prandom_seed_state(&prng, i915_selftest.random_seed); count = 0; kt = ktime_get(); end_time = jiffies + HZ/10; do { u32 x; /* Make sure the compiler doesn't optimise away the prng call */ WRITE_ONCE(x, prandom_u32_state(&prng)); count++; } while (!time_after(jiffies, end_time)); kt = ktime_sub(ktime_get(), kt); pr_debug("%s: %lu random evaluations, %lluns/prng\n", __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); prng32_1M = div64_ul(ktime_to_ns(kt) << 20, count); /* Benchmark (only) setting random context ids */ prandom_seed_state(&prng, i915_selftest.random_seed); count = 0; kt = ktime_get(); end_time = jiffies + HZ/10; do { u64 id = i915_prandom_u64_state(&prng); __intel_timeline_sync_set(&tl, id, 0); count++; } while (!time_after(jiffies, end_time)); kt = ktime_sub(ktime_get(), kt); kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20); pr_info("%s: %lu random insertions, %lluns/insert\n", __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); /* Benchmark looking up the exact same context ids as we just set */ prandom_seed_state(&prng, i915_selftest.random_seed); end_time = count; kt = ktime_get(); while (end_time--) { u64 id = i915_prandom_u64_state(&prng); if (!__intel_timeline_sync_is_later(&tl, id, 0)) { mock_timeline_fini(&tl); pr_err("Lookup of %llu failed\n", id); return -EINVAL; } } kt = ktime_sub(ktime_get(), kt); kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20); pr_info("%s: %lu random lookups, %lluns/lookup\n", __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); mock_timeline_fini(&tl); cond_resched(); mock_timeline_init(&tl, 0); /* Benchmark setting the first N (in order) contexts */ count = 0; kt = ktime_get(); end_time = jiffies + HZ/10; do { __intel_timeline_sync_set(&tl, count++, 0); } while (!time_after(jiffies, end_time)); kt = ktime_sub(ktime_get(), kt); pr_info("%s: %lu in-order insertions, %lluns/insert\n", __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); /* Benchmark looking up the exact same context ids as we just set */ end_time = count; kt = ktime_get(); while (end_time--) { if (!__intel_timeline_sync_is_later(&tl, end_time, 0)) { pr_err("Lookup of %lu failed\n", end_time); mock_timeline_fini(&tl); return -EINVAL; } } kt = ktime_sub(ktime_get(), kt); pr_info("%s: %lu in-order lookups, %lluns/lookup\n", __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); mock_timeline_fini(&tl); cond_resched(); mock_timeline_init(&tl, 0); /* Benchmark searching for a random context id and maybe changing it */ prandom_seed_state(&prng, i915_selftest.random_seed); count = 0; kt = ktime_get(); end_time = jiffies + HZ/10; do { u32 id = random_engine(&prng); u32 seqno = prandom_u32_state(&prng); if (!__intel_timeline_sync_is_later(&tl, id, seqno)) __intel_timeline_sync_set(&tl, id, seqno); count++; } while (!time_after(jiffies, end_time)); kt = ktime_sub(ktime_get(), kt); kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20); pr_info("%s: %lu repeated insert/lookups, %lluns/op\n", __func__, count, (long long)div64_ul(ktime_to_ns(kt), count)); mock_timeline_fini(&tl); cond_resched(); /* Benchmark searching for a known context id and changing the seqno */ for (last_order = 1, order = 1; order < 32; ({ int tmp = last_order; last_order = order; order += tmp; })) { unsigned int mask = BIT(order) - 1; mock_timeline_init(&tl, 0); count = 0; kt = ktime_get(); end_time = jiffies + HZ/10; do { /* Without assuming too many details of the underlying * implementation, try to identify its phase-changes * (if any)! */ u64 id = (u64)(count & mask) << order; __intel_timeline_sync_is_later(&tl, id, 0); __intel_timeline_sync_set(&tl, id, 0); count++; } while (!time_after(jiffies, end_time)); kt = ktime_sub(ktime_get(), kt); pr_info("%s: %lu cyclic/%d insert/lookups, %lluns/op\n", __func__, count, order, (long long)div64_ul(ktime_to_ns(kt), count)); mock_timeline_fini(&tl); cond_resched(); } return 0; } int intel_timeline_mock_selftests(void) { static const struct i915_subtest tests[] = { SUBTEST(mock_hwsp_freelist), SUBTEST(igt_sync), SUBTEST(bench_sync), }; return i915_subtests(tests, NULL); } static int emit_ggtt_store_dw(struct i915_request *rq, u32 addr, u32 value) { u32 *cs; cs = intel_ring_begin(rq, 4); if (IS_ERR(cs)) return PTR_ERR(cs); if (GRAPHICS_VER(rq->i915) >= 8) { *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; *cs++ = addr; *cs++ = 0; *cs++ = value; } else if (GRAPHICS_VER(rq->i915) >= 4) { *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; *cs++ = 0; *cs++ = addr; *cs++ = value; } else { *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; *cs++ = addr; *cs++ = value; *cs++ = MI_NOOP; } intel_ring_advance(rq, cs); return 0; } static struct i915_request * checked_tl_write(struct intel_timeline *tl, struct intel_engine_cs *engine, u32 value) { struct i915_request *rq; int err; err = selftest_tl_pin(tl); if (err) { rq = ERR_PTR(err); goto out; } if (READ_ONCE(*tl->hwsp_seqno) != tl->seqno) { pr_err("Timeline created with incorrect breadcrumb, found %x, expected %x\n", *tl->hwsp_seqno, tl->seqno); intel_timeline_unpin(tl); return ERR_PTR(-EINVAL); } rq = intel_engine_create_kernel_request(engine); if (IS_ERR(rq)) goto out_unpin; i915_request_get(rq); err = emit_ggtt_store_dw(rq, tl->hwsp_offset, value); i915_request_add(rq); if (err) { i915_request_put(rq); rq = ERR_PTR(err); } out_unpin: intel_timeline_unpin(tl); out: if (IS_ERR(rq)) pr_err("Failed to write to timeline!\n"); return rq; } static int live_hwsp_engine(void *arg) { #define NUM_TIMELINES 4096 struct intel_gt *gt = arg; struct intel_timeline **timelines; struct intel_engine_cs *engine; enum intel_engine_id id; unsigned long count, n; int err = 0; /* * Create a bunch of timelines and check we can write * independently to each of their breadcrumb slots. */ timelines = kvmalloc_array(NUM_TIMELINES * I915_NUM_ENGINES, sizeof(*timelines), GFP_KERNEL); if (!timelines) return -ENOMEM; count = 0; for_each_engine(engine, gt, id) { if (!intel_engine_can_store_dword(engine)) continue; intel_engine_pm_get(engine); for (n = 0; n < NUM_TIMELINES; n++) { struct intel_timeline *tl; struct i915_request *rq; tl = intel_timeline_create(gt); if (IS_ERR(tl)) { err = PTR_ERR(tl); break; } rq = checked_tl_write(tl, engine, count); if (IS_ERR(rq)) { intel_timeline_put(tl); err = PTR_ERR(rq); break; } timelines[count++] = tl; i915_request_put(rq); } intel_engine_pm_put(engine); if (err) break; } if (igt_flush_test(gt->i915)) err = -EIO; for (n = 0; n < count; n++) { struct intel_timeline *tl = timelines[n]; if (!err && READ_ONCE(*tl->hwsp_seqno) != n) { GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x, found 0x%x\n", n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno); GEM_TRACE_DUMP(); err = -EINVAL; } intel_timeline_put(tl); } kvfree(timelines); return err; #undef NUM_TIMELINES } static int live_hwsp_alternate(void *arg) { #define NUM_TIMELINES 4096 struct intel_gt *gt = arg; struct intel_timeline **timelines; struct intel_engine_cs *engine; enum intel_engine_id id; unsigned long count, n; int err = 0; /* * Create a bunch of timelines and check we can write * independently to each of their breadcrumb slots with adjacent * engines. */ timelines = kvmalloc_array(NUM_TIMELINES * I915_NUM_ENGINES, sizeof(*timelines), GFP_KERNEL); if (!timelines) return -ENOMEM; count = 0; for (n = 0; n < NUM_TIMELINES; n++) { for_each_engine(engine, gt, id) { struct intel_timeline *tl; struct i915_request *rq; if (!intel_engine_can_store_dword(engine)) continue; tl = intel_timeline_create(gt); if (IS_ERR(tl)) { err = PTR_ERR(tl); goto out; } intel_engine_pm_get(engine); rq = checked_tl_write(tl, engine, count); intel_engine_pm_put(engine); if (IS_ERR(rq)) { intel_timeline_put(tl); err = PTR_ERR(rq); goto out; } timelines[count++] = tl; i915_request_put(rq); } } out: if (igt_flush_test(gt->i915)) err = -EIO; for (n = 0; n < count; n++) { struct intel_timeline *tl = timelines[n]; if (!err && READ_ONCE(*tl->hwsp_seqno) != n) { GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x, found 0x%x\n", n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno); GEM_TRACE_DUMP(); err = -EINVAL; } intel_timeline_put(tl); } kvfree(timelines); return err; #undef NUM_TIMELINES } static int live_hwsp_wrap(void *arg) { struct intel_gt *gt = arg; struct intel_engine_cs *engine; struct intel_timeline *tl; enum intel_engine_id id; int err = 0; /* * Across a seqno wrap, we need to keep the old cacheline alive for * foreign GPU references. */ tl = intel_timeline_create(gt); if (IS_ERR(tl)) return PTR_ERR(tl); if (!tl->has_initial_breadcrumb) goto out_free; err = selftest_tl_pin(tl); if (err) goto out_free; for_each_engine(engine, gt, id) { const u32 *hwsp_seqno[2]; struct i915_request *rq; u32 seqno[2]; if (!intel_engine_can_store_dword(engine)) continue; rq = intel_engine_create_kernel_request(engine); if (IS_ERR(rq)) { err = PTR_ERR(rq); goto out; } tl->seqno = -4u; mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING); err = intel_timeline_get_seqno(tl, rq, &seqno[0]); mutex_unlock(&tl->mutex); if (err) { i915_request_add(rq); goto out; } pr_debug("seqno[0]:%08x, hwsp_offset:%08x\n", seqno[0], tl->hwsp_offset); err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[0]); if (err) { i915_request_add(rq); goto out; } hwsp_seqno[0] = tl->hwsp_seqno; mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING); err = intel_timeline_get_seqno(tl, rq, &seqno[1]); mutex_unlock(&tl->mutex); if (err) { i915_request_add(rq); goto out; } pr_debug("seqno[1]:%08x, hwsp_offset:%08x\n", seqno[1], tl->hwsp_offset); err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[1]); if (err) { i915_request_add(rq); goto out; } hwsp_seqno[1] = tl->hwsp_seqno; /* With wrap should come a new hwsp */ GEM_BUG_ON(seqno[1] >= seqno[0]); GEM_BUG_ON(hwsp_seqno[0] == hwsp_seqno[1]); i915_request_add(rq); if (i915_request_wait(rq, 0, HZ / 5) < 0) { pr_err("Wait for timeline writes timed out!\n"); err = -EIO; goto out; } if (READ_ONCE(*hwsp_seqno[0]) != seqno[0] || READ_ONCE(*hwsp_seqno[1]) != seqno[1]) { pr_err("Bad timeline values: found (%x, %x), expected (%x, %x)\n", *hwsp_seqno[0], *hwsp_seqno[1], seqno[0], seqno[1]); err = -EINVAL; goto out; } intel_gt_retire_requests(gt); /* recycle HWSP */ } out: if (igt_flush_test(gt->i915)) err = -EIO; intel_timeline_unpin(tl); out_free: intel_timeline_put(tl); return err; } static int emit_read_hwsp(struct i915_request *rq, u32 seqno, u32 hwsp, u32 *addr) { const u32 gpr = i915_mmio_reg_offset(GEN8_RING_CS_GPR(rq->engine->mmio_base, 0)); u32 *cs; cs = intel_ring_begin(rq, 12); if (IS_ERR(cs)) return PTR_ERR(cs); *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; *cs++ = *addr; *cs++ = 0; *cs++ = seqno; *addr += 4; *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | MI_USE_GGTT; *cs++ = gpr; *cs++ = hwsp; *cs++ = 0; *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; *cs++ = gpr; *cs++ = *addr; *cs++ = 0; *addr += 4; intel_ring_advance(rq, cs); return 0; } struct hwsp_watcher { struct i915_vma *vma; struct i915_request *rq; u32 addr; u32 *map; }; static bool cmp_lt(u32 a, u32 b) { return a < b; } static bool cmp_gte(u32 a, u32 b) { return a >= b; } static int setup_watcher(struct hwsp_watcher *w, struct intel_gt *gt, struct intel_timeline *tl) { struct drm_i915_gem_object *obj; struct i915_vma *vma; obj = i915_gem_object_create_internal(gt->i915, SZ_2M); if (IS_ERR(obj)) return PTR_ERR(obj); /* keep the same cache settings as timeline */ i915_gem_object_set_pat_index(obj, tl->hwsp_ggtt->obj->pat_index); w->map = i915_gem_object_pin_map_unlocked(obj, page_unmask_bits(tl->hwsp_ggtt->obj->mm.mapping)); if (IS_ERR(w->map)) { i915_gem_object_put(obj); return PTR_ERR(w->map); } vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0); if (IS_ERR(vma)) { i915_gem_object_put(obj); return PTR_ERR(vma); } w->vma = vma; w->addr = i915_ggtt_offset(vma); return 0; } static void switch_tl_lock(struct i915_request *from, struct i915_request *to) { /* some light mutex juggling required; think co-routines */ if (from) { lockdep_unpin_lock(&from->context->timeline->mutex, from->cookie); mutex_unlock(&from->context->timeline->mutex); } if (to) { mutex_lock(&to->context->timeline->mutex); to->cookie = lockdep_pin_lock(&to->context->timeline->mutex); } } static int create_watcher(struct hwsp_watcher *w, struct intel_engine_cs *engine, int ringsz) { struct intel_context *ce; ce = intel_context_create(engine); if (IS_ERR(ce)) return PTR_ERR(ce); ce->ring_size = ringsz; w->rq = intel_context_create_request(ce); intel_context_put(ce); if (IS_ERR(w->rq)) return PTR_ERR(w->rq); w->addr = i915_ggtt_offset(w->vma); switch_tl_lock(w->rq, NULL); return 0; } static int check_watcher(struct hwsp_watcher *w, const char *name, bool (*op)(u32 hwsp, u32 seqno)) { struct i915_request *rq = fetch_and_zero(&w->rq); u32 offset, end; int err; GEM_BUG_ON(w->addr - i915_ggtt_offset(w->vma) > w->vma->size); i915_request_get(rq); switch_tl_lock(NULL, rq); i915_request_add(rq); if (i915_request_wait(rq, 0, HZ) < 0) { err = -ETIME; goto out; } err = 0; offset = 0; end = (w->addr - i915_ggtt_offset(w->vma)) / sizeof(*w->map); while (offset < end) { if (!op(w->map[offset + 1], w->map[offset])) { pr_err("Watcher '%s' found HWSP value %x for seqno %x\n", name, w->map[offset + 1], w->map[offset]); err = -EINVAL; } offset += 2; } out: i915_request_put(rq); return err; } static void cleanup_watcher(struct hwsp_watcher *w) { if (w->rq) { switch_tl_lock(NULL, w->rq); i915_request_add(w->rq); } i915_vma_unpin_and_release(&w->vma, I915_VMA_RELEASE_MAP); } static bool retire_requests(struct intel_timeline *tl) { struct i915_request *rq, *rn; mutex_lock(&tl->mutex); list_for_each_entry_safe(rq, rn, &tl->requests, link) if (!i915_request_retire(rq)) break; mutex_unlock(&tl->mutex); return !i915_active_fence_isset(&tl->last_request); } static struct i915_request *wrap_timeline(struct i915_request *rq) { struct intel_context *ce = rq->context; struct intel_timeline *tl = ce->timeline; u32 seqno = rq->fence.seqno; while (tl->seqno >= seqno) { /* Cause a wrap */ i915_request_put(rq); rq = intel_context_create_request(ce); if (IS_ERR(rq)) return rq; i915_request_get(rq); i915_request_add(rq); } i915_request_put(rq); rq = i915_request_create(ce); if (IS_ERR(rq)) return rq; i915_request_get(rq); i915_request_add(rq); return rq; } static int live_hwsp_read(void *arg) { struct intel_gt *gt = arg; struct hwsp_watcher watcher[2] = {}; struct intel_engine_cs *engine; struct intel_timeline *tl; enum intel_engine_id id; int err = 0; int i; /* * If we take a reference to the HWSP for reading on the GPU, that * read may be arbitrarily delayed (either by foreign fence or * priority saturation) and a wrap can happen within 30 minutes. * When the GPU read is finally submitted it should be correct, * even across multiple wraps. */ if (GRAPHICS_VER(gt->i915) < 8) /* CS convenience [SRM/LRM] */ return 0; tl = intel_timeline_create(gt); if (IS_ERR(tl)) return PTR_ERR(tl); if (!tl->has_initial_breadcrumb) goto out_free; selftest_tl_pin(tl); for (i = 0; i < ARRAY_SIZE(watcher); i++) { err = setup_watcher(&watcher[i], gt, tl); if (err) goto out; } for_each_engine(engine, gt, id) { struct intel_context *ce; unsigned long count = 0; IGT_TIMEOUT(end_time); /* Create a request we can use for remote reading of the HWSP */ err = create_watcher(&watcher[1], engine, SZ_512K); if (err) goto out; do { struct i915_sw_fence *submit; struct i915_request *rq; u32 hwsp, dummy; submit = heap_fence_create(GFP_KERNEL); if (!submit) { err = -ENOMEM; goto out; } err = create_watcher(&watcher[0], engine, SZ_4K); if (err) goto out; ce = intel_context_create(engine); if (IS_ERR(ce)) { err = PTR_ERR(ce); goto out; } ce->timeline = intel_timeline_get(tl); /* Ensure timeline is mapped, done during first pin */ err = intel_context_pin(ce); if (err) { intel_context_put(ce); goto out; } /* * Start at a new wrap, and set seqno right before another wrap, * saving 30 minutes of nops */ tl->seqno = -12u + 2 * (count & 3); __intel_timeline_get_seqno(tl, &dummy); rq = i915_request_create(ce); if (IS_ERR(rq)) { err = PTR_ERR(rq); intel_context_unpin(ce); intel_context_put(ce); goto out; } err = i915_sw_fence_await_dma_fence(&rq->submit, &watcher[0].rq->fence, 0, GFP_KERNEL); if (err < 0) { i915_request_add(rq); intel_context_unpin(ce); intel_context_put(ce); goto out; } switch_tl_lock(rq, watcher[0].rq); err = intel_timeline_read_hwsp(rq, watcher[0].rq, &hwsp); if (err == 0) err = emit_read_hwsp(watcher[0].rq, /* before */ rq->fence.seqno, hwsp, &watcher[0].addr); switch_tl_lock(watcher[0].rq, rq); if (err) { i915_request_add(rq); intel_context_unpin(ce); intel_context_put(ce); goto out; } switch_tl_lock(rq, watcher[1].rq); err = intel_timeline_read_hwsp(rq, watcher[1].rq, &hwsp); if (err == 0) err = emit_read_hwsp(watcher[1].rq, /* after */ rq->fence.seqno, hwsp, &watcher[1].addr); switch_tl_lock(watcher[1].rq, rq); if (err) { i915_request_add(rq); intel_context_unpin(ce); intel_context_put(ce); goto out; } i915_request_get(rq); i915_request_add(rq); rq = wrap_timeline(rq); intel_context_unpin(ce); intel_context_put(ce); if (IS_ERR(rq)) { err = PTR_ERR(rq); goto out; } err = i915_sw_fence_await_dma_fence(&watcher[1].rq->submit, &rq->fence, 0, GFP_KERNEL); if (err < 0) { i915_request_put(rq); goto out; } err = check_watcher(&watcher[0], "before", cmp_lt); i915_sw_fence_commit(submit); heap_fence_put(submit); if (err) { i915_request_put(rq); goto out; } count++; /* Flush the timeline before manually wrapping again */ if (i915_request_wait(rq, I915_WAIT_INTERRUPTIBLE, HZ) < 0) { err = -ETIME; i915_request_put(rq); goto out; } retire_requests(tl); i915_request_put(rq); /* Single requests are limited to half a ring at most */ if (8 * watcher[1].rq->ring->emit > 3 * watcher[1].rq->ring->size) break; } while (!__igt_timeout(end_time, NULL) && count < (PAGE_SIZE / TIMELINE_SEQNO_BYTES - 1) / 2); pr_info("%s: simulated %lu wraps\n", engine->name, count); err = check_watcher(&watcher[1], "after", cmp_gte); if (err) goto out; } out: for (i = 0; i < ARRAY_SIZE(watcher); i++) cleanup_watcher(&watcher[i]); intel_timeline_unpin(tl); if (igt_flush_test(gt->i915)) err = -EIO; out_free: intel_timeline_put(tl); return err; } static int live_hwsp_rollover_kernel(void *arg) { struct intel_gt *gt = arg; struct intel_engine_cs *engine; enum intel_engine_id id; int err = 0; /* * Run the host for long enough, and even the kernel context will * see a seqno rollover. */ for_each_engine(engine, gt, id) { struct intel_context *ce = engine->kernel_context; struct intel_timeline *tl = ce->timeline; struct i915_request *rq[3] = {}; int i; st_engine_heartbeat_disable(engine); if (intel_gt_wait_for_idle(gt, HZ / 2)) { err = -EIO; goto out; } GEM_BUG_ON(i915_active_fence_isset(&tl->last_request)); tl->seqno = -2u; WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno); for (i = 0; i < ARRAY_SIZE(rq); i++) { struct i915_request *this; this = i915_request_create(ce); if (IS_ERR(this)) { err = PTR_ERR(this); goto out; } pr_debug("%s: create fence.seqnp:%d\n", engine->name, lower_32_bits(this->fence.seqno)); GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl); rq[i] = i915_request_get(this); i915_request_add(this); } /* We expected a wrap! */ GEM_BUG_ON(rq[2]->fence.seqno > rq[0]->fence.seqno); if (i915_request_wait(rq[2], 0, HZ / 5) < 0) { pr_err("Wait for timeline wrap timed out!\n"); err = -EIO; goto out; } for (i = 0; i < ARRAY_SIZE(rq); i++) { if (!i915_request_completed(rq[i])) { pr_err("Pre-wrap request not completed!\n"); err = -EINVAL; goto out; } } out: for (i = 0; i < ARRAY_SIZE(rq); i++) i915_request_put(rq[i]); st_engine_heartbeat_enable(engine); if (err) break; } if (igt_flush_test(gt->i915)) err = -EIO; return err; } static int live_hwsp_rollover_user(void *arg) { struct intel_gt *gt = arg; struct intel_engine_cs *engine; enum intel_engine_id id; int err = 0; /* * Simulate a long running user context, and force the seqno wrap * on the user's timeline. */ for_each_engine(engine, gt, id) { struct i915_request *rq[3] = {}; struct intel_timeline *tl; struct intel_context *ce; int i; ce = intel_context_create(engine); if (IS_ERR(ce)) return PTR_ERR(ce); err = intel_context_alloc_state(ce); if (err) goto out; tl = ce->timeline; if (!tl->has_initial_breadcrumb) goto out; err = intel_context_pin(ce); if (err) goto out; tl->seqno = -4u; WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno); for (i = 0; i < ARRAY_SIZE(rq); i++) { struct i915_request *this; this = intel_context_create_request(ce); if (IS_ERR(this)) { err = PTR_ERR(this); goto out_unpin; } pr_debug("%s: create fence.seqnp:%d\n", engine->name, lower_32_bits(this->fence.seqno)); GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl); rq[i] = i915_request_get(this); i915_request_add(this); } /* We expected a wrap! */ GEM_BUG_ON(rq[2]->fence.seqno > rq[0]->fence.seqno); if (i915_request_wait(rq[2], 0, HZ / 5) < 0) { pr_err("Wait for timeline wrap timed out!\n"); err = -EIO; goto out_unpin; } for (i = 0; i < ARRAY_SIZE(rq); i++) { if (!i915_request_completed(rq[i])) { pr_err("Pre-wrap request not completed!\n"); err = -EINVAL; goto out_unpin; } } out_unpin: intel_context_unpin(ce); out: for (i = 0; i < ARRAY_SIZE(rq); i++) i915_request_put(rq[i]); intel_context_put(ce); if (err) break; } if (igt_flush_test(gt->i915)) err = -EIO; return err; } static int live_hwsp_recycle(void *arg) { struct intel_gt *gt = arg; struct intel_engine_cs *engine; enum intel_engine_id id; unsigned long count; int err = 0; /* * Check seqno writes into one timeline at a time. We expect to * recycle the breadcrumb slot between iterations and neither * want to confuse ourselves or the GPU. */ count = 0; for_each_engine(engine, gt, id) { IGT_TIMEOUT(end_time); if (!intel_engine_can_store_dword(engine)) continue; intel_engine_pm_get(engine); do { struct intel_timeline *tl; struct i915_request *rq; tl = intel_timeline_create(gt); if (IS_ERR(tl)) { err = PTR_ERR(tl); break; } rq = checked_tl_write(tl, engine, count); if (IS_ERR(rq)) { intel_timeline_put(tl); err = PTR_ERR(rq); break; } if (i915_request_wait(rq, 0, HZ / 5) < 0) { pr_err("Wait for timeline writes timed out!\n"); i915_request_put(rq); intel_timeline_put(tl); err = -EIO; break; } if (READ_ONCE(*tl->hwsp_seqno) != count) { GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x found 0x%x\n", count, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno); GEM_TRACE_DUMP(); err = -EINVAL; } i915_request_put(rq); intel_timeline_put(tl); count++; if (err) break; } while (!__igt_timeout(end_time, NULL)); intel_engine_pm_put(engine); if (err) break; } return err; } int intel_timeline_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { SUBTEST(live_hwsp_recycle), SUBTEST(live_hwsp_engine), SUBTEST(live_hwsp_alternate), SUBTEST(live_hwsp_wrap), SUBTEST(live_hwsp_read), SUBTEST(live_hwsp_rollover_kernel), SUBTEST(live_hwsp_rollover_user), }; if (intel_gt_is_wedged(to_gt(i915))) return 0; return intel_gt_live_subtests(tests, to_gt(i915)); }
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1