471 lines
11 KiB
C
471 lines
11 KiB
C
|
/*
|
||
|
|
* Copyright © 2016 Intel Corporation
|
||
|
|
*
|
||
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
||
|
|
* copy of this software and associated documentation files (the "Software"),
|
||
|
|
* to deal in the Software without restriction, including without limitation
|
||
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
||
|
|
* Software is furnished to do so, subject to the following conditions:
|
||
|
|
*
|
||
|
|
* The above copyright notice and this permission notice (including the next
|
||
|
|
* paragraph) shall be included in all copies or substantial portions of the
|
||
|
|
* Software.
|
||
|
|
*
|
||
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||
|
|
* IN THE SOFTWARE.
|
||
|
|
*
|
||
|
|
*/
|
||
|
|
|
||
|
|
#include "../i915_selftest.h"
|
||
|
|
#include "i915_random.h"
|
||
|
|
|
||
|
|
#include "mock_gem_device.h"
|
||
|
|
#include "mock_engine.h"
|
||
|
|
|
||
|
|
static int check_rbtree(struct intel_engine_cs *engine,
|
||
|
|
const unsigned long *bitmap,
|
||
|
|
const struct intel_wait *waiters,
|
||
|
|
const int count)
|
||
|
|
{
|
||
|
|
struct intel_breadcrumbs *b = &engine->breadcrumbs;
|
||
|
|
struct rb_node *rb;
|
||
|
|
int n;
|
||
|
|
|
||
|
|
if (&b->irq_wait->node != rb_first(&b->waiters)) {
|
||
|
|
pr_err("First waiter does not match first element of wait-tree\n");
|
||
|
|
return -EINVAL;
|
||
|
|
}
|
||
|
|
|
||
|
|
n = find_first_bit(bitmap, count);
|
||
|
|
for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
|
||
|
|
struct intel_wait *w = container_of(rb, typeof(*w), node);
|
||
|
|
int idx = w - waiters;
|
||
|
|
|
||
|
|
if (!test_bit(idx, bitmap)) {
|
||
|
|
pr_err("waiter[%d, seqno=%d] removed but still in wait-tree\n",
|
||
|
|
idx, w->seqno);
|
||
|
|
return -EINVAL;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (n != idx) {
|
||
|
|
pr_err("waiter[%d, seqno=%d] does not match expected next element in tree [%d]\n",
|
||
|
|
idx, w->seqno, n);
|
||
|
|
return -EINVAL;
|
||
|
|
}
|
||
|
|
|
||
|
|
n = find_next_bit(bitmap, count, n + 1);
|
||
|
|
}
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
static int check_completion(struct intel_engine_cs *engine,
|
||
|
|
const unsigned long *bitmap,
|
||
|
|
const struct intel_wait *waiters,
|
||
|
|
const int count)
|
||
|
|
{
|
||
|
|
int n;
|
||
|
|
|
||
|
|
for (n = 0; n < count; n++) {
|
||
|
|
if (intel_wait_complete(&waiters[n]) != !!test_bit(n, bitmap))
|
||
|
|
continue;
|
||
|
|
|
||
|
|
pr_err("waiter[%d, seqno=%d] is %s, but expected %s\n",
|
||
|
|
n, waiters[n].seqno,
|
||
|
|
intel_wait_complete(&waiters[n]) ? "complete" : "active",
|
||
|
|
test_bit(n, bitmap) ? "active" : "complete");
|
||
|
|
return -EINVAL;
|
||
|
|
}
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
static int check_rbtree_empty(struct intel_engine_cs *engine)
|
||
|
|
{
|
||
|
|
struct intel_breadcrumbs *b = &engine->breadcrumbs;
|
||
|
|
|
||
|
|
if (b->irq_wait) {
|
||
|
|
pr_err("Empty breadcrumbs still has a waiter\n");
|
||
|
|
return -EINVAL;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!RB_EMPTY_ROOT(&b->waiters)) {
|
||
|
|
pr_err("Empty breadcrumbs, but wait-tree not empty\n");
|
||
|
|
return -EINVAL;
|
||
|
|
}
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
static int igt_random_insert_remove(void *arg)
|
||
|
|
{
|
||
|
|
const u32 seqno_bias = 0x1000;
|
||
|
|
I915_RND_STATE(prng);
|
||
|
|
struct intel_engine_cs *engine = arg;
|
||
|
|
struct intel_wait *waiters;
|
||
|
|
const int count = 4096;
|
||
|
|
unsigned int *order;
|
||
|
|
unsigned long *bitmap;
|
||
|
|
int err = -ENOMEM;
|
||
|
|
int n;
|
||
|
|
|
||
|
|
mock_engine_reset(engine);
|
||
|
|
|
||
|
|
waiters = kvmalloc_array(count, sizeof(*waiters), GFP_KERNEL);
|
||
|
|
if (!waiters)
|
||
|
|
goto out_engines;
|
||
|
|
|
||
|
|
bitmap = kcalloc(DIV_ROUND_UP(count, BITS_PER_LONG), sizeof(*bitmap),
|
||
|
|
GFP_KERNEL);
|
||
|
|
if (!bitmap)
|
||
|
|
goto out_waiters;
|
||
|
|
|
||
|
|
order = i915_random_order(count, &prng);
|
||
|
|
if (!order)
|
||
|
|
goto out_bitmap;
|
||
|
|
|
||
|
|
for (n = 0; n < count; n++)
|
||
|
|
intel_wait_init_for_seqno(&waiters[n], seqno_bias + n);
|
||
|
|
|
||
|
|
err = check_rbtree(engine, bitmap, waiters, count);
|
||
|
|
if (err)
|
||
|
|
goto out_order;
|
||
|
|
|
||
|
|
/* Add and remove waiters into the rbtree in random order. At each
|
||
|
|
* step, we verify that the rbtree is correctly ordered.
|
||
|
|
*/
|
||
|
|
for (n = 0; n < count; n++) {
|
||
|
|
int i = order[n];
|
||
|
|
|
||
|
|
intel_engine_add_wait(engine, &waiters[i]);
|
||
|
|
__set_bit(i, bitmap);
|
||
|
|
|
||
|
|
err = check_rbtree(engine, bitmap, waiters, count);
|
||
|
|
if (err)
|
||
|
|
goto out_order;
|
||
|
|
}
|
||
|
|
|
||
|
|
i915_random_reorder(order, count, &prng);
|
||
|
|
for (n = 0; n < count; n++) {
|
||
|
|
int i = order[n];
|
||
|
|
|
||
|
|
intel_engine_remove_wait(engine, &waiters[i]);
|
||
|
|
__clear_bit(i, bitmap);
|
||
|
|
|
||
|
|
err = check_rbtree(engine, bitmap, waiters, count);
|
||
|
|
if (err)
|
||
|
|
goto out_order;
|
||
|
|
}
|
||
|
|
|
||
|
|
err = check_rbtree_empty(engine);
|
||
|
|
out_order:
|
||
|
|
kfree(order);
|
||
|
|
out_bitmap:
|
||
|
|
kfree(bitmap);
|
||
|
|
out_waiters:
|
||
|
|
kvfree(waiters);
|
||
|
|
out_engines:
|
||
|
|
mock_engine_flush(engine);
|
||
|
|
return err;
|
||
|
|
}
|
||
|
|
|
||
|
|
static int igt_insert_complete(void *arg)
|
||
|
|
{
|
||
|
|
const u32 seqno_bias = 0x1000;
|
||
|
|
struct intel_engine_cs *engine = arg;
|
||
|
|
struct intel_wait *waiters;
|
||
|
|
const int count = 4096;
|
||
|
|
unsigned long *bitmap;
|
||
|
|
int err = -ENOMEM;
|
||
|
|
int n, m;
|
||
|
|
|
||
|
|
mock_engine_reset(engine);
|
||
|
|
|
||
|
|
waiters = kvmalloc_array(count, sizeof(*waiters), GFP_KERNEL);
|
||
|
|
if (!waiters)
|
||
|
|
goto out_engines;
|
||
|
|
|
||
|
|
bitmap = kcalloc(DIV_ROUND_UP(count, BITS_PER_LONG), sizeof(*bitmap),
|
||
|
|
GFP_KERNEL);
|
||
|
|
if (!bitmap)
|
||
|
|
goto out_waiters;
|
||
|
|
|
||
|
|
for (n = 0; n < count; n++) {
|
||
|
|
intel_wait_init_for_seqno(&waiters[n], n + seqno_bias);
|
||
|
|
intel_engine_add_wait(engine, &waiters[n]);
|
||
|
|
__set_bit(n, bitmap);
|
||
|
|
}
|
||
|
|
err = check_rbtree(engine, bitmap, waiters, count);
|
||
|
|
if (err)
|
||
|
|
goto out_bitmap;
|
||
|
|
|
||
|
|
/* On each step, we advance the seqno so that several waiters are then
|
||
|
|
* complete (we increase the seqno by increasingly larger values to
|
||
|
|
* retire more and more waiters at once). All retired waiters should
|
||
|
|
* be woken and removed from the rbtree, and so that we check.
|
||
|
|
*/
|
||
|
|
for (n = 0; n < count; n = m) {
|
||
|
|
int seqno = 2 * n;
|
||
|
|
|
||
|
|
GEM_BUG_ON(find_first_bit(bitmap, count) != n);
|
||
|
|
|
||
|
|
if (intel_wait_complete(&waiters[n])) {
|
||
|
|
pr_err("waiter[%d, seqno=%d] completed too early\n",
|
||
|
|
n, waiters[n].seqno);
|
||
|
|
err = -EINVAL;
|
||
|
|
goto out_bitmap;
|
||
|
|
}
|
||
|
|
|
||
|
|
/* complete the following waiters */
|
||
|
|
mock_seqno_advance(engine, seqno + seqno_bias);
|
||
|
|
for (m = n; m <= seqno; m++) {
|
||
|
|
if (m == count)
|
||
|
|
break;
|
||
|
|
|
||
|
|
GEM_BUG_ON(!test_bit(m, bitmap));
|
||
|
|
__clear_bit(m, bitmap);
|
||
|
|
}
|
||
|
|
|
||
|
|
intel_engine_remove_wait(engine, &waiters[n]);
|
||
|
|
RB_CLEAR_NODE(&waiters[n].node);
|
||
|
|
|
||
|
|
err = check_rbtree(engine, bitmap, waiters, count);
|
||
|
|
if (err) {
|
||
|
|
pr_err("rbtree corrupt after seqno advance to %d\n",
|
||
|
|
seqno + seqno_bias);
|
||
|
|
goto out_bitmap;
|
||
|
|
}
|
||
|
|
|
||
|
|
err = check_completion(engine, bitmap, waiters, count);
|
||
|
|
if (err) {
|
||
|
|
pr_err("completions after seqno advance to %d failed\n",
|
||
|
|
seqno + seqno_bias);
|
||
|
|
goto out_bitmap;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
err = check_rbtree_empty(engine);
|
||
|
|
out_bitmap:
|
||
|
|
kfree(bitmap);
|
||
|
|
out_waiters:
|
||
|
|
kvfree(waiters);
|
||
|
|
out_engines:
|
||
|
|
mock_engine_flush(engine);
|
||
|
|
return err;
|
||
|
|
}
|
||
|
|
|
||
|
|
struct igt_wakeup {
|
||
|
|
struct task_struct *tsk;
|
||
|
|
atomic_t *ready, *set, *done;
|
||
|
|
struct intel_engine_cs *engine;
|
||
|
|
unsigned long flags;
|
||
|
|
#define STOP 0
|
||
|
|
#define IDLE 1
|
||
|
|
wait_queue_head_t *wq;
|
||
|
|
u32 seqno;
|
||
|
|
};
|
||
|
|
|
||
|
|
static bool wait_for_ready(struct igt_wakeup *w)
|
||
|
|
{
|
||
|
|
DEFINE_WAIT(ready);
|
||
|
|
|
||
|
|
set_bit(IDLE, &w->flags);
|
||
|
|
if (atomic_dec_and_test(w->done))
|
||
|
|
wake_up_var(w->done);
|
||
|
|
|
||
|
|
if (test_bit(STOP, &w->flags))
|
||
|
|
goto out;
|
||
|
|
|
||
|
|
for (;;) {
|
||
|
|
prepare_to_wait(w->wq, &ready, TASK_INTERRUPTIBLE);
|
||
|
|
if (atomic_read(w->ready) == 0)
|
||
|
|
break;
|
||
|
|
|
||
|
|
schedule();
|
||
|
|
}
|
||
|
|
finish_wait(w->wq, &ready);
|
||
|
|
|
||
|
|
out:
|
||
|
|
clear_bit(IDLE, &w->flags);
|
||
|
|
if (atomic_dec_and_test(w->set))
|
||
|
|
wake_up_var(w->set);
|
||
|
|
|
||
|
|
return !test_bit(STOP, &w->flags);
|
||
|
|
}
|
||
|
|
|
||
|
|
static int igt_wakeup_thread(void *arg)
|
||
|
|
{
|
||
|
|
struct igt_wakeup *w = arg;
|
||
|
|
struct intel_wait wait;
|
||
|
|
|
||
|
|
while (wait_for_ready(w)) {
|
||
|
|
GEM_BUG_ON(kthread_should_stop());
|
||
|
|
|
||
|
|
intel_wait_init_for_seqno(&wait, w->seqno);
|
||
|
|
intel_engine_add_wait(w->engine, &wait);
|
||
|
|
for (;;) {
|
||
|
|
set_current_state(TASK_UNINTERRUPTIBLE);
|
||
|
|
if (i915_seqno_passed(intel_engine_get_seqno(w->engine),
|
||
|
|
w->seqno))
|
||
|
|
break;
|
||
|
|
|
||
|
|
if (test_bit(STOP, &w->flags)) /* emergency escape */
|
||
|
|
break;
|
||
|
|
|
||
|
|
schedule();
|
||
|
|
}
|
||
|
|
intel_engine_remove_wait(w->engine, &wait);
|
||
|
|
__set_current_state(TASK_RUNNING);
|
||
|
|
}
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
static void igt_wake_all_sync(atomic_t *ready,
|
||
|
|
atomic_t *set,
|
||
|
|
atomic_t *done,
|
||
|
|
wait_queue_head_t *wq,
|
||
|
|
int count)
|
||
|
|
{
|
||
|
|
atomic_set(set, count);
|
||
|
|
atomic_set(ready, 0);
|
||
|
|
wake_up_all(wq);
|
||
|
|
|
||
|
|
wait_var_event(set, !atomic_read(set));
|
||
|
|
atomic_set(ready, count);
|
||
|
|
atomic_set(done, count);
|
||
|
|
}
|
||
|
|
|
||
|
|
static int igt_wakeup(void *arg)
|
||
|
|
{
|
||
|
|
I915_RND_STATE(prng);
|
||
|
|
struct intel_engine_cs *engine = arg;
|
||
|
|
struct igt_wakeup *waiters;
|
||
|
|
DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
|
||
|
|
const int count = 4096;
|
||
|
|
const u32 max_seqno = count / 4;
|
||
|
|
atomic_t ready, set, done;
|
||
|
|
int err = -ENOMEM;
|
||
|
|
int n, step;
|
||
|
|
|
||
|
|
mock_engine_reset(engine);
|
||
|
|
|
||
|
|
waiters = kvmalloc_array(count, sizeof(*waiters), GFP_KERNEL);
|
||
|
|
if (!waiters)
|
||
|
|
goto out_engines;
|
||
|
|
|
||
|
|
/* Create a large number of threads, each waiting on a random seqno.
|
||
|
|
* Multiple waiters will be waiting for the same seqno.
|
||
|
|
*/
|
||
|
|
atomic_set(&ready, count);
|
||
|
|
for (n = 0; n < count; n++) {
|
||
|
|
waiters[n].wq = &wq;
|
||
|
|
waiters[n].ready = &ready;
|
||
|
|
waiters[n].set = &set;
|
||
|
|
waiters[n].done = &done;
|
||
|
|
waiters[n].engine = engine;
|
||
|
|
waiters[n].flags = BIT(IDLE);
|
||
|
|
|
||
|
|
waiters[n].tsk = kthread_run(igt_wakeup_thread, &waiters[n],
|
||
|
|
"i915/igt:%d", n);
|
||
|
|
if (IS_ERR(waiters[n].tsk))
|
||
|
|
goto out_waiters;
|
||
|
|
|
||
|
|
get_task_struct(waiters[n].tsk);
|
||
|
|
}
|
||
|
|
|
||
|
|
for (step = 1; step <= max_seqno; step <<= 1) {
|
||
|
|
u32 seqno;
|
||
|
|
|
||
|
|
/* The waiter threads start paused as we assign them a random
|
||
|
|
* seqno and reset the engine. Once the engine is reset,
|
||
|
|
* we signal that the threads may begin their wait upon their
|
||
|
|
* seqno.
|
||
|
|
*/
|
||
|
|
for (n = 0; n < count; n++) {
|
||
|
|
GEM_BUG_ON(!test_bit(IDLE, &waiters[n].flags));
|
||
|
|
waiters[n].seqno =
|
||
|
|
1 + prandom_u32_state(&prng) % max_seqno;
|
||
|
|
}
|
||
|
|
mock_seqno_advance(engine, 0);
|
||
|
|
igt_wake_all_sync(&ready, &set, &done, &wq, count);
|
||
|
|
|
||
|
|
/* Simulate the GPU doing chunks of work, with one or more
|
||
|
|
* seqno appearing to finish at the same time. A random number
|
||
|
|
* of threads will be waiting upon the update and hopefully be
|
||
|
|
* woken.
|
||
|
|
*/
|
||
|
|
for (seqno = 1; seqno <= max_seqno + step; seqno += step) {
|
||
|
|
usleep_range(50, 500);
|
||
|
|
mock_seqno_advance(engine, seqno);
|
||
|
|
}
|
||
|
|
GEM_BUG_ON(intel_engine_get_seqno(engine) < 1 + max_seqno);
|
||
|
|
|
||
|
|
/* With the seqno now beyond any of the waiting threads, they
|
||
|
|
* should all be woken, see that they are complete and signal
|
||
|
|
* that they are ready for the next test. We wait until all
|
||
|
|
* threads are complete and waiting for us (i.e. not a seqno).
|
||
|
|
*/
|
||
|
|
if (!wait_var_event_timeout(&done,
|
||
|
|
!atomic_read(&done), 10 * HZ)) {
|
||
|
|
pr_err("Timed out waiting for %d remaining waiters\n",
|
||
|
|
atomic_read(&done));
|
||
|
|
err = -ETIMEDOUT;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
err = check_rbtree_empty(engine);
|
||
|
|
if (err)
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
out_waiters:
|
||
|
|
for (n = 0; n < count; n++) {
|
||
|
|
if (IS_ERR(waiters[n].tsk))
|
||
|
|
break;
|
||
|
|
|
||
|
|
set_bit(STOP, &waiters[n].flags);
|
||
|
|
}
|
||
|
|
mock_seqno_advance(engine, INT_MAX); /* wakeup any broken waiters */
|
||
|
|
igt_wake_all_sync(&ready, &set, &done, &wq, n);
|
||
|
|
|
||
|
|
for (n = 0; n < count; n++) {
|
||
|
|
if (IS_ERR(waiters[n].tsk))
|
||
|
|
break;
|
||
|
|
|
||
|
|
kthread_stop(waiters[n].tsk);
|
||
|
|
put_task_struct(waiters[n].tsk);
|
||
|
|
}
|
||
|
|
|
||
|
|
kvfree(waiters);
|
||
|
|
out_engines:
|
||
|
|
mock_engine_flush(engine);
|
||
|
|
return err;
|
||
|
|
}
|
||
|
|
|
||
|
|
int intel_breadcrumbs_mock_selftests(void)
|
||
|
|
{
|
||
|
|
static const struct i915_subtest tests[] = {
|
||
|
|
SUBTEST(igt_random_insert_remove),
|
||
|
|
SUBTEST(igt_insert_complete),
|
||
|
|
SUBTEST(igt_wakeup),
|
||
|
|
};
|
||
|
|
struct drm_i915_private *i915;
|
||
|
|
int err;
|
||
|
|
|
||
|
|
i915 = mock_gem_device();
|
||
|
|
if (!i915)
|
||
|
|
return -ENOMEM;
|
||
|
|
|
||
|
|
err = i915_subtests(tests, i915->engine[RCS]);
|
||
|
|
drm_dev_put(&i915->drm);
|
||
|
|
|
||
|
|
return err;
|
||
|
|
}
|