Examples with Listener com.google.devtools.build.skyframe.NotifyingHelper.Listener used on opensource projects

Example 11 with Listener

use of com.google.devtools.build.skyframe.NotifyingHelper.Listener in project bazel by bazelbuild.

the class MemoizingEvaluatorTest method cleanReverseDepFromDirtyNodeNotInBuild.

@Test
public void cleanReverseDepFromDirtyNodeNotInBuild() throws Exception {
    final SkyKey topKey = GraphTester.skyKey("top");
    SkyKey inactiveKey = GraphTester.skyKey("inactive");
    final Thread mainThread = Thread.currentThread();
    final AtomicBoolean shouldInterrupt = new AtomicBoolean(false);
    injectGraphListenerForTesting(new Listener() {

        @Override
        public void accept(SkyKey key, EventType type, Order order, Object context) {
            if (shouldInterrupt.get() && key.equals(topKey) && type == EventType.IS_READY && order == Order.BEFORE) {
                mainThread.interrupt();
                shouldInterrupt.set(false);
                try {
                    // Make sure threadpool propagates interrupt.
                    Thread.sleep(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                }
            }
        }
    }, /*deterministic=*/
    false);
    // When top depends on inactive,
    tester.getOrCreate(topKey).addDependency(inactiveKey).setComputedValue(COPY);
    StringValue val = new StringValue("inactive");
    // And inactive is constant,
    tester.set(inactiveKey, val);
    // Then top evaluates normally.
    assertThat(tester.evalAndGet(/*keepGoing=*/
    true, topKey)).isEqualTo(val);
    // When evaluation will be interrupted as soon as top starts evaluating,
    shouldInterrupt.set(true);
    // And inactive is dirty,
    tester.getOrCreate(inactiveKey, /*markAsModified=*/
    true);
    // And so is top,
    tester.getOrCreate(topKey, /*markAsModified=*/
    true);
    tester.invalidate();
    try {
        // Then evaluation is interrupted,
        tester.eval(/*keepGoing=*/
        false, topKey);
        fail();
    } catch (InterruptedException e) {
    // Expected.
    }
    // But inactive is still present,
    assertThat(tester.driver.getEntryForTesting(inactiveKey)).isNotNull();
    // And still dirty,
    assertThat(tester.driver.getEntryForTesting(inactiveKey).isDirty()).isTrue();
    // And re-evaluates successfully,
    assertThat(tester.evalAndGet(/*keepGoing=*/
    true, inactiveKey)).isEqualTo(val);
    // But top is gone from the graph,
    assertThat(tester.driver.getEntryForTesting(topKey)).isNull();
    // And we can successfully invalidate and re-evaluate inactive again.
    tester.getOrCreate(inactiveKey, /*markAsModified=*/
    true);
    tester.invalidate();
    assertThat(tester.evalAndGet(/*keepGoing=*/
    true, inactiveKey)).isEqualTo(val);
}

Example 12 with Listener

use of com.google.devtools.build.skyframe.NotifyingHelper.Listener in project bazel by bazelbuild.

the class ParallelEvaluatorTest method raceConditionWithNoKeepGoingErrors_FutureError.

@Test
public void raceConditionWithNoKeepGoingErrors_FutureError() throws Exception {
    final CountDownLatch errorCommitted = new CountDownLatch(1);
    final CountDownLatch otherStarted = new CountDownLatch(1);
    final CountDownLatch otherParentSignaled = new CountDownLatch(1);
    final SkyKey errorParentKey = GraphTester.toSkyKey("errorParentKey");
    final SkyKey errorKey = GraphTester.toSkyKey("errorKey");
    final SkyKey otherParentKey = GraphTester.toSkyKey("otherParentKey");
    final SkyKey otherKey = GraphTester.toSkyKey("otherKey");
    final AtomicInteger numOtherParentInvocations = new AtomicInteger(0);
    final AtomicInteger numErrorParentInvocations = new AtomicInteger(0);
    tester.getOrCreate(otherParentKey).setBuilder(new SkyFunction() {

        @Override
        public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, InterruptedException {
            int invocations = numOtherParentInvocations.incrementAndGet();
            assertEquals("otherParentKey should not be restarted", 1, invocations);
            return env.getValue(otherKey);
        }

        @Override
        public String extractTag(SkyKey skyKey) {
            return null;
        }
    });
    tester.getOrCreate(otherKey).setBuilder(new SkyFunction() {

        @Override
        public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
            otherStarted.countDown();
            TrackingAwaiter.INSTANCE.awaitLatchAndTrackExceptions(errorCommitted, "error didn't get committed to the graph in time");
            return new StringValue("other");
        }

        @Override
        public String extractTag(SkyKey skyKey) {
            return null;
        }
    });
    tester.getOrCreate(errorKey).setBuilder(new SkyFunction() {

        @Override
        public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
            TrackingAwaiter.INSTANCE.awaitLatchAndTrackExceptions(otherStarted, "other didn't start in time");
            throw new GenericFunctionException(new SomeErrorException("error"), Transience.PERSISTENT);
        }

        @Override
        public String extractTag(SkyKey skyKey) {
            return null;
        }
    });
    tester.getOrCreate(errorParentKey).setBuilder(new SkyFunction() {

        @Override
        public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, InterruptedException {
            int invocations = numErrorParentInvocations.incrementAndGet();
            try {
                SkyValue value = env.getValueOrThrow(errorKey, SomeErrorException.class);
                assertTrue("bogus non-null value " + value, value == null);
                if (invocations == 1) {
                    return null;
                } else {
                    assertFalse(env.inErrorBubblingForTesting());
                    fail("RACE CONDITION: errorParentKey was restarted!");
                    return null;
                }
            } catch (SomeErrorException e) {
                assertTrue("child error propagated during normal evaluation", env.inErrorBubblingForTesting());
                assertEquals(2, invocations);
                return null;
            }
        }

        @Override
        public String extractTag(SkyKey skyKey) {
            return null;
        }
    });
    graph = new NotifyingHelper.NotifyingProcessableGraph(new InMemoryGraphImpl(), new Listener() {

        @Override
        public void accept(SkyKey key, EventType type, Order order, Object context) {
            if (key.equals(errorKey) && type == EventType.SET_VALUE && order == Order.AFTER) {
                errorCommitted.countDown();
                TrackingAwaiter.INSTANCE.awaitLatchAndTrackExceptions(otherParentSignaled, "otherParent didn't get signaled in time");
                // We try to give some time for ParallelEvaluator to incorrectly re-evaluate
                // 'otherParentKey'. This test case is testing for a real race condition and the
                // 10ms time was chosen experimentally to give a true positive rate of 99.8%
                // (without a sleep it has a 1% true positive rate). There's no good way to do
                // this without sleeping. We *could* introspect ParallelEvaulator's
                // AbstractQueueVisitor to see if the re-evaluation has been enqueued, but that's
                // relying on pretty low-level implementation details.
                Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS);
            }
            if (key.equals(otherParentKey) && type == EventType.SIGNAL && order == Order.AFTER) {
                otherParentSignaled.countDown();
            }
        }
    });
    EvaluationResult<StringValue> result = eval(/*keepGoing=*/
    false, ImmutableList.of(otherParentKey, errorParentKey));
    assertTrue(result.hasError());
    assertEquals(errorKey, result.getError().getRootCauseOfException());
}

Example 13 with Listener

use of com.google.devtools.build.skyframe.NotifyingHelper.Listener in project bazel by bazelbuild.

the class MemoizingEvaluatorTest method alreadyAnalyzedBadTarget.

// ParallelEvaluator notifies ValueProgressReceiver of already-built top-level values in error: we
// built "top" and "mid" as top-level targets; "mid" contains an error. We make sure "mid" is
// built as a dependency of "top" before enqueuing mid as a top-level target (by using a latch),
// so that the top-level enqueuing finds that mid has already been built. The progress receiver
// should be notified that mid has been built.
@Test
public void alreadyAnalyzedBadTarget() throws Exception {
    final SkyKey mid = GraphTester.toSkyKey("zzmid");
    final CountDownLatch valueSet = new CountDownLatch(1);
    injectGraphListenerForTesting(new Listener() {

        @Override
        public void accept(SkyKey key, EventType type, Order order, Object context) {
            if (!key.equals(mid)) {
                return;
            }
            switch(type) {
                case ADD_REVERSE_DEP:
                    if (context == null) {
                        // Context is null when we are enqueuing this value as a top-level job.
                        TrackingAwaiter.INSTANCE.awaitLatchAndTrackExceptions(valueSet, "value not set");
                    }
                    break;
                case SET_VALUE:
                    valueSet.countDown();
                    break;
                default:
                    break;
            }
        }
    }, /*deterministic=*/
    true);
    SkyKey top = GraphTester.skyKey("aatop");
    tester.getOrCreate(top).addDependency(mid).setComputedValue(CONCATENATE);
    tester.getOrCreate(mid).setHasError(true);
    tester.eval(/*keepGoing=*/
    false, top, mid);
    assertEquals(0L, valueSet.getCount());
    assertThat(tester.progressReceiver.evaluated).containsExactly(mid);
}

Example 14 with Listener

use of com.google.devtools.build.skyframe.NotifyingHelper.Listener in project bazel by bazelbuild.

the class MemoizingEvaluatorTest method removeReverseDepFromRebuildingNode.

@Test
public void removeReverseDepFromRebuildingNode() throws Exception {
    SkyKey topKey = GraphTester.skyKey("top");
    final SkyKey midKey = GraphTester.skyKey("mid");
    final SkyKey changedKey = GraphTester.skyKey("changed");
    tester.getOrCreate(changedKey).setConstantValue(new StringValue("first"));
    // When top depends on mid,
    tester.getOrCreate(topKey).addDependency(midKey).setComputedValue(CONCATENATE);
    // And mid depends on changed,
    tester.getOrCreate(midKey).addDependency(changedKey).setComputedValue(CONCATENATE);
    final CountDownLatch changedKeyStarted = new CountDownLatch(1);
    final CountDownLatch changedKeyCanFinish = new CountDownLatch(1);
    final AtomicBoolean controlTiming = new AtomicBoolean(false);
    injectGraphListenerForTesting(new Listener() {

        @Override
        public void accept(SkyKey key, EventType type, Order order, Object context) {
            if (!controlTiming.get()) {
                return;
            }
            if (key.equals(midKey) && type == EventType.CHECK_IF_DONE && order == Order.BEFORE) {
                TrackingAwaiter.INSTANCE.awaitLatchAndTrackExceptions(changedKeyStarted, "changed key didn't start");
            } else if (key.equals(changedKey) && type == EventType.REMOVE_REVERSE_DEP && order == Order.AFTER && midKey.equals(context)) {
                changedKeyCanFinish.countDown();
            }
        }
    }, /*deterministic=*/
    false);
    // Then top builds as expected.
    assertThat(tester.evalAndGet(/*keepGoing=*/
    false, topKey)).isEqualTo(new StringValue("first"));
    // When changed is modified,
    tester.getOrCreate(changedKey, /*markAsModified=*/
    true).setConstantValue(null).setBuilder(// And changed is not allowed to finish building until it is released,
    new ChainedFunction(changedKeyStarted, changedKeyCanFinish, null, false, new StringValue("second"), ImmutableList.<SkyKey>of()));
    // And mid is independently marked as modified,
    tester.getOrCreate(midKey, /*markAsModified=*/
    true).removeDependency(changedKey).setComputedValue(null).setConstantValue(new StringValue("mid"));
    tester.invalidate();
    SkyKey newTopKey = GraphTester.skyKey("newTop");
    // And changed will start rebuilding independently of midKey, because it's requested directly by
    // newTop
    tester.getOrCreate(newTopKey).addDependency(changedKey).setComputedValue(CONCATENATE);
    // And we control the timing using the graph listener above to make sure that:
    // (1) before we do anything with mid, changed has already started, and
    // (2) changed key can't finish until mid tries to remove its reverse dep from changed,
    controlTiming.set(true);
    // Then this evaluation completes without crashing.
    tester.eval(/*keepGoing=*/
    false, newTopKey, topKey);
}

Example 15 with Listener

use of com.google.devtools.build.skyframe.NotifyingHelper.Listener in project bazel by bazelbuild.

the class MemoizingEvaluatorTest method shutDownBuildOnCachedError_Done.

/**
   * The following two tests check that the evaluator shuts down properly when encountering an error
   * that is marked dirty but later verified to be unchanged from a prior build. In that case, the
   * invariant that its parents are not enqueued for evaluation should be maintained.
   */
/**
   * Test that a parent of a cached but invalidated error doesn't successfully build. First build
   * the error. Then invalidate the error via a dependency (so it will not actually change) and
   * build two new parents. Parent A will request error and abort since error isn't done yet. error
   * is then revalidated, and A is restarted. If A does not throw upon encountering the error, and
   * instead sets its value, then we throw in parent B, which waits for error to be done before
   * requesting it. Then there will be the impossible situation of a node that was built during this
   * evaluation depending on a node in error.
   */
@Test
public void shutDownBuildOnCachedError_Done() throws Exception {
    // errorKey will be invalidated due to its dependence on invalidatedKey, but later revalidated
    // since invalidatedKey re-evaluates to the same value on a subsequent build.
    final SkyKey errorKey = GraphTester.toSkyKey("error");
    SkyKey invalidatedKey = GraphTester.toSkyKey("invalidated-leaf");
    tester.set(invalidatedKey, new StringValue("invalidated-leaf-value"));
    tester.getOrCreate(errorKey).addDependency(invalidatedKey).setHasError(true);
    // Names are alphabetized in reverse deps of errorKey.
    final SkyKey fastToRequestSlowToSetValueKey = GraphTester.toSkyKey("A-slow-set-value-parent");
    final SkyKey failingKey = GraphTester.toSkyKey("B-fast-fail-parent");
    tester.getOrCreate(fastToRequestSlowToSetValueKey).addDependency(errorKey).setComputedValue(CONCATENATE);
    tester.getOrCreate(failingKey).addDependency(errorKey).setComputedValue(CONCATENATE);
    // We only want to force a particular order of operations at some points during evaluation.
    final AtomicBoolean synchronizeThreads = new AtomicBoolean(false);
    // We don't expect slow-set-value to actually be built, but if it is, we wait for it.
    final CountDownLatch slowBuilt = new CountDownLatch(1);
    injectGraphListenerForTesting(new Listener() {

        @Override
        public void accept(SkyKey key, EventType type, Order order, Object context) {
            if (!synchronizeThreads.get()) {
                return;
            }
            if (type == EventType.IS_DIRTY && key.equals(failingKey)) {
                // Wait for the build to abort or for the other node to incorrectly build.
                try {
                    assertTrue(slowBuilt.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
                } catch (InterruptedException e) {
                    // This is ok, because it indicates the build is shutting down.
                    Thread.currentThread().interrupt();
                }
            } else if (type == EventType.SET_VALUE && key.equals(fastToRequestSlowToSetValueKey) && order == Order.AFTER) {
                // This indicates a problem -- this parent shouldn't be built since it depends on
                // an error.
                slowBuilt.countDown();
                // for the other node to throw an exception.
                try {
                    Thread.sleep(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
                    throw new IllegalStateException("uninterrupted in " + key);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                }
            }
        }
    }, /*deterministic=*/
    true);
    // Initialize graph.
    tester.eval(/*keepGoing=*/
    true, errorKey);
    tester.getOrCreate(invalidatedKey, /*markAsModified=*/
    true);
    tester.invalidate();
    synchronizeThreads.set(true);
    tester.eval(/*keepGoing=*/
    false, fastToRequestSlowToSetValueKey, failingKey);
}