Examples with Listener com.google.devtools.build.skyframe.NotifyingHelper.Listener used on opensource projects

Example 6 with Listener

use of com.google.devtools.build.skyframe.NotifyingHelper.Listener in project bazel by bazelbuild.

the class MemoizingEvaluatorTest method sameDepInTwoGroups.

/**
   * The same dep is requested in two groups, but its value determines what the other dep in the
   * second group is. When it changes, the other dep in the second group should not be requested.
   */
@Test
public void sameDepInTwoGroups() throws Exception {
    initializeTester();
    // leaf4 should not built in the second build.
    final SkyKey leaf4 = GraphTester.toSkyKey("leaf4");
    final AtomicBoolean shouldNotBuildLeaf4 = new AtomicBoolean(false);
    injectGraphListenerForTesting(new Listener() {

        @Override
        public void accept(SkyKey key, EventType type, Order order, Object context) {
            if (shouldNotBuildLeaf4.get() && key.equals(leaf4) && type != EventType.REMOVE_REVERSE_DEP) {
                throw new IllegalStateException("leaf4 should not have been considered this build: " + type + ", " + order + ", " + context);
            }
        }
    }, /*deterministic=*/
    false);
    tester.set(leaf4, new StringValue("leaf4"));
    // Create leaf0, leaf1 and leaf2 values with values "leaf2", "leaf3", "leaf4" respectively.
    // These will be requested as one dependency group. In the second build, leaf2 will have the
    // value "leaf5".
    final List<SkyKey> leaves = new ArrayList<>();
    for (int i = 0; i <= 2; i++) {
        SkyKey leaf = GraphTester.toSkyKey("leaf" + i);
        leaves.add(leaf);
        tester.set(leaf, new StringValue("leaf" + (i + 2)));
    }
    // Create "top" value. It depends on all leaf values in two overlapping dependency groups.
    SkyKey topKey = GraphTester.toSkyKey("top");
    final SkyValue topValue = new StringValue("top");
    tester.getOrCreate(topKey).setBuilder(new NoExtractorFunction() {

        @Override
        public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, InterruptedException {
            // Request the first group, [leaf0, leaf1, leaf2].
            // In the first build, it has values ["leaf2", "leaf3", "leaf4"].
            // In the second build it has values ["leaf2", "leaf3", "leaf5"]
            Map<SkyKey, SkyValue> values = env.getValues(leaves);
            if (env.valuesMissing()) {
                return null;
            }
            // Request the second group. In the first build it's [leaf2, leaf4].
            // In the second build it's [leaf2, leaf5]
            env.getValues(ImmutableList.of(leaves.get(2), GraphTester.toSkyKey(((StringValue) values.get(leaves.get(2))).getValue())));
            if (env.valuesMissing()) {
                return null;
            }
            return topValue;
        }
    });
    // First build: assert we can evaluate "top".
    assertEquals(topValue, tester.evalAndGet(/*keepGoing=*/
    false, topKey));
    // Second build: replace "leaf4" by "leaf5" in leaf2's value. Assert leaf4 is not requested.
    final SkyKey leaf5 = GraphTester.toSkyKey("leaf5");
    tester.set(leaf5, new StringValue("leaf5"));
    tester.set(leaves.get(2), new StringValue("leaf5"));
    tester.invalidate();
    shouldNotBuildLeaf4.set(true);
    assertEquals(topValue, tester.evalAndGet(/*keepGoing=*/
    false, topKey));
}

Example 7 with Listener

use of com.google.devtools.build.skyframe.NotifyingHelper.Listener in project bazel by bazelbuild.

the class BuildViewTest method testGenQueryWithBadTargetAndUnfinishedTarget.

/**
   * Regression test for bug when a configured target has missing deps, but also depends
   * transitively on an error. We build //foo:query, which depends on a valid and an invalid target
   * pattern. We ensure that by the time it requests its dependent target patterns, the invalid one
   * is ready, and throws (though not before the request is registered). Then, when bubbling the
   * invalid target pattern error up, we ensure that it bubbles into //foo:query, which must cope
   * with the combination of an error and a missing dep.
   */
@Test
public void testGenQueryWithBadTargetAndUnfinishedTarget() throws Exception {
    // The target //foo:zquery is used to force evaluation of //foo:nosuchtarget before the target
    // patterns in //foo:query are enqueued for evaluation. That way, //foo:query will depend on one
    // invalid target pattern and two target patterns that haven't been evaluated yet.
    // It is important that 'query' come before 'zquery' alphabetically, so that when the error is
    // bubbling up, it goes to the //foo:query node -- we use a graph implementation in which the
    // reverse deps of each entry are ordered alphabetically. It is also important that a missing
    // target pattern is requested before the exception is thrown, so we have both //foo:b and
    // //foo:z missing from the deps, in the hopes that at least one of them will come before
    // //foo:nosuchtarget.
    scratch.file("foo/BUILD", "genquery(name = 'query',", "         expression = 'deps(//foo:b) except //foo:nosuchtarget except //foo:z',", "         scope = ['//foo:a'])", "genquery(name = 'zquery',", "         expression = 'deps(//foo:nosuchtarget)',", "         scope = ['//foo:a'])", "sh_library(name = 'a')", "sh_library(name = 'b')", "sh_library(name = 'z')");
    Listener listener = new Listener() {

        private final CountDownLatch errorDone = new CountDownLatch(1);

        private final CountDownLatch realQueryStarted = new CountDownLatch(1);

        @Override
        public void accept(SkyKey key, EventType type, Order order, Object context) {
            if (!key.functionName().equals(SkyFunctions.TARGET_PATTERN)) {
                return;
            }
            String label = ((TargetPatternKey) key.argument()).getPattern();
            if (label.equals("//foo:nosuchtarget")) {
                if (type == EventType.SET_VALUE) {
                    // Inform //foo:query-dep-registering thread that it may proceed.
                    errorDone.countDown();
                    // Wait to make sure //foo:query-dep-registering process has started.
                    TrackingAwaiter.INSTANCE.awaitLatchAndTrackExceptions(realQueryStarted, "//foo:query did not request dep in time");
                } else if (type == EventType.ADD_REVERSE_DEP && context.toString().contains("foo:query")) {
                    // Make sure that when foo:query requests foo:nosuchtarget, it's already done.
                    TrackingAwaiter.INSTANCE.awaitLatchAndTrackExceptions(errorDone, "//foo:nosuchtarget did not evaluate in time");
                }
            } else if ((label.equals("//foo:b") || label.equals("//foo:z")) && type == EventType.CREATE_IF_ABSENT) {
                // Inform error-evaluating thread that it may throw an exception.
                realQueryStarted.countDown();
                TrackingAwaiter.INSTANCE.awaitLatchAndTrackExceptions(errorDone, "//foo:nosuchtarget did not evaluate in time");
                // Don't let the target pattern //foo:{b,z} get enqueued for evaluation until we
                // receive an interrupt signal from the threadpool. The interrupt means that
                // evaluation is shutting down, and so //foo:{b,z} definitely won't get evaluated.
                CountDownLatch waitForInterrupt = new CountDownLatch(1);
                try {
                    waitForInterrupt.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
                    throw new IllegalStateException("node was not interrupted in time");
                } catch (InterruptedException e) {
                    // Expected.
                    Thread.currentThread().interrupt();
                }
            }
        }
    };
    injectGraphListenerForTesting(listener, /*deterministic=*/
    true);
    reporter.removeHandler(failFastHandler);
    try {
        update("//foo:query", "//foo:zquery");
        fail();
    } catch (ViewCreationFailedException e) {
        Truth.assertThat(e.getMessage()).contains("Analysis of target '//foo:query' failed; build aborted");
    }
    TrackingAwaiter.INSTANCE.assertNoErrors();
}

Example 8 with Listener

use of com.google.devtools.build.skyframe.NotifyingHelper.Listener in project bazel by bazelbuild.

the class BuildViewTest method testErrorBelowCycle.

// Regression test: cycle node depends on error.
@Test
public void testErrorBelowCycle() throws Exception {
    scratch.file("foo/BUILD", "sh_library(name = 'top', deps = ['mid'])", "sh_library(name = 'mid', deps = ['bad', 'cycle1'])", "sh_library(name = 'bad', srcs = ['//badbuild:isweird'])", "sh_library(name = 'cycle1', deps = ['cycle2', 'mid'])", "sh_library(name = 'cycle2', deps = ['cycle1'])");
    scratch.file("badbuild/BUILD", "");
    reporter.removeHandler(failFastHandler);
    injectGraphListenerForTesting(new Listener() {

        @Override
        public void accept(SkyKey key, EventType type, Order order, Object context) {
        }
    }, /*deterministic=*/
    true);
    try {
        update("//foo:top");
        fail();
    } catch (ViewCreationFailedException e) {
    // Expected.
    }
    assertContainsEvent("no such target '//badbuild:isweird': target 'isweird' not declared in " + "package 'badbuild'");
    assertContainsEvent("and referenced by '//foo:bad'");
    assertContainsEvent("in sh_library rule //foo");
    assertContainsEvent("cycle in dependency graph");
    assertEventCountAtLeast(2, eventCollector);
}

Example 9 with Listener

use of com.google.devtools.build.skyframe.NotifyingHelper.Listener in project bazel by bazelbuild.

the class MemoizingEvaluatorTest method dirtyChildEnqueuesParentDuringCheckDependencies.

/**
   * We are checking here that we are resilient to a race condition in which a value that is
   * checking its children for dirtiness is signaled by all of its children, putting it in a ready
   * state, before the thread has terminated. Optionally, one of its children may throw an error,
   * shutting down the threadpool. The essential race is that a child about to throw signals its
   * parent and the parent's builder restarts itself before the exception is thrown. Here, the
   * signaling happens while dirty dependencies are being checked. We control the timing by blocking
   * "top"'s registering itself on its deps.
   */
private void dirtyChildEnqueuesParentDuringCheckDependencies(final boolean throwError) throws Exception {
    // Value to be built. It will be signaled to rebuild before it has finished checking its deps.
    final SkyKey top = GraphTester.toSkyKey("top");
    // Dep that blocks before it acknowledges being added as a dep by top, so the firstKey value has
    // time to signal top. (Importantly its key is alphabetically after 'firstKey').
    final SkyKey slowAddingDep = GraphTester.toSkyKey("slowDep");
    // Don't perform any blocking on the first build.
    final AtomicBoolean delayTopSignaling = new AtomicBoolean(false);
    final CountDownLatch topSignaled = new CountDownLatch(1);
    final CountDownLatch topRestartedBuild = new CountDownLatch(1);
    injectGraphListenerForTesting(new Listener() {

        @Override
        public void accept(SkyKey key, EventType type, Order order, @Nullable Object context) {
            if (!delayTopSignaling.get()) {
                return;
            }
            if (key.equals(top) && type == EventType.SIGNAL && order == Order.AFTER) {
                // top is signaled by firstKey (since slowAddingDep is blocking), so slowAddingDep
                // is now free to acknowledge top as a parent.
                topSignaled.countDown();
                return;
            }
            if (key.equals(slowAddingDep) && type == EventType.ADD_REVERSE_DEP && top.equals(context) && order == Order.BEFORE) {
                // If top is trying to declare a dep on slowAddingDep, wait until firstKey has
                // signaled top. Then this add dep will return DONE and top will be signaled,
                // making it ready, so it will be enqueued.
                TrackingAwaiter.INSTANCE.awaitLatchAndTrackExceptions(topSignaled, "first key didn't signal top in time");
            }
        }
    }, /*deterministic=*/
    true);
    // Value that is modified on the second build. Its thread won't finish until it signals top,
    // which will wait for the signal before it enqueues its next dep. We prevent the thread from
    // finishing by having the listener to which it reports its warning block until top's builder
    // starts.
    final SkyKey firstKey = GraphTester.skyKey("first");
    tester.set(firstKey, new StringValue("biding"));
    tester.set(slowAddingDep, new StringValue("dep"));
    final AtomicInteger numTopInvocations = new AtomicInteger(0);
    tester.getOrCreate(top).setBuilder(new NoExtractorFunction() {

        @Override
        public SkyValue compute(SkyKey key, SkyFunction.Environment env) throws InterruptedException {
            numTopInvocations.incrementAndGet();
            if (delayTopSignaling.get()) {
                // The reporter will be given firstKey's warning to emit when it is requested as a dep
                // below, if firstKey is already built, so we release the reporter's latch beforehand.
                topRestartedBuild.countDown();
            }
            // top's builder just requests both deps in a group.
            env.getValuesOrThrow(ImmutableList.of(firstKey, slowAddingDep), SomeErrorException.class);
            return env.valuesMissing() ? null : new StringValue("top");
        }
    });
    reporter = new DelegatingEventHandler(reporter) {

        @Override
        public void handle(Event e) {
            super.handle(e);
            if (e.getKind() == EventKind.WARNING) {
                if (!throwError) {
                    TrackingAwaiter.INSTANCE.awaitLatchAndTrackExceptions(topRestartedBuild, "top's builder did not start in time");
                }
            }
        }
    };
    // First build : just prime the graph.
    EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/
    false, top);
    assertFalse(result.hasError());
    assertEquals(new StringValue("top"), result.get(top));
    assertEquals(2, numTopInvocations.get());
    // Now dirty the graph, and maybe have firstKey throw an error.
    String warningText = "warning text";
    tester.getOrCreate(firstKey, /*markAsModified=*/
    true).setHasError(throwError).setWarning(warningText);
    tester.invalidate();
    delayTopSignaling.set(true);
    result = tester.eval(/*keepGoing=*/
    false, top);
    if (throwError) {
        assertTrue(result.hasError());
        // No successfully evaluated values.
        assertThat(result.keyNames()).isEmpty();
        ErrorInfo errorInfo = result.getError(top);
        assertThat(errorInfo.getRootCauses()).containsExactly(firstKey);
        assertEquals("on the incremental build, top's builder should have only been used in error " + "bubbling", 3, numTopInvocations.get());
    } else {
        assertEquals(new StringValue("top"), result.get(top));
        assertFalse(result.hasError());
        assertEquals("on the incremental build, top's builder should have only been executed once in " + "normal evaluation", 3, numTopInvocations.get());
    }
    assertContainsEvent(eventCollector, warningText);
    assertEquals(0, topSignaled.getCount());
    assertEquals(0, topRestartedBuild.getCount());
}

Example 10 with Listener

use of com.google.devtools.build.skyframe.NotifyingHelper.Listener in project bazel by bazelbuild.

the class MemoizingEvaluatorTest method cycleAndErrorAndReady.

/**
   * Regression test: IllegalStateException in BuildingState.isReady(). The ParallelEvaluator used
   * to assume during cycle-checking that all values had been built as fully as possible -- that
   * evaluation had not been interrupted. However, we also do cycle-checking in nokeep-going mode
   * when a value throws an error (possibly prematurely shutting down evaluation) but that error
   * then bubbles up into a cycle.
   *
   * <p>We want to achieve the following state: we are checking for a cycle; the value we examine
   * has not yet finished checking its children to see if they are dirty; but all children checked
   * so far have been unchanged. This value is "otherTop". We first build otherTop, then mark its
   * first child changed (without actually changing it), and then do a second build. On the second
   * build, we also build "top", which requests a cycle that depends on an error. We wait to signal
   * otherTop that its first child is done until the error throws and shuts down evaluation. The
   * error then bubbles up to the cycle, and so the bubbling is aborted. Finally, cycle checking
   * happens, and otherTop is examined, as desired.
   */
@Test
public void cycleAndErrorAndReady() throws Exception {
    // This value will not have finished building on the second build when the error is thrown.
    final SkyKey otherTop = GraphTester.toSkyKey("otherTop");
    final SkyKey errorKey = GraphTester.toSkyKey("error");
    // Is the graph state all set up and ready for the error to be thrown? The three values are
    // exceptionMarker, cycle2Key, and dep1 (via signaling otherTop).
    final CountDownLatch valuesReady = new CountDownLatch(3);
    // Is evaluation being shut down? This is counted down by the exceptionMarker's builder, after
    // it has waited for the threadpool's exception latch to be released.
    final CountDownLatch errorThrown = new CountDownLatch(1);
    // We don't do anything on the first build.
    final AtomicBoolean secondBuild = new AtomicBoolean(false);
    injectGraphListenerForTesting(new Listener() {

        @Override
        public void accept(SkyKey key, EventType type, Order order, Object context) {
            if (!secondBuild.get()) {
                return;
            }
            if (key.equals(otherTop) && type == EventType.SIGNAL) {
                // otherTop is being signaled that dep1 is done. Tell the error value that it is
                // ready, then wait until the error is thrown, so that otherTop's builder is not
                // re-entered.
                valuesReady.countDown();
                TrackingAwaiter.INSTANCE.awaitLatchAndTrackExceptions(errorThrown, "error not thrown");
                return;
            }
        }
    }, /*deterministic=*/
    true);
    final SkyKey dep1 = GraphTester.toSkyKey("dep1");
    tester.set(dep1, new StringValue("dep1"));
    final SkyKey dep2 = GraphTester.toSkyKey("dep2");
    tester.set(dep2, new StringValue("dep2"));
    // otherTop should request the deps one at a time, so that it can be in the CHECK_DEPENDENCIES
    // state even after one dep is re-evaluated.
    tester.getOrCreate(otherTop).setBuilder(new NoExtractorFunction() {

        @Override
        public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException {
            env.getValue(dep1);
            if (env.valuesMissing()) {
                return null;
            }
            env.getValue(dep2);
            return env.valuesMissing() ? null : new StringValue("otherTop");
        }
    });
    // Prime the graph with otherTop, so we can dirty it next build.
    assertEquals(new StringValue("otherTop"), tester.evalAndGet(/*keepGoing=*/
    false, otherTop));
    // Mark dep1 changed, so otherTop will be dirty and request re-evaluation of dep1.
    tester.getOrCreate(dep1, /*markAsModified=*/
    true);
    SkyKey topKey = GraphTester.toSkyKey("top");
    // Note that since DeterministicHelper alphabetizes reverse deps, it is important that
    // "cycle2" comes before "top".
    final SkyKey cycle1Key = GraphTester.toSkyKey("cycle1");
    final SkyKey cycle2Key = GraphTester.toSkyKey("cycle2");
    tester.getOrCreate(topKey).addDependency(cycle1Key).setComputedValue(CONCATENATE);
    tester.getOrCreate(cycle1Key).addDependency(errorKey).addDependency(cycle2Key).setComputedValue(CONCATENATE);
    tester.getOrCreate(errorKey).setBuilder(new ChainedFunction(/*notifyStart=*/
    null, /*waitToFinish=*/
    valuesReady, /*notifyFinish=*/
    null, /*waitForException=*/
    false, /*value=*/
    null, ImmutableList.<SkyKey>of()));
    // Make sure cycle2Key has declared its dependence on cycle1Key before error throws.
    tester.getOrCreate(cycle2Key).setBuilder(new ChainedFunction(/*notifyStart=*/
    valuesReady, null, null, false, new StringValue("never returned"), ImmutableList.<SkyKey>of(cycle1Key)));
    // Value that waits until an exception is thrown to finish building. We use it just to be
    // informed when the threadpool is shutting down.
    final SkyKey exceptionMarker = GraphTester.toSkyKey("exceptionMarker");
    tester.getOrCreate(exceptionMarker).setBuilder(new ChainedFunction(/*notifyStart=*/
    valuesReady, /*waitToFinish=*/
    new CountDownLatch(0), /*notifyFinish=*/
    errorThrown, /*waitForException=*/
    true, new StringValue("exception marker"), ImmutableList.<SkyKey>of()));
    tester.invalidate();
    secondBuild.set(true);
    // otherTop must be first, since we check top-level values for cycles in the order in which
    // they appear here.
    EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/
    false, otherTop, topKey, exceptionMarker);
    Iterable<CycleInfo> cycleInfos = result.getError(topKey).getCycleInfo();
    assertWithMessage(result.toString()).that(cycleInfos).isNotEmpty();
    CycleInfo cycleInfo = Iterables.getOnlyElement(cycleInfos);
    if (cyclesDetected()) {
        assertThat(result.errorMap().keySet()).containsExactly(topKey);
        assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey);
        assertThat(cycleInfo.getCycle()).containsExactly(cycle1Key, cycle2Key);
    }
}