Examples with Environment com.google.devtools.build.skyframe.SkyFunction.Environment used on opensource projects

Example 11 with Environment

use of com.google.devtools.build.skyframe.SkyFunction.Environment in project bazel by bazelbuild.

the class MemoizingEvaluatorTest method errorInDependencyGroup.

/**
   * "top" requests a dependency group in which the first value, called "error", throws an
   * exception, so "mid" and "mid2", which depend on "slow", never get built.
   */
@Test
public void errorInDependencyGroup() throws Exception {
    initializeTester();
    SkyKey topKey = GraphTester.toSkyKey("top");
    CountDownLatch slowStart = new CountDownLatch(1);
    CountDownLatch errorFinish = new CountDownLatch(1);
    final SkyKey errorKey = GraphTester.toSkyKey("error");
    tester.getOrCreate(errorKey).setBuilder(new ChainedFunction(/*notifyStart=*/
    null, /*waitToFinish=*/
    slowStart, /*notifyFinish=*/
    errorFinish, /*waitForException=*/
    false, /*value=*/
    null, /*deps=*/
    ImmutableList.<SkyKey>of()));
    SkyKey slowKey = GraphTester.toSkyKey("slow");
    tester.getOrCreate(slowKey).setBuilder(new ChainedFunction(/*notifyStart=*/
    slowStart, /*waitToFinish=*/
    errorFinish, /*notifyFinish=*/
    null, /*waitForException=*/
    true, new StringValue("slow"), /*deps=*/
    ImmutableList.<SkyKey>of()));
    final SkyKey midKey = GraphTester.toSkyKey("mid");
    tester.getOrCreate(midKey).addDependency(slowKey).setComputedValue(COPY);
    final SkyKey mid2Key = GraphTester.toSkyKey("mid2");
    tester.getOrCreate(mid2Key).addDependency(slowKey).setComputedValue(COPY);
    tester.set(topKey, null);
    tester.getOrCreate(topKey).setBuilder(new SkyFunction() {

        @Override
        public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, InterruptedException {
            env.getValues(ImmutableList.of(errorKey, midKey, mid2Key));
            if (env.valuesMissing()) {
                return null;
            }
            return new StringValue("top");
        }

        @Override
        public String extractTag(SkyKey skyKey) {
            return null;
        }
    });
    // Assert that build fails and "error" really is in error.
    EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/
    false, topKey);
    assertTrue(result.hasError());
    assertThat(result.getError(topKey).getRootCauses()).containsExactly(errorKey);
    // Ensure that evaluation succeeds if errorKey does not throw an error.
    tester.getOrCreate(errorKey).setBuilder(null);
    tester.set(errorKey, new StringValue("ok"));
    tester.invalidate();
    assertEquals(new StringValue("top"), tester.evalAndGet("top"));
}

Example 12 with Environment

use of com.google.devtools.build.skyframe.SkyFunction.Environment in project bazel by bazelbuild.

the class MemoizingEvaluatorTest method manyDirtyValuesClearChildrenOnFail.

/**
   * Regression test: enqueue so many values that some of them won't have started processing, and
   * then either interrupt processing or have a child throw an error. In the latter case, this also
   * tests that a value that hasn't started processing can still have a child error bubble up to it.
   * In both cases, it tests that the graph is properly cleaned of the dirty values and references
   * to them.
   */
private void manyDirtyValuesClearChildrenOnFail(boolean interrupt) throws Exception {
    SkyKey leafKey = GraphTester.toSkyKey("leaf");
    tester.set(leafKey, new StringValue("leafy"));
    SkyKey lastKey = GraphTester.toSkyKey("last");
    tester.set(lastKey, new StringValue("last"));
    final List<SkyKey> tops = new ArrayList<>();
    // leaf child is enqueued for processing.
    for (int i = 0; i < 10000; i++) {
        SkyKey topKey = GraphTester.toSkyKey("top" + i);
        tester.getOrCreate(topKey).addDependency(leafKey).addDependency(lastKey).setComputedValue(CONCATENATE);
        tops.add(topKey);
    }
    tester.eval(/*keepGoing=*/
    false, tops.toArray(new SkyKey[0]));
    final CountDownLatch notifyStart = new CountDownLatch(1);
    tester.set(leafKey, null);
    if (interrupt) {
        // leaf will wait for an interrupt if desired. We cannot use the usual ChainedFunction
        // because we need to actually throw the interrupt.
        final AtomicBoolean shouldSleep = new AtomicBoolean(true);
        tester.getOrCreate(leafKey, /*markAsModified=*/
        true).setBuilder(new NoExtractorFunction() {

            @Override
            public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException {
                notifyStart.countDown();
                if (shouldSleep.get()) {
                    // Should be interrupted within 5 seconds.
                    Thread.sleep(5000);
                    throw new AssertionError("leaf was not interrupted");
                }
                return new StringValue("crunchy");
            }
        });
        tester.invalidate();
        TestThread evalThread = new TestThread() {

            @Override
            public void runTest() {
                try {
                    tester.eval(/*keepGoing=*/
                    false, tops.toArray(new SkyKey[0]));
                    Assert.fail();
                } catch (InterruptedException e) {
                // Expected.
                }
            }
        };
        evalThread.start();
        assertTrue(notifyStart.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
        evalThread.interrupt();
        evalThread.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
        // Free leafKey to compute next time.
        shouldSleep.set(false);
    } else {
        // Non-interrupt case. Just throw an error in the child.
        tester.getOrCreate(leafKey, /*markAsModified=*/
        true).setHasError(true);
        tester.invalidate();
        // The error thrown may non-deterministically bubble up to a parent that has not yet started
        // processing, but has been enqueued for processing.
        tester.eval(/*keepGoing=*/
        false, tops.toArray(new SkyKey[0]));
        tester.getOrCreate(leafKey, /*markAsModified=*/
        true).setHasError(false);
        tester.set(leafKey, new StringValue("crunchy"));
    }
    // lastKey was not touched during the previous build, but its reverse deps on its parents should
    // still be accurate.
    tester.set(lastKey, new StringValue("new last"));
    tester.invalidate();
    EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/
    false, tops.toArray(new SkyKey[0]));
    for (SkyKey topKey : tops) {
        assertEquals(topKey.toString(), "crunchynew last", result.get(topKey).getValue());
    }
}

Example 13 with Environment

use of com.google.devtools.build.skyframe.SkyFunction.Environment in project bazel by bazelbuild.

the class SkyframeAwareActionTest method testRaceConditionBetweenInputAcquisitionAndSkyframeDeps.

/**
   * Regression test to avoid a potential race condition in {@link ActionExecutionFunction}.
   *
   * <p>The test ensures that when ActionExecutionFunction executes a Skyframe-aware action
   * (implementor of {@link SkyframeAwareAction}), ActionExecutionFunction first requests the inputs
   * of the action and ensures they are built before requesting any of its Skyframe dependencies.
   *
   * <p>This strict ordering is very important to avoid the race condition, which could arise if the
   * compute method were too eager to request all dependencies: request input files but even if some
   * are missing, request also the skyframe-dependencies. The race is described in this method's
   * body.
   */
@Test
public void testRaceConditionBetweenInputAcquisitionAndSkyframeDeps() throws Exception {
    // Sequence of events on threads A and B, showing SkyFunctions and requested SkyKeys, leading
    // to an InconsistentFilesystemException:
    //
    // _______________[Thread A]_________________|_______________[Thread B]_________________
    // ActionExecutionFunction(gen2_action:      | idle
    //   genfiles/gen1 -> genfiles/foo/bar/gen2) |
    // ARTIFACT:genfiles/gen1                    |
    // MOCK_VALUE:dummy_argument                 |
    // env.valuesMissing():yes ==> return        |
    //                                           |
    // ArtifactFunction(genfiles/gen1)           | MockFunction()
    // CONFIGURED_TARGET://foo:gen1              | FILE:genfiles/foo
    // ACTION_EXECUTION:gen1_action              | env.valuesMissing():yes ==> return
    // env.valuesMissing():yes ==> return        |
    //                                           | FileFunction(genfiles/foo)
    // ActionExecutionFunction(gen1_action)      | FILE:genfiles
    // ARTIFACT:genfiles/gen0                    | env.valuesMissing():yes ==> return
    // env.valuesMissing():yes ==> return        |
    //                                           | FileFunction(genfiles)
    // ArtifactFunction(genfiles/gen0)           | FILE_STATE:genfiles
    // CONFIGURED_TARGET://foo:gen0              | env.valuesMissing():yes ==> return
    // ACTION_EXECUTION:gen0_action              |
    // env.valuesMissing():yes ==> return        | FileStateFunction(genfiles)
    //                                           | stat genfiles
    // ActionExecutionFunction(gen0_action)      | return FileStateValue:non-existent
    // create output directory: genfiles         |
    // working                                   | FileFunction(genfiles/foo)
    //                                           | FILE:genfiles
    //                                           | FILE_STATE:genfiles/foo
    //                                           | env.valuesMissing():yes ==> return
    //                                           |
    //                                           | FileStateFunction(genfiles/foo)
    //                                           | stat genfiles/foo
    //                                           | return FileStateValue:non-existent
    //                                           |
    // done, created genfiles/gen0               | FileFunction(genfiles/foo)
    // return ActionExecutionValue(gen0_action)  | FILE:genfiles
    //                                           | FILE_STATE:genfiles/foo
    // ArtifactFunction(genfiles/gen0)           | return FileValue(genfiles/foo:non-existent)
    // CONFIGURED_TARGET://foo:gen0              |
    // ACTION_EXECUTION:gen0_action              | MockFunction()
    // return ArtifactSkyKey(genfiles/gen0)      | FILE:genfiles/foo
    //                                           | FILE:genfiles/foo/bar/gen1
    // ActionExecutionFunction(gen1_action)      | env.valuesMissing():yes ==> return
    // ARTIFACT:genfiles/gen0                    |
    // create output directory: genfiles/foo/bar | FileFunction(genfiles/foo/bar/gen1)
    // done, created genfiles/foo/bar/gen1       | FILE:genfiles/foo/bar
    // return ActionExecutionValue(gen1_action)  | env.valuesMissing():yes ==> return
    //                                           |
    // idle                                      | FileFunction(genfiles/foo/bar)
    //                                           | FILE:genfiles/foo
    //                                           | FILE_STATE:genfiles/foo/bar
    //                                           | env.valuesMissing():yes ==> return
    //                                           |
    //                                           | FileStateFunction(genfiles/foo/bar)
    //                                           | stat genfiles/foo/bar
    //                                           | return FileStateValue:directory
    //                                           |
    //                                           | FileFunction(genfiles/foo/bar)
    //                                           | FILE:genfiles/foo
    //                                           | FILE_STATE:genfiles/foo/bar
    //                                           | throw InconsistentFilesystemException:
    //                                           |     genfiles/foo doesn't exist but
    //                                           |     genfiles/foo/bar does!
    Artifact genFile1 = createDerivedArtifact("foo/bar/gen1.txt");
    Artifact genFile2 = createDerivedArtifact("gen2.txt");
    registerAction(new SingleOutputAction(null, genFile1) {

        @Override
        public void execute(ActionExecutionContext actionExecutionContext) throws ActionExecutionException, InterruptedException {
            writeOutput(null, "gen1");
        }
    });
    registerAction(new SingleOutputSkyframeAwareAction(genFile1, genFile2) {

        @Override
        public void establishSkyframeDependencies(Environment env) throws ExceptionBase {
            assertThat(env.valuesMissing()).isFalse();
        }

        @Override
        public void execute(ActionExecutionContext actionExecutionContext) throws ActionExecutionException, InterruptedException {
            writeOutput(readInput(), "gen2");
        }
    });
    builder.buildArtifacts(reporter, ImmutableSet.of(genFile2), null, null, null, null, executor, null, false, null, null);
}

Example 14 with Environment

use of com.google.devtools.build.skyframe.SkyFunction.Environment in project bazel by bazelbuild.

the class MemoizingEvaluatorTest method interruptAfterFailFails.

@Test
public void interruptAfterFailFails() throws Exception {
    SkyKey failKey = GraphTester.skyKey("fail");
    SkyKey interruptedKey = GraphTester.skyKey("interrupted");
    // Given a SkyFunction implementation that is properly coded to as not to throw a
    // runtime exception when it is interrupted,
    final CountDownLatch interruptStarted = new CountDownLatch(1);
    tester.getOrCreate(interruptedKey).setBuilder(new SkyFunction() {

        @Nullable
        @Override
        public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException {
            interruptStarted.countDown();
            Thread.sleep(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
            throw new AssertionError("Shouldn't have slept so long");
        }

        @Nullable
        @Override
        public String extractTag(SkyKey skyKey) {
            return null;
        }
    });
    // And another SkyFunction that waits for the first to start, and then throws,
    tester.getOrCreate(failKey).setBuilder(new ChainedFunction(null, interruptStarted, null, /*waitForException=*/
    false, null, ImmutableList.<SkyKey>of()));
    // When it is interrupted during evaluation (here, caused by the failure of a sibling node
    // during a no-keep-going evaluation),
    EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/
    false, interruptedKey, failKey);
    // Then the Evaluator#evaluate call returns an EvaluationResult that has no error for the
    // interrupted SkyFunction.
    assertWithMessage(result.toString()).that(result.hasError()).isTrue();
    assertWithMessage(result.toString()).that(result.getError(failKey)).isNotNull();
    assertWithMessage(result.toString()).that(result.getError(interruptedKey)).isNull();
}

Example 15 with Environment

use of com.google.devtools.build.skyframe.SkyFunction.Environment in project bazel by bazelbuild.

the class MemoizingEvaluatorTest method dirtyWithOwnErrorDependsOnTransientErrorTurningGood.

/** Regression test for crash bug. */
@Test
public void dirtyWithOwnErrorDependsOnTransientErrorTurningGood() throws Exception {
    initializeTester();
    final SkyKey error = GraphTester.toSkyKey("error");
    tester.getOrCreate(error).setHasTransientError(true);
    SkyKey topKey = GraphTester.toSkyKey("top");
    SkyFunction errorFunction = new SkyFunction() {

        @Override
        public SkyValue compute(SkyKey skyKey, Environment env) throws GenericFunctionException, InterruptedException {
            try {
                return env.getValueOrThrow(error, SomeErrorException.class);
            } catch (SomeErrorException e) {
                throw new GenericFunctionException(e, Transience.PERSISTENT);
            }
        }

        @Override
        public String extractTag(SkyKey skyKey) {
            throw new UnsupportedOperationException();
        }
    };
    tester.getOrCreate(topKey).setBuilder(errorFunction);
    EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/
    false, topKey);
    tester.invalidateTransientErrors();
    assertThat(result.getError(topKey).getRootCauses()).containsExactly(topKey);
    tester.getOrCreate(error).setHasTransientError(false);
    StringValue reformed = new StringValue("reformed");
    tester.set(error, reformed);
    tester.getOrCreate(topKey).setBuilder(null).addDependency(error).setComputedValue(COPY);
    tester.invalidate();
    tester.invalidateTransientErrors();
    result = tester.eval(/*keepGoing=*/
    false, topKey);
    assertEquals(reformed, result.get(topKey));
    assertFalse(result.hasError());
}