Examples with RecordingDifferencer com.google.devtools.build.skyframe.RecordingDifferencer used on opensource projects

Example 6 with RecordingDifferencer

use of com.google.devtools.build.skyframe.RecordingDifferencer in project bazel by bazelbuild.

the class TimestampBuilderTestCase method createBuilder.

protected Builder createBuilder(final ActionCache actionCache, final int threadCount, final boolean keepGoing, @Nullable EvaluationProgressReceiver evaluationProgressReceiver) throws Exception {
    AtomicReference<PathPackageLocator> pkgLocator = new AtomicReference<>(new PathPackageLocator(outputBase, ImmutableList.of(rootDirectory)));
    AtomicReference<TimestampGranularityMonitor> tsgmRef = new AtomicReference<>(tsgm);
    BlazeDirectories directories = new BlazeDirectories(rootDirectory, outputBase, rootDirectory, TestConstants.PRODUCT_NAME);
    ExternalFilesHelper externalFilesHelper = new ExternalFilesHelper(pkgLocator, ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS, directories);
    differencer = new RecordingDifferencer();
    ActionExecutionStatusReporter statusReporter = ActionExecutionStatusReporter.create(new StoredEventHandler());
    final SkyframeActionExecutor skyframeActionExecutor = new SkyframeActionExecutor(eventBusRef, new AtomicReference<>(statusReporter));
    Path actionOutputBase = scratch.dir("/usr/local/google/_blaze_jrluser/FAKEMD5/action_out/");
    skyframeActionExecutor.setActionLogBufferPathGenerator(new ActionLogBufferPathGenerator(actionOutputBase));
    ActionInputFileCache cache = new SingleBuildFileCache(rootDirectory.getPathString(), scratch.getFileSystem());
    skyframeActionExecutor.setFileCache(cache);
    final InMemoryMemoizingEvaluator evaluator = new InMemoryMemoizingEvaluator(ImmutableMap.<SkyFunctionName, SkyFunction>builder().put(SkyFunctions.FILE_STATE, new FileStateFunction(tsgmRef, externalFilesHelper)).put(SkyFunctions.FILE, new FileFunction(pkgLocator)).put(SkyFunctions.ARTIFACT, new ArtifactFunction(Predicates.<PathFragment>alwaysFalse())).put(SkyFunctions.ACTION_EXECUTION, new ActionExecutionFunction(skyframeActionExecutor, tsgmRef)).put(SkyFunctions.PACKAGE, new PackageFunction(null, null, null, null, null, null, null)).put(SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction(null, CrossRepositoryLabelViolationStrategy.ERROR, ImmutableList.of(BuildFileName.BUILD_DOT_BAZEL, BuildFileName.BUILD))).put(SkyFunctions.WORKSPACE_AST, new WorkspaceASTFunction(TestRuleClassProvider.getRuleClassProvider())).put(SkyFunctions.WORKSPACE_FILE, new WorkspaceFileFunction(TestRuleClassProvider.getRuleClassProvider(), TestConstants.PACKAGE_FACTORY_FACTORY_FOR_TESTING.create(TestRuleClassProvider.getRuleClassProvider(), scratch.getFileSystem()), directories)).put(SkyFunctions.EXTERNAL_PACKAGE, new ExternalPackageFunction()).put(SkyFunctions.ACTION_TEMPLATE_EXPANSION, new DelegatingActionTemplateExpansionFunction()).build(), differencer, evaluationProgressReceiver);
    final SequentialBuildDriver driver = new SequentialBuildDriver(evaluator);
    PrecomputedValue.BUILD_ID.set(differencer, UUID.randomUUID());
    PrecomputedValue.ACTION_ENV.set(differencer, ImmutableMap.<String, String>of());
    PrecomputedValue.PATH_PACKAGE_LOCATOR.set(differencer, pkgLocator.get());
    return new Builder() {

        private void setGeneratingActions() {
            if (evaluator.getExistingValueForTesting(OWNER_KEY) == null) {
                differencer.inject(ImmutableMap.of(OWNER_KEY, new ActionLookupValue(ImmutableList.copyOf(actions))));
            }
        }

        @Override
        public void buildArtifacts(Reporter reporter, Set<Artifact> artifacts, Set<ConfiguredTarget> parallelTests, Set<ConfiguredTarget> exclusiveTests, Collection<ConfiguredTarget> targetsToBuild, Collection<AspectValue> aspects, Executor executor, Set<ConfiguredTarget> builtTargets, boolean explain, Range<Long> lastExecutionTimeRange, TopLevelArtifactContext topLevelArtifactContext) throws BuildFailedException, AbruptExitException, InterruptedException, TestExecException {
            skyframeActionExecutor.prepareForExecution(reporter, executor, keepGoing, /*explain=*/
            false, new ActionCacheChecker(actionCache, null, ALWAYS_EXECUTE_FILTER, null), null);
            List<SkyKey> keys = new ArrayList<>();
            for (Artifact artifact : artifacts) {
                keys.add(ArtifactSkyKey.key(artifact, true));
            }
            setGeneratingActions();
            EvaluationResult<SkyValue> result = driver.evaluate(keys, keepGoing, threadCount, reporter);
            if (result.hasError()) {
                boolean hasCycles = false;
                for (Map.Entry<SkyKey, ErrorInfo> entry : result.errorMap().entrySet()) {
                    Iterable<CycleInfo> cycles = entry.getValue().getCycleInfo();
                    hasCycles |= !Iterables.isEmpty(cycles);
                }
                if (hasCycles) {
                    throw new BuildFailedException(CYCLE_MSG);
                } else if (result.errorMap().isEmpty() || keepGoing) {
                    throw new BuildFailedException();
                } else {
                    SkyframeBuilder.rethrow(Preconditions.checkNotNull(result.getError().getException()));
                }
            }
        }
    };
}

Example 7 with RecordingDifferencer

use of com.google.devtools.build.skyframe.RecordingDifferencer in project bazel by bazelbuild.

the class GlobFunctionTest method setUp.

@Before
public final void setUp() throws Exception {
    fs = new CustomInMemoryFs(new ManualClock());
    root = fs.getRootDirectory().getRelative("root/workspace");
    writableRoot = fs.getRootDirectory().getRelative("writableRoot/workspace");
    outputBase = fs.getRootDirectory().getRelative("output_base");
    pkgPath = root.getRelative(PKG_ID.getPackageFragment());
    pkgLocator = new AtomicReference<>(new PathPackageLocator(outputBase, ImmutableList.of(writableRoot, root)));
    differencer = new RecordingDifferencer();
    evaluator = new InMemoryMemoizingEvaluator(createFunctionMap(), differencer);
    driver = new SequentialBuildDriver(evaluator);
    PrecomputedValue.BUILD_ID.set(differencer, UUID.randomUUID());
    PrecomputedValue.PATH_PACKAGE_LOCATOR.set(differencer, pkgLocator.get());
    PrecomputedValue.BLACKLISTED_PACKAGE_PREFIXES_FILE.set(differencer, PathFragment.EMPTY_FRAGMENT);
    createTestFiles();
}

Example 8 with RecordingDifferencer

use of com.google.devtools.build.skyframe.RecordingDifferencer in project bazel by bazelbuild.

the class PackageFunctionTest method testPropagatesFilesystemInconsistencies.

@Test
public void testPropagatesFilesystemInconsistencies() throws Exception {
    reporter.removeHandler(failFastHandler);
    RecordingDifferencer differencer = getSkyframeExecutor().getDifferencerForTesting();
    Path pkgRoot = getSkyframeExecutor().getPathEntries().get(0);
    Path fooBuildFile = scratch.file("foo/BUILD");
    Path fooDir = fooBuildFile.getParentDirectory();
    // Our custom filesystem says "foo/BUILD" exists but its parent "foo" is a file.
    FileStatus inconsistentParentFileStatus = new FileStatus() {

        @Override
        public boolean isFile() {
            return true;
        }

        @Override
        public boolean isDirectory() {
            return false;
        }

        @Override
        public boolean isSymbolicLink() {
            return false;
        }

        @Override
        public boolean isSpecialFile() {
            return false;
        }

        @Override
        public long getSize() throws IOException {
            return 0;
        }

        @Override
        public long getLastModifiedTime() throws IOException {
            return 0;
        }

        @Override
        public long getLastChangeTime() throws IOException {
            return 0;
        }

        @Override
        public long getNodeId() throws IOException {
            return 0;
        }
    };
    fs.stubStat(fooDir, inconsistentParentFileStatus);
    RootedPath pkgRootedPath = RootedPath.toRootedPath(pkgRoot, fooDir);
    SkyValue fooDirValue = FileStateValue.create(pkgRootedPath, tsgm);
    differencer.inject(ImmutableMap.of(FileStateValue.key(pkgRootedPath), fooDirValue));
    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
    String expectedMessage = "/workspace/foo/BUILD exists but its parent path /workspace/foo isn't " + "an existing directory";
    EvaluationResult<PackageValue> result = SkyframeExecutorTestUtils.evaluate(getSkyframeExecutor(), skyKey, /*keepGoing=*/
    false, reporter);
    assertTrue(result.hasError());
    ErrorInfo errorInfo = result.getError(skyKey);
    String errorMessage = errorInfo.getException().getMessage();
    assertThat(errorMessage).contains("Inconsistent filesystem operations");
    assertThat(errorMessage).contains(expectedMessage);
}

Example 9 with RecordingDifferencer

use of com.google.devtools.build.skyframe.RecordingDifferencer in project bazel by bazelbuild.

the class PackageFunctionTest method testPropagatesFilesystemInconsistencies_Globbing.

@Test
public void testPropagatesFilesystemInconsistencies_Globbing() throws Exception {
    reporter.removeHandler(failFastHandler);
    RecordingDifferencer differencer = getSkyframeExecutor().getDifferencerForTesting();
    Path pkgRoot = getSkyframeExecutor().getPathEntries().get(0);
    scratch.file("foo/BUILD", "subinclude('//a:a')", "sh_library(name = 'foo', srcs = glob(['bar/**/baz.sh']))");
    scratch.file("a/BUILD");
    scratch.file("a/a");
    Path bazFile = scratch.file("foo/bar/baz/baz.sh");
    Path bazDir = bazFile.getParentDirectory();
    Path barDir = bazDir.getParentDirectory();
    // Our custom filesystem says "foo/bar/baz" does not exist but it also says that "foo/bar"
    // has a child directory "baz".
    fs.stubStat(bazDir, null);
    RootedPath barDirRootedPath = RootedPath.toRootedPath(pkgRoot, barDir);
    FileStateValue barDirFileStateValue = FileStateValue.create(barDirRootedPath, tsgm);
    FileValue barDirFileValue = FileValue.value(barDirRootedPath, barDirFileStateValue, barDirRootedPath, barDirFileStateValue);
    DirectoryListingValue barDirListing = DirectoryListingValue.value(barDirRootedPath, barDirFileValue, DirectoryListingStateValue.create(ImmutableList.of(new Dirent("baz", Dirent.Type.DIRECTORY))));
    differencer.inject(ImmutableMap.of(DirectoryListingValue.key(barDirRootedPath), barDirListing));
    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
    String expectedMessage = "/workspace/foo/bar/baz is no longer an existing directory";
    EvaluationResult<PackageValue> result = SkyframeExecutorTestUtils.evaluate(getSkyframeExecutor(), skyKey, /*keepGoing=*/
    false, reporter);
    assertTrue(result.hasError());
    ErrorInfo errorInfo = result.getError(skyKey);
    String errorMessage = errorInfo.getException().getMessage();
    assertThat(errorMessage).contains("Inconsistent filesystem operations");
    assertThat(errorMessage).contains(expectedMessage);
}

Example 10 with RecordingDifferencer

use of com.google.devtools.build.skyframe.RecordingDifferencer in project bazel by bazelbuild.

the class PackageLookupFunctionTest method setUp.

@Before
public final void setUp() throws Exception {
    emptyPackagePath = rootDirectory.getRelative("somewhere/else");
    scratch.file("parentpackage/BUILD");
    AnalysisMock analysisMock = AnalysisMock.get();
    AtomicReference<PathPackageLocator> pkgLocator = new AtomicReference<>(new PathPackageLocator(outputBase, ImmutableList.of(emptyPackagePath, rootDirectory)));
    deletedPackages = new AtomicReference<>(ImmutableSet.<PackageIdentifier>of());
    BlazeDirectories directories = new BlazeDirectories(rootDirectory, outputBase, rootDirectory, analysisMock.getProductName());
    ExternalFilesHelper externalFilesHelper = new ExternalFilesHelper(pkgLocator, ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS, directories);
    Map<SkyFunctionName, SkyFunction> skyFunctions = new HashMap<>();
    skyFunctions.put(SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction(deletedPackages, crossRepositoryLabelViolationStrategy(), ImmutableList.of(BuildFileName.BUILD_DOT_BAZEL, BuildFileName.BUILD)));
    skyFunctions.put(SkyFunctions.PACKAGE, new PackageFunction(null, null, null, null, null, null, null));
    skyFunctions.put(SkyFunctions.FILE_STATE, new FileStateFunction(new AtomicReference<TimestampGranularityMonitor>(), externalFilesHelper));
    skyFunctions.put(SkyFunctions.FILE, new FileFunction(pkgLocator));
    skyFunctions.put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction());
    skyFunctions.put(SkyFunctions.DIRECTORY_LISTING_STATE, new DirectoryListingStateFunction(externalFilesHelper));
    skyFunctions.put(SkyFunctions.BLACKLISTED_PACKAGE_PREFIXES, new BlacklistedPackagePrefixesFunction());
    RuleClassProvider ruleClassProvider = analysisMock.createRuleClassProvider();
    skyFunctions.put(SkyFunctions.WORKSPACE_AST, new WorkspaceASTFunction(ruleClassProvider));
    skyFunctions.put(SkyFunctions.WORKSPACE_FILE, new WorkspaceFileFunction(ruleClassProvider, analysisMock.getPackageFactoryForTesting().create(ruleClassProvider, new PackageFactory.EmptyEnvironmentExtension(), scratch.getFileSystem()), directories));
    skyFunctions.put(SkyFunctions.EXTERNAL_PACKAGE, new ExternalPackageFunction());
    skyFunctions.put(SkyFunctions.LOCAL_REPOSITORY_LOOKUP, new LocalRepositoryLookupFunction());
    skyFunctions.put(SkyFunctions.FILE_SYMLINK_CYCLE_UNIQUENESS, new FileSymlinkCycleUniquenessFunction());
    ImmutableMap<String, RepositoryFunction> repositoryHandlers = ImmutableMap.of(LocalRepositoryRule.NAME, (RepositoryFunction) new LocalRepositoryFunction());
    skyFunctions.put(SkyFunctions.REPOSITORY_DIRECTORY, new RepositoryDelegatorFunction(repositoryHandlers, null, new AtomicBoolean(true)));
    skyFunctions.put(SkyFunctions.REPOSITORY, new RepositoryLoaderFunction());
    differencer = new RecordingDifferencer();
    evaluator = new InMemoryMemoizingEvaluator(skyFunctions, differencer);
    driver = new SequentialBuildDriver(evaluator);
    PrecomputedValue.BUILD_ID.set(differencer, UUID.randomUUID());
    PrecomputedValue.PATH_PACKAGE_LOCATOR.set(differencer, pkgLocator.get());
    PrecomputedValue.BLACKLISTED_PACKAGE_PREFIXES_FILE.set(differencer, PathFragment.EMPTY_FRAGMENT);
    PrecomputedValue.BLAZE_DIRECTORIES.set(differencer, directories);
}