Examples with OutputDirectory build.buildfarm.worker.OutputDirectory used on opensource projects

Example 1 with OutputDirectory

use of build.buildfarm.worker.OutputDirectory in project bazel-buildfarm by bazelbuild.

the class OperationQueueWorkerContext method createExecDir.

@Override
public Path createExecDir(String operationName, Map<Digest, Directory> directoriesIndex, Action action, Command command) throws IOException, InterruptedException {
    OutputDirectory outputDirectory = OutputDirectory.parse(command.getOutputFilesList(), command.getOutputDirectoriesList(), command.getEnvironmentVariablesList());
    Path execDir = root.resolve(operationName);
    if (Files.exists(execDir)) {
        Directories.remove(execDir);
    }
    Files.createDirectories(execDir);
    ImmutableList.Builder<String> inputFiles = new ImmutableList.Builder<>();
    ImmutableList.Builder<Digest> inputDirectories = new ImmutableList.Builder<>();
    boolean fetched = false;
    try {
        fetchInputs(execDir, action.getInputRootDigest(), directoriesIndex, outputDirectory, inputFiles, inputDirectories);
        fetched = true;
    } finally {
        if (!fetched) {
            fileCache.decrementReferences(inputFiles.build(), inputDirectories.build());
        }
    }
    rootInputFiles.put(execDir, inputFiles.build());
    rootInputDirectories.put(execDir, inputDirectories.build());
    boolean stamped = false;
    try {
        outputDirectory.stamp(execDir);
        stamped = true;
    } finally {
        if (!stamped) {
            destroyExecDir(execDir);
        }
    }
    if (owner != null) {
        Directories.setAllOwner(execDir, owner);
    }
    return execDir;
}

Example 2 with OutputDirectory

use of build.buildfarm.worker.OutputDirectory in project bazel-buildfarm by bazelbuild.

the class OperationQueueWorkerContext method fetchInputs.

private void fetchInputs(Path execDir, Digest inputRoot, Map<Digest, Directory> directoriesIndex, OutputDirectory outputDirectory, ImmutableList.Builder<String> inputFiles, ImmutableList.Builder<Digest> inputDirectories) throws IOException, InterruptedException {
    Directory directory;
    if (inputRoot.getSizeBytes() == 0) {
        directory = Directory.getDefaultInstance();
    } else {
        directory = directoriesIndex.get(inputRoot);
        if (directory == null) {
            throw new IOException("Directory " + DigestUtil.toString(inputRoot) + " is not in input index");
        }
    }
    getInterruptiblyOrIOException(allAsList(fileCache.putFiles(directory.getFilesList(), directory.getSymlinksList(), execDir, inputFiles, newDirectExecutorService())));
    for (DirectoryNode directoryNode : directory.getDirectoriesList()) {
        Digest digest = directoryNode.getDigest();
        String name = directoryNode.getName();
        OutputDirectory childOutputDirectory = outputDirectory != null ? outputDirectory.getChild(name) : null;
        Path dirPath = execDir.resolve(name);
        if (childOutputDirectory != null || !config.getLinkInputDirectories()) {
            Files.createDirectories(dirPath);
            fetchInputs(dirPath, digest, directoriesIndex, childOutputDirectory, inputFiles, inputDirectories);
        } else {
            inputDirectories.add(digest);
            linkDirectory(dirPath, digest, directoriesIndex);
        }
    }
}

Example 3 with OutputDirectory

use of build.buildfarm.worker.OutputDirectory in project bazel-buildfarm by bazelbuild.

the class CFCExecFileSystem method createExecDir.

@Override
public Path createExecDir(String operationName, Map<Digest, Directory> directoriesIndex, Action action, Command command) throws IOException, InterruptedException {
    Digest inputRootDigest = action.getInputRootDigest();
    OutputDirectory outputDirectory = OutputDirectory.parse(command.getOutputFilesList(), concat(command.getOutputDirectoriesList(), realDirectories(directoriesIndex, inputRootDigest)), command.getEnvironmentVariablesList());
    Path execDir = root.resolve(operationName);
    if (Files.exists(execDir)) {
        Directories.remove(execDir);
    }
    Files.createDirectories(execDir);
    ImmutableList.Builder<String> inputFiles = new ImmutableList.Builder<>();
    ImmutableList.Builder<Digest> inputDirectories = new ImmutableList.Builder<>();
    logger.log(Level.FINE, "ExecFileSystem::createExecDir(" + operationName + ") calling fetchInputs");
    Iterable<ListenableFuture<Void>> fetchedFutures = fetchInputs(execDir, inputRootDigest, directoriesIndex, outputDirectory, inputFiles, inputDirectories);
    boolean success = false;
    try {
        InterruptedException exception = null;
        boolean wasInterrupted = false;
        ImmutableList.Builder<Throwable> exceptions = ImmutableList.builder();
        for (ListenableFuture<Void> fetchedFuture : fetchedFutures) {
            if (exception != null || wasInterrupted) {
                fetchedFuture.cancel(true);
            } else {
                try {
                    fetchedFuture.get();
                } catch (ExecutionException e) {
                    // just to ensure that no other code can react to interrupt status
                    exceptions.add(e.getCause());
                } catch (InterruptedException e) {
                    fetchedFuture.cancel(true);
                    exception = e;
                }
            }
            wasInterrupted = Thread.interrupted() || wasInterrupted;
        }
        if (wasInterrupted) {
            Thread.currentThread().interrupt();
            // unlikely, but worth guarding
            if (exception == null) {
                exception = new InterruptedException();
            }
        }
        if (exception != null) {
            throw exception;
        }
        checkExecErrors(execDir, exceptions.build());
        success = true;
    } finally {
        if (!success) {
            fileCache.decrementReferences(inputFiles.build(), inputDirectories.build());
            Directories.remove(execDir);
        }
    }
    rootInputFiles.put(execDir, inputFiles.build());
    rootInputDirectories.put(execDir, inputDirectories.build());
    logger.log(Level.FINE, "ExecFileSystem::createExecDir(" + operationName + ") stamping output directories");
    boolean stamped = false;
    try {
        outputDirectory.stamp(execDir);
        stamped = true;
    } finally {
        if (!stamped) {
            destroyExecDir(execDir);
        }
    }
    if (owner != null) {
        Directories.setAllOwner(execDir, owner);
    }
    return execDir;
}

Example 4 with OutputDirectory

use of build.buildfarm.worker.OutputDirectory in project bazel-buildfarm by bazelbuild.

the class CFCExecFileSystem method fetchInputs.

private Iterable<ListenableFuture<Void>> fetchInputs(Path path, Digest directoryDigest, Map<Digest, Directory> directoriesIndex, OutputDirectory outputDirectory, ImmutableList.Builder<String> inputFiles, ImmutableList.Builder<Digest> inputDirectories) throws IOException {
    Directory directory = directoriesIndex.get(directoryDigest);
    if (directory == null) {
        // not quite IO...
        throw new IOException("Directory " + DigestUtil.toString(directoryDigest) + " is not in directories index");
    }
    Iterable<ListenableFuture<Void>> downloads = directory.getFilesList().stream().map(fileNode -> put(path, fileNode, inputFiles)).collect(ImmutableList.toImmutableList());
    downloads = concat(downloads, directory.getSymlinksList().stream().map(symlinkNode -> putSymlink(path, symlinkNode)).collect(ImmutableList.toImmutableList()));
    for (DirectoryNode directoryNode : directory.getDirectoriesList()) {
        Digest digest = directoryNode.getDigest();
        String name = directoryNode.getName();
        OutputDirectory childOutputDirectory = outputDirectory != null ? outputDirectory.getChild(name) : null;
        Path dirPath = path.resolve(name);
        if (childOutputDirectory != null || !linkInputDirectories) {
            Files.createDirectories(dirPath);
            downloads = concat(downloads, fetchInputs(dirPath, digest, directoriesIndex, childOutputDirectory, inputFiles, inputDirectories));
        } else {
            downloads = concat(downloads, ImmutableList.of(transform(linkDirectory(dirPath, digest, directoriesIndex), (result) -> {
                // we saw null entries in the built immutable list without synchronization
                synchronized (inputDirectories) {
                    inputDirectories.add(digest);
                }
                return null;
            }, fetchService)));
        }
        if (Thread.currentThread().isInterrupted()) {
            break;
        }
    }
    return downloads;
}