Examples with FileInfo com.facebook.buck.distributed.thrift.FileInfo used on opensource projects

Example 1 with FileInfo

use of com.facebook.buck.distributed.thrift.FileInfo in project buck by facebook.

the class BuckVersionUtil method createFromLocalBinary.

public static BuckVersion createFromLocalBinary(Path localBuckBinary) throws IOException {
    byte[] data = Files.readAllBytes(localBuckBinary);
    String hash = Hashing.sha1().newHasher().putBytes(data).hash().toString();
    FileInfo buckBinary = new FileInfo();
    buckBinary.setContent(data);
    buckBinary.setContentHash(hash);
    BuckVersion buckVersion = new BuckVersion();
    buckVersion.setType(BuckVersionType.DEVELOPMENT);
    buckVersion.setDevelopmentVersion(buckBinary);
    return buckVersion;
}

Example 2 with FileInfo

use of com.facebook.buck.distributed.thrift.FileInfo in project buck by facebook.

the class DistBuildService method uploadBuckDotFiles.

public ListenableFuture<Void> uploadBuckDotFiles(final StampedeId id, final ProjectFilesystem filesystem, FileHashCache fileHashCache, ListeningExecutorService executorService) throws IOException {
    ListenableFuture<Pair<List<FileInfo>, List<PathInfo>>> filesFuture = executorService.submit(() -> {
        List<Path> buckDotFilesExceptConfig = Lists.newArrayList();
        for (Path path : filesystem.getDirectoryContents(filesystem.getRootPath())) {
            String fileName = path.getFileName().toString();
            if (!filesystem.isDirectory(path) && !filesystem.isSymLink(path) && fileName.startsWith(".") && fileName.contains("buck") && !fileName.startsWith(".buckconfig")) {
                buckDotFilesExceptConfig.add(path);
            }
        }
        List<FileInfo> fileEntriesToUpload = new LinkedList<>();
        List<PathInfo> pathEntriesToUpload = new LinkedList<>();
        for (Path path : buckDotFilesExceptConfig) {
            FileInfo fileInfoObject = new FileInfo();
            fileInfoObject.setContent(filesystem.readFileIfItExists(path).get().getBytes());
            fileInfoObject.setContentHash(fileHashCache.get(path.toAbsolutePath()).toString());
            fileEntriesToUpload.add(fileInfoObject);
            PathInfo pathInfoObject = new PathInfo();
            pathInfoObject.setPath(path.toString());
            pathInfoObject.setContentHash(fileHashCache.get(path.toAbsolutePath()).toString());
            pathEntriesToUpload.add(pathInfoObject);
        }
        return new Pair<>(fileEntriesToUpload, pathEntriesToUpload);
    });
    ListenableFuture<Void> setFilesFuture = Futures.transformAsync(filesFuture, filesAndPaths -> {
        setBuckDotFiles(id, filesAndPaths.getSecond());
        return Futures.immediateFuture(null);
    }, executorService);
    ListenableFuture<Void> uploadFilesFuture = Futures.transformAsync(filesFuture, filesAndPaths -> {
        uploadMissingFilesFromList(filesAndPaths.getFirst(), executorService);
        return Futures.immediateFuture(null);
    }, executorService);
    return Futures.transform(Futures.allAsList(ImmutableList.of(setFilesFuture, uploadFilesFuture)), new Function<List<Void>, Void>() {

        @Nullable
        @Override
        public Void apply(@Nullable List<Void> input) {
            return null;
        }
    });
}

Example 3 with FileInfo

use of com.facebook.buck.distributed.thrift.FileInfo in project buck by facebook.

the class DistBuildService method fetchSourceFile.

public InputStream fetchSourceFile(String hashCode) throws IOException {
    FrontendRequest request = createFetchSourceFileRequest(hashCode);
    FrontendResponse response = makeRequestChecked(request);
    Preconditions.checkState(response.isSetFetchSourceFilesResponse());
    Preconditions.checkState(response.getFetchSourceFilesResponse().isSetFiles());
    FetchSourceFilesResponse fetchSourceFilesResponse = response.getFetchSourceFilesResponse();
    Preconditions.checkState(1 == fetchSourceFilesResponse.getFilesSize());
    FileInfo file = fetchSourceFilesResponse.getFiles().get(0);
    Preconditions.checkState(file.isSetContent());
    return new ByteArrayInputStream(file.getContent());
}

Example 4 with FileInfo

use of com.facebook.buck.distributed.thrift.FileInfo in project buck by facebook.

the class DistBuildService method uploadMissingFiles.

public ListenableFuture<Void> uploadMissingFiles(final List<BuildJobStateFileHashes> fileHashes, ListeningExecutorService executorService) {
    List<FileInfo> requiredFiles = new ArrayList<>();
    for (BuildJobStateFileHashes filesystem : fileHashes) {
        if (!filesystem.isSetEntries()) {
            continue;
        }
        for (BuildJobStateFileHashEntry file : filesystem.entries) {
            if (file.isSetRootSymLink()) {
                LOG.info("File with path [%s] is a symlink. Skipping upload..", file.path.getPath());
                continue;
            } else if (file.isIsDirectory()) {
                LOG.info("Path [%s] is a directory. Skipping upload..", file.path.getPath());
                continue;
            } else if (file.isPathIsAbsolute()) {
                LOG.info("Path [%s] is absolute. Skipping upload..", file.path.getPath());
                continue;
            }
            // TODO(shivanker): Eventually, we won't have file contents in BuildJobState.
            // Then change this code to load file contents inline (only for missing files)
            FileInfo fileInfo = new FileInfo();
            fileInfo.setContent(file.getContents());
            fileInfo.setContentHash(file.getHashCode());
            requiredFiles.add(fileInfo);
        }
    }
    return uploadMissingFilesFromList(requiredFiles, executorService);
}

Example 5 with FileInfo

use of com.facebook.buck.distributed.thrift.FileInfo in project buck by facebook.

the class DistBuildService method uploadMissingFilesFromList.

private ListenableFuture<Void> uploadMissingFilesFromList(final List<FileInfo> fileList, ListeningExecutorService executorService) {
    return executorService.submit(() -> {
        Map<String, FileInfo> sha1ToFileInfo = new HashMap<>();
        for (FileInfo file : fileList) {
            sha1ToFileInfo.put(file.getContentHash(), file);
        }
        List<String> contentHashes = ImmutableList.copyOf(sha1ToFileInfo.keySet());
        CASContainsRequest containsReq = new CASContainsRequest();
        containsReq.setContentSha1s(contentHashes);
        FrontendRequest request = new FrontendRequest();
        request.setType(FrontendRequestType.CAS_CONTAINS);
        request.setCasContainsRequest(containsReq);
        FrontendResponse response = makeRequestChecked(request);
        Preconditions.checkState(response.getCasContainsResponse().exists.size() == contentHashes.size());
        List<Boolean> isPresent = response.getCasContainsResponse().exists;
        List<FileInfo> filesToBeUploaded = new LinkedList<>();
        for (int i = 0; i < isPresent.size(); ++i) {
            if (isPresent.get(i)) {
                continue;
            }
            filesToBeUploaded.add(sha1ToFileInfo.get(contentHashes.get(i)));
        }
        LOG.info("%d out of %d files already exist in the cache. Uploading %d files..", sha1ToFileInfo.size() - filesToBeUploaded.size(), sha1ToFileInfo.size(), filesToBeUploaded.size());
        request = new FrontendRequest();
        StoreLocalChangesRequest storeReq = new StoreLocalChangesRequest();
        storeReq.setFiles(filesToBeUploaded);
        request.setType(FrontendRequestType.STORE_LOCAL_CHANGES);
        request.setStoreLocalChangesRequest(storeReq);
        makeRequestChecked(request);
        // No response expected.
        return null;
    });
}