Examples with PathWithUnixSeparators com.facebook.buck.distributed.thrift.PathWithUnixSeparators used on opensource projects

Example 1 with PathWithUnixSeparators

use of com.facebook.buck.distributed.thrift.PathWithUnixSeparators in project buck by facebook.

the class RecordingProjectFileHashCache method processDirectory.

private List<PathWithUnixSeparators> processDirectory(Path path, Queue<Path> remainingPaths) throws IOException {
    List<PathWithUnixSeparators> childrenRelativePaths = new ArrayList<>();
    for (Path relativeChildPath : projectFilesystem.getDirectoryContents(path)) {
        childrenRelativePaths.add(new PathWithUnixSeparators(MorePaths.pathWithUnixSeparators(relativeChildPath)));
        remainingPaths.add(relativeChildPath);
    }
    return childrenRelativePaths;
}

Example 2 with PathWithUnixSeparators

use of com.facebook.buck.distributed.thrift.PathWithUnixSeparators in project buck by facebook.

the class DistBuildServiceTest method canUploadFiles.

@Test
public void canUploadFiles() throws Exception {
    final List<Boolean> fileExistence = Arrays.asList(true, false, true);
    Capture<FrontendRequest> containsRequest = EasyMock.newCapture();
    FrontendResponse containsResponse = new FrontendResponse();
    containsResponse.setType(FrontendRequestType.CAS_CONTAINS);
    CASContainsResponse casContainsResponse = new CASContainsResponse();
    casContainsResponse.setExists(fileExistence);
    containsResponse.setCasContainsResponse(casContainsResponse);
    containsResponse.setWasSuccessful(true);
    EasyMock.expect(frontendService.makeRequest(EasyMock.capture(containsRequest))).andReturn(containsResponse).once();
    Capture<FrontendRequest> storeRequest = EasyMock.newCapture();
    FrontendResponse storeResponse = new FrontendResponse();
    storeResponse.setType(FrontendRequestType.STORE_LOCAL_CHANGES);
    storeResponse.setWasSuccessful(true);
    EasyMock.expect(frontendService.makeRequest(EasyMock.capture(storeRequest))).andReturn(storeResponse).once();
    EasyMock.replay(frontendService);
    BuildJobStateFileHashEntry[] files = new BuildJobStateFileHashEntry[3];
    for (int i = 0; i < 3; i++) {
        files[i] = new BuildJobStateFileHashEntry();
        files[i].setHashCode(Integer.toString(i));
        files[i].setContents(("content" + Integer.toString(i)).getBytes());
        files[i].setPath(new PathWithUnixSeparators("/tmp/" + i));
    }
    List<BuildJobStateFileHashes> fileHashes = new ArrayList<>();
    fileHashes.add(new BuildJobStateFileHashes());
    fileHashes.get(0).setCellIndex(0);
    fileHashes.get(0).setEntries(new ArrayList<BuildJobStateFileHashEntry>());
    fileHashes.get(0).getEntries().add(files[0]);
    fileHashes.get(0).getEntries().add(files[1]);
    fileHashes.add(new BuildJobStateFileHashes());
    fileHashes.get(1).setCellIndex(1);
    fileHashes.get(1).setEntries(new ArrayList<BuildJobStateFileHashEntry>());
    fileHashes.get(1).getEntries().add(files[2]);
    distBuildService.uploadMissingFiles(fileHashes, executor).get();
    Assert.assertEquals(containsRequest.getValue().getType(), FrontendRequestType.CAS_CONTAINS);
    Assert.assertTrue(containsRequest.getValue().isSetCasContainsRequest());
    Assert.assertTrue(containsRequest.getValue().getCasContainsRequest().isSetContentSha1s());
    Assert.assertEquals(new HashSet<String>(containsRequest.getValue().getCasContainsRequest().getContentSha1s()), new HashSet<String>(Arrays.asList("0", "1", "2")));
    Assert.assertEquals(storeRequest.getValue().getType(), FrontendRequestType.STORE_LOCAL_CHANGES);
    Assert.assertTrue(storeRequest.getValue().isSetStoreLocalChangesRequest());
    Assert.assertTrue(storeRequest.getValue().getStoreLocalChangesRequest().isSetFiles());
    Assert.assertEquals(storeRequest.getValue().getStoreLocalChangesRequest().getFiles().size(), 1);
    Assert.assertEquals(storeRequest.getValue().getStoreLocalChangesRequest().getFiles().get(0).getContentHash(), "1");
    Assert.assertTrue(Arrays.equals(storeRequest.getValue().getStoreLocalChangesRequest().getFiles().get(0).getContent(), "content1".getBytes()));
}

Example 3 with PathWithUnixSeparators

use of com.facebook.buck.distributed.thrift.PathWithUnixSeparators in project buck by facebook.

the class MaterializerProjectFileHashCache method materializeDirectory.

private synchronized void materializeDirectory(Path path, BuildJobStateFileHashEntry fileHashEntry, Queue<Path> remainingPaths) throws IOException {
    if (materializedPaths.contains(path)) {
        return;
    }
    projectFilesystem.mkdirs(path);
    for (PathWithUnixSeparators unixPath : fileHashEntry.getChildren()) {
        Path absPath = projectFilesystem.resolve(Paths.get(unixPath.getPath()));
        Path relPath = projectFilesystem.getPathRelativeToProjectRoot(absPath).get();
        remainingPaths.add(relPath);
    }
}

Example 4 with PathWithUnixSeparators

use of com.facebook.buck.distributed.thrift.PathWithUnixSeparators in project buck by facebook.

the class RecordingProjectFileHashCache method get.

@Override
public HashCode get(Path relPath) throws IOException {
    checkIsRelative(relPath);
    Queue<Path> remainingPaths = new LinkedList<>();
    remainingPaths.add(relPath);
    while (remainingPaths.size() > 0) {
        Path nextPath = remainingPaths.remove();
        HashCode hashCode = delegate.get(nextPath);
        List<PathWithUnixSeparators> children = ImmutableList.of();
        if (projectFilesystem.isDirectory(nextPath)) {
            children = processDirectory(nextPath, remainingPaths);
        }
        synchronized (this) {
            if (!remoteFileHashes.containsAndAddPath(nextPath)) {
                record(nextPath, Optional.empty(), hashCode, children);
            }
        }
    }
    return delegate.get(relPath);
}

Example 5 with PathWithUnixSeparators

use of com.facebook.buck.distributed.thrift.PathWithUnixSeparators in project buck by facebook.

the class RecordingProjectFileHashCache method record.

private synchronized void record(Path relPath, Optional<String> memRelPath, HashCode hashCode, List<PathWithUnixSeparators> children) {
    LOG.verbose("Recording path: [%s]", projectFilesystem.resolve(relPath).toAbsolutePath());
    Optional<Path> pathRelativeToProjectRoot = projectFilesystem.getPathRelativeToProjectRoot(relPath);
    BuildJobStateFileHashEntry fileHashEntry = new BuildJobStateFileHashEntry();
    boolean pathIsAbsolute = allRecordedPathsAreAbsolute;
    fileHashEntry.setPathIsAbsolute(pathIsAbsolute);
    Path entryKey = pathIsAbsolute ? projectFilesystem.resolve(relPath).toAbsolutePath() : pathRelativeToProjectRoot.get();
    boolean isDirectory = projectFilesystem.isDirectory(relPath);
    Path realPath = findRealPath(relPath);
    boolean realPathInsideProject = projectFilesystem.getPathRelativeToProjectRoot(realPath).isPresent();
    // meta-data about this before it is re-materialized. See findSymlinkRoot() for more details.
    if (!realPathInsideProject && !pathIsAbsolute) {
        Pair<Path, Path> symLinkRootAndTarget = findSymlinkRoot(projectFilesystem.resolve(relPath).toAbsolutePath());
        Path symLinkRoot = projectFilesystem.getPathRelativeToProjectRoot(symLinkRootAndTarget.getFirst()).get();
        fileHashEntry.setRootSymLink(new PathWithUnixSeparators(MorePaths.pathWithUnixSeparators(symLinkRoot)));
        fileHashEntry.setRootSymLinkTarget(new PathWithUnixSeparators(MorePaths.pathWithUnixSeparators(symLinkRootAndTarget.getSecond().toAbsolutePath())));
    }
    fileHashEntry.setIsDirectory(isDirectory);
    fileHashEntry.setHashCode(hashCode.toString());
    fileHashEntry.setPath(new PathWithUnixSeparators(MorePaths.pathWithUnixSeparators(entryKey)));
    if (memRelPath.isPresent()) {
        fileHashEntry.setArchiveMemberPath(memRelPath.get().toString());
    }
    if (!isDirectory && !pathIsAbsolute && realPathInsideProject) {
        try {
            // TODO(shivanker, ruibm): Don't read everything in memory right away.
            Path absPath = projectFilesystem.resolve(relPath).toAbsolutePath();
            fileHashEntry.setContents(Files.readAllBytes(absPath));
            fileHashEntry.setIsExecutable(absPath.toFile().canExecute());
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    } else if (isDirectory && !pathIsAbsolute && realPathInsideProject) {
        fileHashEntry.setChildren(children);
    }
    fileHashEntry.setMaterializeDuringPreloading(materializeCurrentFileDuringPreloading);
    // TODO(alisdair04): handling for symlink to internal directory (including infinite loop).
    remoteFileHashes.addEntry(fileHashEntry);
}