Examples with Backplane build.buildfarm.backplane.Backplane used on opensource projects

Example 1 with Backplane

use of build.buildfarm.backplane.Backplane in project bazel-buildfarm by bazelbuild.

the class RemoteInputStreamFactory method newInput.

@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
public InputStream newInput(Digest blobDigest, long offset, long deadlineAfter, TimeUnit deadlineAfterUnits, RequestMetadata requestMetadata) throws IOException, InterruptedException {
    Set<String> remoteWorkers;
    Set<String> locationSet;
    try {
        Set<String> workers = backplane.getWorkers();
        if (publicName == null) {
            remoteWorkers = workers;
        } else {
            synchronized (workers) {
                remoteWorkers = Sets.difference(workers, ImmutableSet.of(publicName)).immutableCopy();
            }
        }
        locationSet = Sets.newHashSet(Sets.intersection(backplane.getBlobLocationSet(blobDigest), workers));
    } catch (IOException e) {
        throw Status.fromThrowable(e).asRuntimeException();
    }
    if (publicName != null && locationSet.remove(publicName)) {
        backplane.removeBlobLocation(blobDigest, publicName);
    }
    List<String> workersList = new ArrayList<>(locationSet);
    boolean emptyWorkerList = workersList.isEmpty();
    final ListenableFuture<List<String>> populatedWorkerListFuture;
    if (emptyWorkerList) {
        populatedWorkerListFuture = transform(correctMissingBlob(backplane, remoteWorkers, locationSet, this::workerStub, blobDigest, newDirectExecutorService(), requestMetadata), (foundOnWorkers) -> {
            Iterables.addAll(workersList, foundOnWorkers);
            return workersList;
        }, directExecutor());
    } else {
        populatedWorkerListFuture = immediateFuture(workersList);
    }
    SettableFuture<InputStream> inputStreamFuture = SettableFuture.create();
    addCallback(populatedWorkerListFuture, new WorkersCallback(rand) {

        boolean triedCheck = emptyWorkerList;

        @Override
        public void onQueue(Deque<String> workers) {
            Set<String> locationSet = Sets.newHashSet(workers);
            boolean complete = false;
            while (!complete && !workers.isEmpty()) {
                try {
                    inputStreamFuture.set(fetchBlobFromRemoteWorker(blobDigest, workers, offset, deadlineAfter, deadlineAfterUnits, requestMetadata));
                    complete = true;
                } catch (IOException e) {
                    if (workers.isEmpty()) {
                        if (triedCheck) {
                            onFailure(e);
                            return;
                        }
                        triedCheck = true;
                        workersList.clear();
                        ListenableFuture<List<String>> checkedWorkerListFuture = transform(correctMissingBlob(backplane, remoteWorkers, locationSet, RemoteInputStreamFactory.this::workerStub, blobDigest, newDirectExecutorService(), requestMetadata), (foundOnWorkers) -> {
                            Iterables.addAll(workersList, foundOnWorkers);
                            return workersList;
                        }, directExecutor());
                        addCallback(checkedWorkerListFuture, this, directExecutor());
                        complete = true;
                    }
                } catch (InterruptedException e) {
                    complete = true;
                    onFailure(e);
                }
            }
        }

        @SuppressWarnings("NullableProblems")
        @Override
        public void onFailure(Throwable t) {
            Status status = Status.fromThrowable(t);
            if (status.getCode() == Code.NOT_FOUND) {
                inputStreamFuture.setException(new NoSuchFileException(DigestUtil.toString(blobDigest)));
            } else {
                inputStreamFuture.setException(t);
            }
        }
    }, directExecutor());
    try {
        return inputStreamFuture.get();
    } catch (ExecutionException e) {
        Throwable cause = e.getCause();
        Throwables.throwIfUnchecked(cause);
        Throwables.throwIfInstanceOf(cause, IOException.class);
        Throwables.throwIfInstanceOf(cause, InterruptedException.class);
        throw new UncheckedExecutionException(cause);
    }
}

Example 2 with Backplane

use of build.buildfarm.backplane.Backplane in project bazel-buildfarm by bazelbuild.

the class UtilTest method correctMissingBlobChecksAllWorkers.

@Test
public void correctMissingBlobChecksAllWorkers() throws Exception {
    String worker1Name = "worker1";
    String worker2Name = "worker2";
    String worker3Name = "worker3";
    Set<String> workerSet = ImmutableSet.of(worker1Name, worker2Name, worker3Name);
    Digest digest = Digest.newBuilder().setHash("digest").setSizeBytes(1).build();
    ImmutableList<Digest> digests = ImmutableList.of(digest);
    Instance foundInstance = mock(Instance.class);
    when(foundInstance.findMissingBlobs(eq(digests), any(RequestMetadata.class))).thenReturn(immediateFuture(ImmutableList.of()));
    Instance missingInstance = mock(Instance.class);
    when(missingInstance.findMissingBlobs(eq(digests), any(RequestMetadata.class))).thenReturn(immediateFuture(ImmutableList.of(digest)));
    Backplane backplane = mock(Backplane.class);
    Function<String, Instance> workerInstanceFactory = worker -> {
        if (worker.equals(worker1Name)) {
            return missingInstance;
        }
        if (worker.equals(worker2Name) || worker.equals(worker3Name)) {
            return foundInstance;
        }
        return null;
    };
    ListenableFuture<Set<String>> correctFuture = correctMissingBlob(backplane, workerSet, /* originalLocationSet=*/
    ImmutableSet.of(), workerInstanceFactory, digest, directExecutor(), RequestMetadata.getDefaultInstance());
    assertThat(correctFuture.get()).isEqualTo(ImmutableSet.of(worker2Name, worker3Name));
    verify(foundInstance, times(2)).findMissingBlobs(eq(digests), any(RequestMetadata.class));
    verify(missingInstance, times(1)).findMissingBlobs(eq(digests), any(RequestMetadata.class));
    verify(backplane, times(1)).adjustBlobLocations(eq(digest), eq(ImmutableSet.of(worker2Name, worker3Name)), eq(ImmutableSet.of()));
}

Example 3 with Backplane

use of build.buildfarm.backplane.Backplane in project bazel-buildfarm by bazelbuild.

the class UtilTest method correctMissingBlobIgnoresBackplaneException.

@Test
public void correctMissingBlobIgnoresBackplaneException() throws Exception {
    String workerName = "worker";
    Set<String> workerSet = ImmutableSet.of(workerName);
    Digest digest = Digest.newBuilder().setHash("digest").setSizeBytes(1).build();
    ImmutableList<Digest> digests = ImmutableList.of(digest);
    Instance instance = mock(Instance.class);
    when(instance.findMissingBlobs(eq(digests), any(RequestMetadata.class))).thenReturn(immediateFailedFuture(Status.UNKNOWN.asRuntimeException())).thenReturn(immediateFuture(ImmutableList.of()));
    Backplane backplane = mock(Backplane.class);
    doThrow(new IOException("failed to adjustBlobLocations")).when(backplane).adjustBlobLocations(eq(digest), eq(ImmutableSet.of(workerName)), eq(ImmutableSet.of()));
    Function<String, Instance> workerInstanceFactory = worker -> {
        if (worker.equals(workerName)) {
            return instance;
        }
        return null;
    };
    ListenableFuture<Set<String>> correctFuture = correctMissingBlob(backplane, workerSet, /* originalLocationSet=*/
    ImmutableSet.of(), workerInstanceFactory, digest, directExecutor(), RequestMetadata.getDefaultInstance());
    assertThat(correctFuture.get()).isEqualTo(ImmutableSet.of(workerName));
    verify(instance, times(2)).findMissingBlobs(eq(digests), any(RequestMetadata.class));
    verify(backplane, times(1)).adjustBlobLocations(eq(digest), eq(ImmutableSet.of(workerName)), eq(ImmutableSet.of()));
}

Example 4 with Backplane

use of build.buildfarm.backplane.Backplane in project bazel-buildfarm by bazelbuild.

the class UtilTest method correctMissingBlobFailsImmediatelyOnUnretriable.

@Test
public void correctMissingBlobFailsImmediatelyOnUnretriable() throws InterruptedException {
    String workerName = "worker";
    Backplane backplane = mock(Backplane.class);
    Set<String> workerSet = ImmutableSet.of(workerName);
    Digest digest = Digest.newBuilder().setHash("digest-throws-exception-on-find-missing").setSizeBytes(1).build();
    ImmutableList<Digest> digests = ImmutableList.of(digest);
    Instance instance = mock(Instance.class);
    when(instance.findMissingBlobs(eq(digests), any(RequestMetadata.class))).thenReturn(immediateFailedFuture(Status.INVALID_ARGUMENT.asRuntimeException()));
    Function<String, Instance> workerInstanceFactory = worker -> {
        if (worker.equals(workerName)) {
            return instance;
        }
        return null;
    };
    ListenableFuture<Set<String>> correctFuture = correctMissingBlob(backplane, workerSet, /* originalLocationSet=*/
    ImmutableSet.of(), workerInstanceFactory, digest, directExecutor(), RequestMetadata.getDefaultInstance());
    boolean caughtException = false;
    try {
        correctFuture.get();
    } catch (ExecutionException e) {
        caughtException = true;
        Status status = Status.fromThrowable(e.getCause());
        assertThat(status.getCode()).isEqualTo(Code.INVALID_ARGUMENT);
    }
    verify(instance, times(1)).findMissingBlobs(eq(digests), any(RequestMetadata.class));
    assertThat(caughtException).isTrue();
    verifyZeroInteractions(backplane);
}

Example 5 with Backplane

use of build.buildfarm.backplane.Backplane in project bazel-buildfarm by bazelbuild.

the class UtilTest method correctMissingBlobIgnoresUnavailableWorkers.

@Test
public void correctMissingBlobIgnoresUnavailableWorkers() throws Exception {
    String workerName = "worker";
    String unavailableWorkerName = "unavailableWorker";
    Backplane backplane = mock(Backplane.class);
    Set<String> workerSet = ImmutableSet.of(workerName, unavailableWorkerName);
    Digest digest = Digest.newBuilder().setHash("digest").setSizeBytes(1).build();
    ImmutableList<Digest> digests = ImmutableList.of(digest);
    Instance instance = mock(Instance.class);
    when(instance.findMissingBlobs(eq(digests), any(RequestMetadata.class))).thenReturn(immediateFuture(ImmutableList.of()));
    Instance unavailableInstance = mock(Instance.class);
    when(unavailableInstance.findMissingBlobs(eq(digests), any(RequestMetadata.class))).thenReturn(immediateFailedFuture(Status.UNAVAILABLE.asRuntimeException()));
    Function<String, Instance> workerInstanceFactory = worker -> {
        if (worker.equals(workerName)) {
            return instance;
        }
        if (worker.equals(unavailableWorkerName)) {
            return unavailableInstance;
        }
        return null;
    };
    ListenableFuture<Set<String>> correctFuture = correctMissingBlob(backplane, workerSet, /* originalLocationSet=*/
    ImmutableSet.of(), workerInstanceFactory, digest, directExecutor(), RequestMetadata.getDefaultInstance());
    assertThat(correctFuture.get()).isEqualTo(ImmutableSet.of(workerName));
    verify(instance, times(1)).findMissingBlobs(eq(digests), any(RequestMetadata.class));
    verify(unavailableInstance, times(1)).findMissingBlobs(eq(digests), any(RequestMetadata.class));
    verify(backplane, times(1)).adjustBlobLocations(eq(digest), eq(ImmutableSet.of(workerName)), eq(ImmutableSet.of()));
}