Examples with NullWrite build.buildfarm.common.Write.NullWrite used on opensource projects

Example 1 with NullWrite

use of build.buildfarm.common.Write.NullWrite in project bazel-buildfarm by bazelbuild.

the class CASFileCacheTest method interruptDeferredDuringExpirations.

@SuppressWarnings("unchecked")
@Test
public void interruptDeferredDuringExpirations() throws IOException, InterruptedException {
    Blob expiringBlob;
    try (ByteString.Output out = ByteString.newOutput(1024)) {
        for (int i = 0; i < 1024; i++) {
            out.write(0);
        }
        expiringBlob = new Blob(out.toByteString(), DIGEST_UTIL);
    }
    fileCache.put(expiringBlob);
    // state of CAS
    // 1024-byte key
    AtomicReference<Throwable> exRef = new AtomicReference(null);
    // 0 = not blocking
    // 1 = blocking
    // 2 = delegate write
    AtomicInteger writeState = new AtomicInteger(0);
    // this will ensure that the discharge task is blocked until we release it
    Future<Void> blockingExpiration = expireService.submit(() -> {
        writeState.getAndIncrement();
        while (writeState.get() != 0) {
            try {
                MICROSECONDS.sleep(1);
            } catch (InterruptedException e) {
            // ignore
            }
        }
        return null;
    });
    when(delegate.getWrite(eq(expiringBlob.getDigest()), any(UUID.class), any(RequestMetadata.class))).thenReturn(new NullWrite() {

        @Override
        public FeedbackOutputStream getOutput(long deadlineAfter, TimeUnit deadlineAfterUnits, Runnable onReadyHandler) throws IOException {
            try {
                while (writeState.get() != 1) {
                    MICROSECONDS.sleep(1);
                }
            } catch (InterruptedException e) {
                throw new IOException(e);
            }
            // move into output stream state
            writeState.getAndIncrement();
            return super.getOutput(deadlineAfter, deadlineAfterUnits, onReadyHandler);
        }
    });
    Thread expiringThread = new Thread(() -> {
        try {
            fileCache.put(new Blob(ByteString.copyFromUtf8("Hello, World"), DIGEST_UTIL));
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
        fail("should not get here");
    });
    expiringThread.setUncaughtExceptionHandler((t, e) -> exRef.set(e));
    // wait for blocking state
    while (writeState.get() != 1) {
        MICROSECONDS.sleep(1);
    }
    expiringThread.start();
    while (writeState.get() != 2) {
        MICROSECONDS.sleep(1);
    }
    // expiry has been initiated, thread should be waiting
    // just trying to ensure that we've reached the future wait point
    MICROSECONDS.sleep(10);
    // hopefully this will be scheduled *after* the discharge task
    Future<Void> completedExpiration = expireService.submit(() -> null);
    // interrupt it
    expiringThread.interrupt();
    assertThat(expiringThread.isAlive()).isTrue();
    assertThat(completedExpiration.isDone()).isFalse();
    writeState.set(0);
    while (!blockingExpiration.isDone()) {
        MICROSECONDS.sleep(1);
    }
    expiringThread.join();
    // CAS should now be empty due to expiration and failed put
    while (!completedExpiration.isDone()) {
        MICROSECONDS.sleep(1);
    }
    assertThat(fileCache.size()).isEqualTo(0);
    Throwable t = exRef.get();
    assertThat(t).isNotNull();
    t = t.getCause();
    assertThat(t).isNotNull();
    assertThat(t).isInstanceOf(InterruptedException.class);
}

Example 2 with NullWrite

use of build.buildfarm.common.Write.NullWrite in project bazel-buildfarm by bazelbuild.

the class CASFileCacheTest method readThroughSwitchedToLocalContinues.

@Test
public void readThroughSwitchedToLocalContinues() throws Exception {
    ByteString content = ByteString.copyFromUtf8("Hello, World");
    Blob blob = new Blob(content, DIGEST_UTIL);
    ExecutorService service = newSingleThreadExecutor();
    SettableFuture<Void> writeComplete = SettableFuture.create();
    // we need to register callbacks on the shared write future
    Write write = new NullWrite() {

        @Override
        public ListenableFuture<Long> getFuture() {
            return Futures.transform(writeComplete, result -> blob.getDigest().getSizeBytes(), directExecutor());
        }

        @Override
        public FeedbackOutputStream getOutput(long deadlineAfter, TimeUnit deadlineAfterUnits, Runnable onReadyHandler) {
            return new FeedbackOutputStream() {

                int offset = 0;

                @Override
                public void write(int b) {
                    throw new UnsupportedOperationException();
                }

                @Override
                public void write(byte[] buf, int ofs, int len) throws IOException {
                    // hangs on second read
                    if (offset == 6) {
                        service.submit(() -> writeComplete.set(null));
                        throw new ClosedChannelException();
                    }
                    offset += len;
                }

                @Override
                public boolean isReady() {
                    return true;
                }
            };
        }
    };
    when(delegate.getWrite(eq(blob.getDigest()), any(UUID.class), any(RequestMetadata.class))).thenReturn(write);
    when(delegate.newInput(eq(blob.getDigest()), eq(0L))).thenReturn(content.newInput());
    // the switch will reset to this point
    InputStream switchedIn = content.newInput();
    switchedIn.skip(6);
    when(delegate.newInput(eq(blob.getDigest()), eq(6L))).thenReturn(switchedIn);
    InputStream in = fileCache.newReadThroughInput(blob.getDigest(), 0, write);
    byte[] buf = new byte[content.size()];
    // advance to the middle of the content
    assertThat(in.read(buf, 0, 6)).isEqualTo(6);
    assertThat(ByteString.copyFrom(buf, 0, 6)).isEqualTo(content.substring(0, 6));
    verify(delegate, times(1)).newInput(blob.getDigest(), 0L);
    // read the remaining content
    int remaining = content.size() - 6;
    assertThat(in.read(buf, 6, remaining)).isEqualTo(remaining);
    assertThat(ByteString.copyFrom(buf)).isEqualTo(content);
    if (!shutdownAndAwaitTermination(service, 1, SECONDS)) {
        throw new RuntimeException("could not shut down service");
    }
}

Example 3 with NullWrite

use of build.buildfarm.common.Write.NullWrite in project bazel-buildfarm by bazelbuild.

the class CASFileCacheTest method setUp.

@Before
public void setUp() throws IOException, InterruptedException {
    MockitoAnnotations.initMocks(this);
    when(delegate.getWrite(any(Digest.class), any(UUID.class), any(RequestMetadata.class))).thenReturn(new NullWrite());
    when(delegate.newInput(any(Digest.class), any(Long.class))).thenThrow(new NoSuchFileException("null sink delegate"));
    blobs = Maps.newHashMap();
    putService = newSingleThreadExecutor();
    storage = Maps.newConcurrentMap();
    expireService = newSingleThreadExecutor();
    fileCache = new CASFileCache(root, /* maxSizeInBytes=*/
    1024, /* maxEntrySizeInBytes=*/
    1024, /* hexBucketLevels=*/
    1, storeFileDirsIndexInMemory, DIGEST_UTIL, expireService, /* accessRecorder=*/
    directExecutor(), storage, /* directoriesIndexDbName=*/
    ":memory:", onPut, onExpire, delegate) {

        @Override
        protected InputStream newExternalInput(Digest digest) throws IOException {
            ByteString content = blobs.get(digest);
            if (content == null) {
                return fileCache.newTransparentInput(digest, 0);
            }
            return content.substring((int) (long) 0).newInput();
        }
    };
    // do this so that we can remove the cache root dir
    fileCache.initializeRootDirectory();
}

Example 4 with NullWrite

use of build.buildfarm.common.Write.NullWrite in project bazel-buildfarm by bazelbuild.

the class ShardInstanceTest method queueWithFailedCacheCheckContinues.

@Test
public void queueWithFailedCacheCheckContinues() throws Exception {
    Action action = createAction();
    ActionKey actionKey = DIGEST_UTIL.computeActionKey(action);
    ExecuteEntry executeEntry = ExecuteEntry.newBuilder().setOperationName("operation-with-erroring-action-result").setActionDigest(actionKey.getDigest()).build();
    when(mockBackplane.propertiesEligibleForQueue(Matchers.anyList())).thenReturn(true);
    when(mockBackplane.canQueue()).thenReturn(true);
    when(mockBackplane.getActionResult(eq(actionKey))).thenThrow(new IOException(Status.UNAVAILABLE.asException()));
    doAnswer(answer((digest, uuid) -> new NullWrite())).when(mockWorkerInstance).getBlobWrite(any(Digest.class), any(UUID.class), any(RequestMetadata.class));
    Poller poller = mock(Poller.class);
    instance.queue(executeEntry, poller, DEFAULT_TIMEOUT).get(QUEUE_TEST_TIMEOUT_SECONDS, SECONDS);
    verify(mockBackplane, times(1)).queue(any(QueueEntry.class), any(Operation.class));
    verify(mockBackplane, times(1)).putOperation(any(Operation.class), eq(CACHE_CHECK));
    verify(poller, atLeastOnce()).pause();
}

Example 5 with NullWrite

use of build.buildfarm.common.Write.NullWrite in project bazel-buildfarm by bazelbuild.

the class ShardInstanceTest method queueOperationPutFailureCancelsOperation.

@SuppressWarnings("unchecked")
@Test
public void queueOperationPutFailureCancelsOperation() throws Exception {
    Action action = createAction();
    Digest actionDigest = DIGEST_UTIL.compute(action);
    when(mockWorkerInstance.findMissingBlobs(any(Iterable.class), any(RequestMetadata.class))).thenReturn(immediateFuture(ImmutableList.of()));
    doAnswer(answer((digest, uuid) -> new NullWrite())).when(mockWorkerInstance).getBlobWrite(any(Digest.class), any(UUID.class), any(RequestMetadata.class));
    StatusRuntimeException queueException = Status.UNAVAILABLE.asRuntimeException();
    doAnswer((invocation) -> {
        throw new IOException(queueException);
    }).when(mockBackplane).queue(any(QueueEntry.class), any(Operation.class));
    ExecuteEntry executeEntry = ExecuteEntry.newBuilder().setOperationName("queue-operation-put-failure-cancels-operation").setActionDigest(actionDigest).setSkipCacheLookup(true).build();
    when(mockBackplane.propertiesEligibleForQueue(Matchers.anyList())).thenReturn(true);
    when(mockBackplane.canQueue()).thenReturn(true);
    Poller poller = mock(Poller.class);
    boolean unavailableExceptionCaught = false;
    try {
        // anything more would be unreasonable
        instance.queue(executeEntry, poller, DEFAULT_TIMEOUT).get(QUEUE_TEST_TIMEOUT_SECONDS, SECONDS);
    } catch (ExecutionException e) {
        com.google.rpc.Status status = StatusProto.fromThrowable(e);
        if (status.getCode() == Code.UNAVAILABLE.getNumber()) {
            unavailableExceptionCaught = true;
        } else {
            throw e;
        }
    }
    assertThat(unavailableExceptionCaught).isTrue();
    verify(mockBackplane, times(1)).queue(any(QueueEntry.class), any(Operation.class));
    ExecuteResponse executeResponse = ExecuteResponse.newBuilder().setStatus(com.google.rpc.Status.newBuilder().setCode(queueException.getStatus().getCode().value())).build();
    assertResponse(executeResponse);
    verify(poller, atLeastOnce()).pause();
}