Examples with Blob build.buildfarm.cas.ContentAddressableStorage.Blob used on opensource projects

Example 1 with Blob

use of build.buildfarm.cas.ContentAddressableStorage.Blob in project bazel-buildfarm by bazelbuild.

the class CASFileCacheTest method interruptDeferredDuringExpirations.

@SuppressWarnings("unchecked")
@Test
public void interruptDeferredDuringExpirations() throws IOException, InterruptedException {
    Blob expiringBlob;
    try (ByteString.Output out = ByteString.newOutput(1024)) {
        for (int i = 0; i < 1024; i++) {
            out.write(0);
        }
        expiringBlob = new Blob(out.toByteString(), DIGEST_UTIL);
    }
    fileCache.put(expiringBlob);
    // state of CAS
    // 1024-byte key
    AtomicReference<Throwable> exRef = new AtomicReference(null);
    // 0 = not blocking
    // 1 = blocking
    // 2 = delegate write
    AtomicInteger writeState = new AtomicInteger(0);
    // this will ensure that the discharge task is blocked until we release it
    Future<Void> blockingExpiration = expireService.submit(() -> {
        writeState.getAndIncrement();
        while (writeState.get() != 0) {
            try {
                MICROSECONDS.sleep(1);
            } catch (InterruptedException e) {
            // ignore
            }
        }
        return null;
    });
    when(delegate.getWrite(eq(expiringBlob.getDigest()), any(UUID.class), any(RequestMetadata.class))).thenReturn(new NullWrite() {

        @Override
        public FeedbackOutputStream getOutput(long deadlineAfter, TimeUnit deadlineAfterUnits, Runnable onReadyHandler) throws IOException {
            try {
                while (writeState.get() != 1) {
                    MICROSECONDS.sleep(1);
                }
            } catch (InterruptedException e) {
                throw new IOException(e);
            }
            // move into output stream state
            writeState.getAndIncrement();
            return super.getOutput(deadlineAfter, deadlineAfterUnits, onReadyHandler);
        }
    });
    Thread expiringThread = new Thread(() -> {
        try {
            fileCache.put(new Blob(ByteString.copyFromUtf8("Hello, World"), DIGEST_UTIL));
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
        fail("should not get here");
    });
    expiringThread.setUncaughtExceptionHandler((t, e) -> exRef.set(e));
    // wait for blocking state
    while (writeState.get() != 1) {
        MICROSECONDS.sleep(1);
    }
    expiringThread.start();
    while (writeState.get() != 2) {
        MICROSECONDS.sleep(1);
    }
    // expiry has been initiated, thread should be waiting
    // just trying to ensure that we've reached the future wait point
    MICROSECONDS.sleep(10);
    // hopefully this will be scheduled *after* the discharge task
    Future<Void> completedExpiration = expireService.submit(() -> null);
    // interrupt it
    expiringThread.interrupt();
    assertThat(expiringThread.isAlive()).isTrue();
    assertThat(completedExpiration.isDone()).isFalse();
    writeState.set(0);
    while (!blockingExpiration.isDone()) {
        MICROSECONDS.sleep(1);
    }
    expiringThread.join();
    // CAS should now be empty due to expiration and failed put
    while (!completedExpiration.isDone()) {
        MICROSECONDS.sleep(1);
    }
    assertThat(fileCache.size()).isEqualTo(0);
    Throwable t = exRef.get();
    assertThat(t).isNotNull();
    t = t.getCause();
    assertThat(t).isNotNull();
    assertThat(t).isInstanceOf(InterruptedException.class);
}

Example 2 with Blob

use of build.buildfarm.cas.ContentAddressableStorage.Blob in project bazel-buildfarm by bazelbuild.

the class CASFileCacheTest method duplicateExpiredEntrySuppressesDigestExpiration.

@Test
public void duplicateExpiredEntrySuppressesDigestExpiration() throws IOException, InterruptedException {
    Blob expiringBlob;
    try (ByteString.Output out = ByteString.newOutput(512)) {
        for (int i = 0; i < 512; i++) {
            out.write(0);
        }
        expiringBlob = new Blob(out.toByteString(), DIGEST_UTIL);
    }
    blobs.put(expiringBlob.getDigest(), expiringBlob.getData());
    decrementReference(// expected eviction
    fileCache.put(expiringBlob.getDigest(), /* isExecutable=*/
    false));
    blobs.clear();
    decrementReference(fileCache.put(expiringBlob.getDigest(), /* isExecutable=*/
    true));
    // should be fed from storage directly, not through delegate
    fileCache.put(new Blob(ByteString.copyFromUtf8("Hello, World"), DIGEST_UTIL));
    verifyZeroInteractions(onExpire);
    // assert expiration of non-executable digest
    String expiringKey = fileCache.getKey(expiringBlob.getDigest(), /* isExecutable=*/
    false);
    assertThat(storage.containsKey(expiringKey)).isFalse();
    assertThat(Files.exists(fileCache.getPath(expiringKey))).isFalse();
}

Example 3 with Blob

use of build.buildfarm.cas.ContentAddressableStorage.Blob in project bazel-buildfarm by bazelbuild.

the class CASFileCacheTest method readThroughSwitchedToLocalContinues.

@Test
public void readThroughSwitchedToLocalContinues() throws Exception {
    ByteString content = ByteString.copyFromUtf8("Hello, World");
    Blob blob = new Blob(content, DIGEST_UTIL);
    ExecutorService service = newSingleThreadExecutor();
    SettableFuture<Void> writeComplete = SettableFuture.create();
    // we need to register callbacks on the shared write future
    Write write = new NullWrite() {

        @Override
        public ListenableFuture<Long> getFuture() {
            return Futures.transform(writeComplete, result -> blob.getDigest().getSizeBytes(), directExecutor());
        }

        @Override
        public FeedbackOutputStream getOutput(long deadlineAfter, TimeUnit deadlineAfterUnits, Runnable onReadyHandler) {
            return new FeedbackOutputStream() {

                int offset = 0;

                @Override
                public void write(int b) {
                    throw new UnsupportedOperationException();
                }

                @Override
                public void write(byte[] buf, int ofs, int len) throws IOException {
                    // hangs on second read
                    if (offset == 6) {
                        service.submit(() -> writeComplete.set(null));
                        throw new ClosedChannelException();
                    }
                    offset += len;
                }

                @Override
                public boolean isReady() {
                    return true;
                }
            };
        }
    };
    when(delegate.getWrite(eq(blob.getDigest()), any(UUID.class), any(RequestMetadata.class))).thenReturn(write);
    when(delegate.newInput(eq(blob.getDigest()), eq(0L))).thenReturn(content.newInput());
    // the switch will reset to this point
    InputStream switchedIn = content.newInput();
    switchedIn.skip(6);
    when(delegate.newInput(eq(blob.getDigest()), eq(6L))).thenReturn(switchedIn);
    InputStream in = fileCache.newReadThroughInput(blob.getDigest(), 0, write);
    byte[] buf = new byte[content.size()];
    // advance to the middle of the content
    assertThat(in.read(buf, 0, 6)).isEqualTo(6);
    assertThat(ByteString.copyFrom(buf, 0, 6)).isEqualTo(content.substring(0, 6));
    verify(delegate, times(1)).newInput(blob.getDigest(), 0L);
    // read the remaining content
    int remaining = content.size() - 6;
    assertThat(in.read(buf, 6, remaining)).isEqualTo(remaining);
    assertThat(ByteString.copyFrom(buf)).isEqualTo(content);
    if (!shutdownAndAwaitTermination(service, 1, SECONDS)) {
        throw new RuntimeException("could not shut down service");
    }
}

Example 4 with Blob

use of build.buildfarm.cas.ContentAddressableStorage.Blob in project bazel-buildfarm by bazelbuild.

the class CASFileCacheTest method readRemovesNonexistentEntry.

@Test
public void readRemovesNonexistentEntry() throws IOException, InterruptedException {
    ByteString content = ByteString.copyFromUtf8("Hello, World");
    Blob blob = new Blob(content, DIGEST_UTIL);
    fileCache.put(blob);
    String key = fileCache.getKey(blob.getDigest(), /* isExecutable=*/
    false);
    // putCreatesFile verifies this
    Files.delete(fileCache.getPath(key));
    // update entry with expired deadline
    storage.get(key).existsDeadline = Deadline.after(0, SECONDS);
    try (InputStream in = fileCache.newInput(blob.getDigest(), /* offset=*/
    0)) {
        fail("should not get here");
    } catch (NoSuchFileException e) {
    // success
    }
    assertThat(storage.containsKey(key)).isFalse();
}

Example 5 with Blob

use of build.buildfarm.cas.ContentAddressableStorage.Blob in project bazel-buildfarm by bazelbuild.

the class CASFileCacheTest method readThroughSwitchesToLocalOnComplete.

@Test
public void readThroughSwitchesToLocalOnComplete() throws IOException, InterruptedException {
    ByteString content = ByteString.copyFromUtf8("Hello, World");
    Blob blob = new Blob(content, DIGEST_UTIL);
    when(delegate.newInput(eq(blob.getDigest()), eq(0L))).thenReturn(content.newInput());
    InputStream in = fileCache.newInput(blob.getDigest(), 0);
    byte[] buf = new byte[content.size()];
    // advance to the middle of the content
    assertThat(in.read(buf, 0, 6)).isEqualTo(6);
    assertThat(ByteString.copyFrom(buf, 0, 6)).isEqualTo(content.substring(0, 6));
    verify(delegate, times(1)).newInput(blob.getDigest(), 0L);
    // trigger the read through to complete immediately by supplying the blob
    fileCache.put(blob);
    // read the remaining content
    int remaining = content.size() - 6;
    assertThat(in.read(buf, 6, remaining)).isEqualTo(remaining);
    assertThat(ByteString.copyFrom(buf)).isEqualTo(content);
}