Examples with Write build.buildfarm.common.Write used on opensource projects

Example 1 with Write

use of build.buildfarm.common.Write in project bazel-buildfarm by bazelbuild.

the class CASFileCache method expireEntryFallback.

private void expireEntryFallback(Entry e) throws IOException, InterruptedException {
    if (delegate != null) {
        FileEntryKey fileEntryKey = parseFileEntryKey(e.key, e.size);
        if (fileEntryKey == null) {
            logger.log(Level.SEVERE, format("error parsing expired key %s", e.key));
        } else {
            Write write = delegate.getWrite(fileEntryKey.getDigest(), UUID.randomUUID(), RequestMetadata.getDefaultInstance());
            performCopy(write, e);
        }
    }
}

Example 2 with Write

use of build.buildfarm.common.Write in project bazel-buildfarm by bazelbuild.

the class MemoryCAS method expireEntry.

@GuardedBy("this")
private void expireEntry(Entry e) {
    Digest digest = DigestUtil.buildDigest(e.key, e.value.size());
    logger.log(Level.INFO, "MemoryLRUCAS: expiring " + DigestUtil.toString(digest));
    if (delegate != null) {
        try {
            Write write = delegate.getWrite(digest, UUID.randomUUID(), RequestMetadata.getDefaultInstance());
            try (OutputStream out = write.getOutput(1, MINUTES, () -> {
            })) {
                e.value.getData().writeTo(out);
            }
        } catch (IOException ioEx) {
            logger.log(Level.SEVERE, String.format("error delegating %s", DigestUtil.toString(digest)), ioEx);
        }
    }
    storage.remove(e.key);
    e.expire();
    sizeInBytes -= digest.getSizeBytes();
}

Example 3 with Write

use of build.buildfarm.common.Write in project bazel-buildfarm by bazelbuild.

the class CASFileCacheTest method readThroughSwitchedToLocalContinues.

@Test
public void readThroughSwitchedToLocalContinues() throws Exception {
    ByteString content = ByteString.copyFromUtf8("Hello, World");
    Blob blob = new Blob(content, DIGEST_UTIL);
    ExecutorService service = newSingleThreadExecutor();
    SettableFuture<Void> writeComplete = SettableFuture.create();
    // we need to register callbacks on the shared write future
    Write write = new NullWrite() {

        @Override
        public ListenableFuture<Long> getFuture() {
            return Futures.transform(writeComplete, result -> blob.getDigest().getSizeBytes(), directExecutor());
        }

        @Override
        public FeedbackOutputStream getOutput(long deadlineAfter, TimeUnit deadlineAfterUnits, Runnable onReadyHandler) {
            return new FeedbackOutputStream() {

                int offset = 0;

                @Override
                public void write(int b) {
                    throw new UnsupportedOperationException();
                }

                @Override
                public void write(byte[] buf, int ofs, int len) throws IOException {
                    // hangs on second read
                    if (offset == 6) {
                        service.submit(() -> writeComplete.set(null));
                        throw new ClosedChannelException();
                    }
                    offset += len;
                }

                @Override
                public boolean isReady() {
                    return true;
                }
            };
        }
    };
    when(delegate.getWrite(eq(blob.getDigest()), any(UUID.class), any(RequestMetadata.class))).thenReturn(write);
    when(delegate.newInput(eq(blob.getDigest()), eq(0L))).thenReturn(content.newInput());
    // the switch will reset to this point
    InputStream switchedIn = content.newInput();
    switchedIn.skip(6);
    when(delegate.newInput(eq(blob.getDigest()), eq(6L))).thenReturn(switchedIn);
    InputStream in = fileCache.newReadThroughInput(blob.getDigest(), 0, write);
    byte[] buf = new byte[content.size()];
    // advance to the middle of the content
    assertThat(in.read(buf, 0, 6)).isEqualTo(6);
    assertThat(ByteString.copyFrom(buf, 0, 6)).isEqualTo(content.substring(0, 6));
    verify(delegate, times(1)).newInput(blob.getDigest(), 0L);
    // read the remaining content
    int remaining = content.size() - 6;
    assertThat(in.read(buf, 6, remaining)).isEqualTo(remaining);
    assertThat(ByteString.copyFrom(buf)).isEqualTo(content);
    if (!shutdownAndAwaitTermination(service, 1, SECONDS)) {
        throw new RuntimeException("could not shut down service");
    }
}

Example 4 with Write

use of build.buildfarm.common.Write in project bazel-buildfarm by bazelbuild.

the class CASFileCacheTest method incompleteWriteFileIsResumed.

@Test
public void incompleteWriteFileIsResumed() throws IOException {
    ByteString content = ByteString.copyFromUtf8("Hello, World");
    Digest digest = DIGEST_UTIL.compute(content);
    UUID writeId = UUID.randomUUID();
    String key = fileCache.getKey(digest, false);
    Path writePath = fileCache.getPath(key).resolveSibling(key + "." + writeId);
    try (OutputStream out = Files.newOutputStream(writePath)) {
        content.substring(0, 6).writeTo(out);
    }
    Write write = fileCache.getWrite(digest, writeId, RequestMetadata.getDefaultInstance());
    AtomicBoolean notified = new AtomicBoolean(false);
    write.getFuture().addListener(() -> notified.set(true), directExecutor());
    assertThat(write.getCommittedSize()).isEqualTo(6);
    try (OutputStream out = write.getOutput(1, SECONDS, () -> {
    })) {
        content.substring(6).writeTo(out);
    }
    assertThat(notified.get()).isTrue();
    assertThat(write.getCommittedSize()).isEqualTo(digest.getSizeBytes());
    assertThat(write.isComplete()).isTrue();
}

Example 5 with Write

use of build.buildfarm.common.Write in project bazel-buildfarm by bazelbuild.

the class GrpcCASTest method writeIsResumable.

@Test
public void writeIsResumable() throws Exception {
    UUID uuid = UUID.randomUUID();
    ByteString writeContent = ByteString.copyFromUtf8("written");
    Digest digest = DIGEST_UTIL.compute(writeContent);
    String instanceName = "test";
    HashCode hash = HashCode.fromString(digest.getHash());
    String resourceName = ByteStreamUploader.uploadResourceName(instanceName, uuid, hash, digest.getSizeBytes());
    // better test might just put a full gRPC CAS behind an in-process and validate state
    SettableFuture<ByteString> content = SettableFuture.create();
    serviceRegistry.addService(new ByteStreamServiceWriter(resourceName, content, (int) digest.getSizeBytes()));
    Channel channel = InProcessChannelBuilder.forName(fakeServerName).directExecutor().build();
    GrpcCAS cas = new GrpcCAS(instanceName, channel, /* uploader=*/
    null, onExpirations);
    RequestMetadata requestMetadata = RequestMetadata.getDefaultInstance();
    Write initialWrite = cas.getWrite(digest, uuid, requestMetadata);
    try (OutputStream writeOut = initialWrite.getOutput(1, SECONDS, () -> {
    })) {
        writeContent.substring(0, 4).writeTo(writeOut);
    }
    Write finalWrite = cas.getWrite(digest, uuid, requestMetadata);
    try (OutputStream writeOut = finalWrite.getOutput(1, SECONDS, () -> {
    })) {
        writeContent.substring(4).writeTo(writeOut);
    }
    assertThat(content.get(1, TimeUnit.SECONDS)).isEqualTo(writeContent);
}