Examples with Digest com.google.cloud.kms.v1.Digest used on opensource projects

Example 21 with Digest

use of com.google.cloud.kms.v1.Digest in project bazel-buildfarm by bazelbuild.

the class AbstractServerInstance method fetchBlob.

@Override
public ListenableFuture<Digest> fetchBlob(Iterable<String> uris, Digest expectedDigest, RequestMetadata requestMetadata) {
    for (String uri : uris) {
        try {
            // some minor abuse here, we want the download to set our built digest size as side effect
            Digest.Builder actualDigestBuilder = expectedDigest.toBuilder();
            downloadUri(uri, contentLength -> {
                Digest actualDigest = actualDigestBuilder.setSizeBytes(contentLength).build();
                if (expectedDigest.getSizeBytes() >= 0 && expectedDigest.getSizeBytes() != contentLength) {
                    throw new DigestMismatchException(actualDigest, expectedDigest);
                }
                return getBlobWrite(actualDigest, UUID.randomUUID(), requestMetadata).getOutput(1, DAYS, () -> {
                });
            });
            return immediateFuture(actualDigestBuilder.build());
        } catch (Exception e) {
            logger.log(Level.WARNING, "download attempt failed", e);
        // ignore?
        }
    }
    return immediateFailedFuture(new NoSuchFileException(expectedDigest.getHash()));
}

Example 22 with Digest

use of com.google.cloud.kms.v1.Digest in project bazel-buildfarm by bazelbuild.

the class JedisCasWorkerMap method addAll.

/**
 * @brief Update multiple blob entries for a worker.
 * @details This may add a new key if the blob did not previously exist, or it will adjust the
 *     worker values based on the worker name. The expiration time is always refreshed.
 * @param client Client used for interacting with redis when not using cacheMap.
 * @param blobDigests The blob digests to adjust worker information from.
 * @param workerName The worker to add for looking up the blobs.
 */
@Override
public void addAll(RedisClient client, Iterable<Digest> blobDigests, String workerName) throws IOException {
    client.run(jedis -> {
        JedisClusterPipeline p = jedis.pipelined();
        for (Digest blobDigest : blobDigests) {
            String key = redisCasKey(blobDigest);
            p.sadd(key, workerName);
            p.expire(key, keyExpiration_s);
        }
        p.sync();
    });
}

Example 23 with Digest

use of com.google.cloud.kms.v1.Digest in project bazel-buildfarm by bazelbuild.

the class JedisCasWorkerMap method removeAll.

/**
 * @brief Remove worker value from all blob keys.
 * @details If the blob is already missing, or the worker doesn't exist, this will be no effect on
 *     the key.
 * @param client Client used for interacting with redis when not using cacheMap.
 * @param blobDigests The blob digests to remove the worker from.
 * @param workerName The worker name to remove.
 */
@Override
public void removeAll(RedisClient client, Iterable<Digest> blobDigests, String workerName) throws IOException {
    client.run(jedis -> {
        JedisClusterPipeline p = jedis.pipelined();
        for (Digest blobDigest : blobDigests) {
            String key = redisCasKey(blobDigest);
            p.srem(key, workerName);
        }
        p.sync();
    });
}

Example 24 with Digest

use of com.google.cloud.kms.v1.Digest in project bazel-buildfarm by bazelbuild.

the class Executor method loadFilesIntoCAS.

private static void loadFilesIntoCAS(String instanceName, Channel channel, Path blobsDir) throws Exception {
    ContentAddressableStorageBlockingStub casStub = ContentAddressableStorageGrpc.newBlockingStub(channel);
    List<Digest> missingDigests = findMissingBlobs(instanceName, blobsDir, casStub);
    UUID uploadId = UUID.randomUUID();
    int[] bucketSizes = new int[128];
    BatchUpdateBlobsRequest.Builder[] buckets = new BatchUpdateBlobsRequest.Builder[128];
    for (int i = 0; i < 128; i++) {
        bucketSizes[i] = 0;
        buckets[i] = BatchUpdateBlobsRequest.newBuilder().setInstanceName(instanceName);
    }
    ByteStreamStub bsStub = ByteStreamGrpc.newStub(channel);
    for (Digest missingDigest : missingDigests) {
        Path path = blobsDir.resolve(missingDigest.getHash() + "_" + missingDigest.getSizeBytes());
        if (missingDigest.getSizeBytes() < Size.mbToBytes(1)) {
            Request request = Request.newBuilder().setDigest(missingDigest).setData(ByteString.copyFrom(Files.readAllBytes(path))).build();
            int maxBucketSize = 0;
            long minBucketSize = Size.mbToBytes(2) + 1;
            int maxBucketIndex = 0;
            int minBucketIndex = -1;
            int size = (int) missingDigest.getSizeBytes() + 48;
            for (int i = 0; i < 128; i++) {
                int newBucketSize = bucketSizes[i] + size;
                if (newBucketSize < Size.mbToBytes(2) && bucketSizes[i] < minBucketSize) {
                    minBucketSize = bucketSizes[i];
                    minBucketIndex = i;
                }
                if (bucketSizes[i] > maxBucketSize) {
                    maxBucketSize = bucketSizes[i];
                    maxBucketIndex = i;
                }
            }
            if (minBucketIndex < 0) {
                bucketSizes[maxBucketIndex] = size;
                BatchUpdateBlobsRequest batchRequest = buckets[maxBucketIndex].build();
                Stopwatch stopwatch = Stopwatch.createStarted();
                BatchUpdateBlobsResponse batchResponse = casStub.batchUpdateBlobs(batchRequest);
                long usecs = stopwatch.elapsed(MICROSECONDS);
                checkState(batchResponse.getResponsesList().stream().allMatch(response -> Code.forNumber(response.getStatus().getCode()) == Code.OK));
                System.out.println("Updated " + batchRequest.getRequestsCount() + " blobs in " + (usecs / 1000.0) + "ms");
                buckets[maxBucketIndex] = BatchUpdateBlobsRequest.newBuilder().setInstanceName(instanceName).addRequests(request);
            } else {
                bucketSizes[minBucketIndex] += size;
                buckets[minBucketIndex].addRequests(request);
            }
        } else {
            Stopwatch stopwatch = Stopwatch.createStarted();
            SettableFuture<WriteResponse> writtenFuture = SettableFuture.create();
            StreamObserver<WriteRequest> requestObserver = bsStub.write(new StreamObserver<WriteResponse>() {

                @Override
                public void onNext(WriteResponse response) {
                    writtenFuture.set(response);
                }

                @Override
                public void onCompleted() {
                }

                @Override
                public void onError(Throwable t) {
                    writtenFuture.setException(t);
                }
            });
            HashCode hash = HashCode.fromString(missingDigest.getHash());
            String resourceName = uploadResourceName(instanceName, uploadId, hash, missingDigest.getSizeBytes());
            try (InputStream in = Files.newInputStream(path)) {
                boolean first = true;
                long writtenBytes = 0;
                byte[] buf = new byte[64 * 1024];
                while (writtenBytes != missingDigest.getSizeBytes()) {
                    int len = in.read(buf);
                    WriteRequest.Builder request = WriteRequest.newBuilder();
                    if (first) {
                        request.setResourceName(resourceName);
                    }
                    request.setData(ByteString.copyFrom(buf, 0, len)).setWriteOffset(writtenBytes);
                    if (writtenBytes + len == missingDigest.getSizeBytes()) {
                        request.setFinishWrite(true);
                    }
                    requestObserver.onNext(request.build());
                    writtenBytes += len;
                    first = false;
                }
                writtenFuture.get();
                System.out.println("Wrote long " + DigestUtil.toString(missingDigest) + " in " + (stopwatch.elapsed(MICROSECONDS) / 1000.0) + "ms");
            }
        }
    }
    for (int i = 0; i < 128; i++) {
        if (bucketSizes[i] > 0) {
            BatchUpdateBlobsRequest batchRequest = buckets[i].build();
            Stopwatch stopwatch = Stopwatch.createStarted();
            BatchUpdateBlobsResponse batchResponse = casStub.batchUpdateBlobs(batchRequest);
            long usecs = stopwatch.elapsed(MICROSECONDS);
            checkState(batchResponse.getResponsesList().stream().allMatch(response -> Code.forNumber(response.getStatus().getCode()) == Code.OK));
            System.out.println("Updated " + batchRequest.getRequestsCount() + " blobs in " + (usecs / 1000.0) + "ms");
        }
    }
}

Example 25 with Digest

use of com.google.cloud.kms.v1.Digest in project bazel-buildfarm by bazelbuild.

the class Executor method findMissingBlobs.

private static List<Digest> findMissingBlobs(String instanceName, Path blobsDir, ContentAddressableStorageBlockingStub casStub) throws IOException {
    FindMissingBlobsRequest.Builder request = FindMissingBlobsRequest.newBuilder().setInstanceName(instanceName);
    int size = 0;
    ImmutableList.Builder<Digest> missingDigests = ImmutableList.builder();
    System.out.println("Looking for missing blobs");
    final int messagesPerRequest = 2 * 1024 * 1024 / 80;
    System.out.println("Looking for missing blobs");
    Stopwatch stopwatch = Stopwatch.createUnstarted();
    FileStore fileStore = Files.getFileStore(blobsDir);
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(blobsDir)) {
        for (Path file : stream) {
            FileStatus stat = stat(file, /* followSymlinks=*/
            false, fileStore);
            Digest digest = DigestUtil.buildDigest(file.getFileName().toString().split("_")[0], stat.getSize());
            request.addBlobDigests(digest);
            size++;
            if (size == messagesPerRequest) {
                stopwatch.reset().start();
                FindMissingBlobsResponse response = casStub.findMissingBlobs(request.build());
                System.out.println("Found " + response.getMissingBlobDigestsCount() + " missing digests in " + (stopwatch.elapsed(MICROSECONDS) / 1000.0) + "ms");
                missingDigests.addAll(response.getMissingBlobDigestsList());
                request = FindMissingBlobsRequest.newBuilder().setInstanceName(instanceName);
                size = 0;
            }
        }
    }
    if (size > 0) {
        FindMissingBlobsResponse response = casStub.findMissingBlobs(request.build());
        System.out.println("Found " + response.getMissingBlobDigestsCount() + " missing digests");
        missingDigests.addAll(response.getMissingBlobDigestsList());
    }
    return missingDigests.build();
}