Search in sources :

Example 1 with FuseCAS

use of build.buildfarm.worker.FuseCAS in project bazel-buildfarm by bazelbuild.

the class Mount method main.

@SuppressWarnings("BusyWait")
public static void main(String[] args) throws Exception {
    String host = args[0];
    String instanceName = args[1];
    DigestUtil digestUtil = DigestUtil.forHash(args[2]);
    ManagedChannel channel = createChannel(host);
    Instance instance = new StubInstance(instanceName, digestUtil, channel);
    Path cwd = Paths.get(".");
    FuseCAS fuse = new FuseCAS(cwd.resolve(args[3]), new InputStreamFactory() {

        final Map<Digest, ByteString> cache = new HashMap<>();

        public synchronized InputStream newInput(Digest blobDigest, long offset) {
            if (cache.containsKey(blobDigest)) {
                return cache.get(blobDigest).substring((int) offset).newInput();
            }
            try {
                ByteString value = getBlob(instance, blobDigest, RequestMetadata.getDefaultInstance());
                if (offset == 0) {
                    cache.put(blobDigest, value);
                }
                return value.newInput();
            } catch (IOException e) {
                return null;
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                return null;
            }
        }
    });
    // FIXME make bettar
    fuse.createInputRoot(args[5], DigestUtil.parseDigest(args[4]));
    try {
        // noinspection InfiniteLoopStatement
        for (; ; ) {
            Thread.sleep(1000);
        }
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    } finally {
        fuse.stop();
    }
}
Also used : Path(java.nio.file.Path) StubInstance(build.buildfarm.instance.stub.StubInstance) Instance(build.buildfarm.instance.Instance) Digest(build.bazel.remote.execution.v2.Digest) HashMap(java.util.HashMap) ByteString(com.google.protobuf.ByteString) InputStream(java.io.InputStream) FuseCAS(build.buildfarm.worker.FuseCAS) ByteString(com.google.protobuf.ByteString) IOException(java.io.IOException) DigestUtil(build.buildfarm.common.DigestUtil) InputStreamFactory(build.buildfarm.common.InputStreamFactory) StubInstance(build.buildfarm.instance.stub.StubInstance) ManagedChannel(io.grpc.ManagedChannel)

Example 2 with FuseCAS

use of build.buildfarm.worker.FuseCAS in project bazel-buildfarm by bazelbuild.

the class Worker method createFuseExecFileSystem.

private ExecFileSystem createFuseExecFileSystem(InputStreamFactory remoteInputStreamFactory, ContentAddressableStorage storage) {
    InputStreamFactory storageInputStreamFactory = (digest, offset) -> storage.get(digest).getData().substring((int) offset).newInput();
    InputStreamFactory localPopulatingInputStreamFactory = (blobDigest, offset) -> {
        // FIXME use write
        ByteString content = ByteString.readFrom(remoteInputStreamFactory.newInput(blobDigest, offset));
        if (offset == 0) {
            // extra computations
            Blob blob = new Blob(content, digestUtil);
            // here's hoping that our digest matches...
            storage.put(blob);
        }
        return content.newInput();
    };
    return new FuseExecFileSystem(root, new FuseCAS(root, new EmptyInputStreamFactory(new FailoverInputStreamFactory(storageInputStreamFactory, localPopulatingInputStreamFactory))), storage);
}
Also used : Arrays(java.util.Arrays) LoadingCache(com.google.common.cache.LoadingCache) Blob(build.buildfarm.cas.ContentAddressableStorage.Blob) ShardWorker(build.buildfarm.v1test.ShardWorker) MemoryCAS(build.buildfarm.cas.MemoryCAS) PutOperationStage(build.buildfarm.worker.PutOperationStage) Status(io.grpc.Status) IOSupplier(build.buildfarm.common.function.IOSupplier) Path(java.nio.file.Path) ReportResultStage(build.buildfarm.worker.ReportResultStage) Set(java.util.Set) MoreExecutors.directExecutor(com.google.common.util.concurrent.MoreExecutors.directExecutor) ByteStreams(com.google.common.io.ByteStreams) DigestUtil(build.buildfarm.common.DigestUtil) Strings(com.google.common.base.Strings) Code(io.grpc.Status.Code) UserPrincipal(java.nio.file.attribute.UserPrincipal) Lists(com.google.common.collect.Lists) ServerBuilder(io.grpc.ServerBuilder) Server(io.grpc.Server) Nullable(javax.annotation.Nullable) Backplane(build.buildfarm.backplane.Backplane) ContentAddressableStorageConfig(build.buildfarm.v1test.ContentAddressableStorageConfig) ExecuteActionStage(build.buildfarm.worker.ExecuteActionStage) ServingStatus(io.grpc.health.v1.HealthCheckResponse.ServingStatus) CASFileCache(build.buildfarm.cas.cfc.CASFileCache) Files(java.nio.file.Files) Executor(java.util.concurrent.Executor) PipelineStage(build.buildfarm.worker.PipelineStage) ShardWorkerOptions(build.buildfarm.common.config.ShardWorkerOptions) Throwables(com.google.common.base.Throwables) IOException(java.io.IOException) ContentAddressableStorages.createGrpcCAS(build.buildfarm.cas.ContentAddressableStorages.createGrpcCAS) AdminGrpc(build.buildfarm.v1test.AdminGrpc) InputStreamReader(java.io.InputStreamReader) File(java.io.File) ExecutionException(java.util.concurrent.ExecutionException) InputFetchStage(build.buildfarm.worker.InputFetchStage) Paths(java.nio.file.Paths) ManagedChannel(io.grpc.ManagedChannel) OptionsParser(com.google.devtools.common.options.OptionsParser) NegotiationType(io.grpc.netty.NegotiationType) RequestMetadata(build.bazel.remote.execution.v2.RequestMetadata) Random(java.util.Random) ContentAddressableStorage(build.buildfarm.cas.ContentAddressableStorage) WorkerStubs(build.buildfarm.instance.shard.WorkerStubs) InputStreamFactory(build.buildfarm.common.InputStreamFactory) SettableFuture(com.google.common.util.concurrent.SettableFuture) RemoteInputStreamFactory(build.buildfarm.instance.shard.RemoteInputStreamFactory) ConfigAdjuster(build.buildfarm.common.config.ConfigAdjuster) Digest(build.bazel.remote.execution.v2.Digest) Gauge(io.prometheus.client.Gauge) Utils.getUser(build.buildfarm.common.io.Utils.getUser) PrometheusPublisher(build.buildfarm.metrics.prometheus.PrometheusPublisher) TextFormat(com.google.protobuf.TextFormat) ByteStreamService(build.buildfarm.server.ByteStreamService) MatchStage(build.buildfarm.worker.MatchStage) Executors.newSingleThreadExecutor(java.util.concurrent.Executors.newSingleThreadExecutor) UUID(java.util.UUID) FileSystem(java.nio.file.FileSystem) Logger(java.util.logging.Logger) String.format(java.lang.String.format) Instance(build.buildfarm.instance.Instance) Preconditions.checkState(com.google.common.base.Preconditions.checkState) ByteString(com.google.protobuf.ByteString) List(java.util.List) HashFunction(build.buildfarm.common.DigestUtil.HashFunction) Size(build.buildfarm.common.Size) ReindexCasRequest(build.buildfarm.v1test.ReindexCasRequest) LoggingMain(build.buildfarm.common.LoggingMain) ContentAddressableStorageService(build.buildfarm.server.ContentAddressableStorageService) ThreadFactoryBuilder(com.google.common.util.concurrent.ThreadFactoryBuilder) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) DisableScaleInProtectionRequest(build.buildfarm.v1test.DisableScaleInProtectionRequest) INFO(java.util.logging.Level.INFO) ConfigurationException(javax.naming.ConfigurationException) Operation(com.google.longrunning.Operation) Counter(io.prometheus.client.Counter) SEVERE(java.util.logging.Level.SEVERE) Level(java.util.logging.Level) ImmutableList(com.google.common.collect.ImmutableList) DAYS(java.util.concurrent.TimeUnit.DAYS) FeedbackOutputStream(build.buildfarm.common.io.FeedbackOutputStream) FuseCAS(build.buildfarm.worker.FuseCAS) HealthStatusManager(io.grpc.services.HealthStatusManager) ExecutorService(java.util.concurrent.ExecutorService) OutputStream(java.io.OutputStream) DequeueMatchSettings(build.buildfarm.worker.DequeueMatchSettings) FilesystemCASConfig(build.buildfarm.v1test.FilesystemCASConfig) Iterator(java.util.Iterator) RedisShardBackplane(build.buildfarm.instance.shard.RedisShardBackplane) Executors.newFixedThreadPool(java.util.concurrent.Executors.newFixedThreadPool) Write(build.buildfarm.common.Write) NettyChannelBuilder(io.grpc.netty.NettyChannelBuilder) Pipeline(build.buildfarm.worker.Pipeline) ShardWorkerConfig(build.buildfarm.v1test.ShardWorkerConfig) Collections(java.util.Collections) SECONDS(java.util.concurrent.TimeUnit.SECONDS) InputStream(java.io.InputStream) Blob(build.buildfarm.cas.ContentAddressableStorage.Blob) ByteString(com.google.protobuf.ByteString) FuseCAS(build.buildfarm.worker.FuseCAS) InputStreamFactory(build.buildfarm.common.InputStreamFactory) RemoteInputStreamFactory(build.buildfarm.instance.shard.RemoteInputStreamFactory)

Aggregations

Digest (build.bazel.remote.execution.v2.Digest)2 DigestUtil (build.buildfarm.common.DigestUtil)2 InputStreamFactory (build.buildfarm.common.InputStreamFactory)2 Instance (build.buildfarm.instance.Instance)2 FuseCAS (build.buildfarm.worker.FuseCAS)2 RequestMetadata (build.bazel.remote.execution.v2.RequestMetadata)1 Backplane (build.buildfarm.backplane.Backplane)1 ContentAddressableStorage (build.buildfarm.cas.ContentAddressableStorage)1 Blob (build.buildfarm.cas.ContentAddressableStorage.Blob)1 ContentAddressableStorages.createGrpcCAS (build.buildfarm.cas.ContentAddressableStorages.createGrpcCAS)1 MemoryCAS (build.buildfarm.cas.MemoryCAS)1 CASFileCache (build.buildfarm.cas.cfc.CASFileCache)1 HashFunction (build.buildfarm.common.DigestUtil.HashFunction)1 LoggingMain (build.buildfarm.common.LoggingMain)1 Size (build.buildfarm.common.Size)1 Write (build.buildfarm.common.Write)1 ConfigAdjuster (build.buildfarm.common.config.ConfigAdjuster)1 ShardWorkerOptions (build.buildfarm.common.config.ShardWorkerOptions)1 IOSupplier (build.buildfarm.common.function.IOSupplier)1 FeedbackOutputStream (build.buildfarm.common.io.FeedbackOutputStream)1