Examples with WorkerQueue build.buildfarm.instance.queues.WorkerQueue used on opensource projects

Example 1 with WorkerQueue

use of build.buildfarm.instance.queues.WorkerQueue in project bazel-buildfarm by bazelbuild.

the class MemoryInstance method backplaneStatus.

@Override
public BackplaneStatus backplaneStatus() {
    BackplaneStatus.Builder status = BackplaneStatus.newBuilder();
    OperationQueueStatus.Builder queueStatus = status.getOperationQueueBuilder();
    long totalSize = 0;
    for (WorkerQueue queue : queuedOperations) {
        long size = queue.operations.size();
        queueStatus.addProvisionsBuilder().setName(queue.name).setSize(size);
        totalSize += size;
    }
    queueStatus.setSize(totalSize);
    // TODO active workers - available, but not with any discerning identifier, should rectify this
    return status.build();
}

Example 2 with WorkerQueue

use of build.buildfarm.instance.queues.WorkerQueue in project bazel-buildfarm by bazelbuild.

the class MemoryInstance method matchSynchronized.

@SuppressWarnings("ConstantConditions")
private void matchSynchronized(Platform platform, MatchListener listener) throws InterruptedException {
    ImmutableList.Builder<Operation> rejectedOperations = ImmutableList.builder();
    boolean matched = false;
    SetMultimap<String, String> provisions = createProvisions(platform);
    WorkerQueue queue = queuedOperations.MatchEligibleQueue(provisions);
    while (!matched && !queue.operations.isEmpty()) {
        Operation operation = queue.operations.remove(0);
        ExecuteOperationMetadata metadata = expectExecuteOperationMetadata(operation);
        Preconditions.checkState(metadata != null, "metadata not found");
        Action action = getUnchecked(expect(metadata.getActionDigest(), Action.parser(), newDirectExecutorService(), RequestMetadata.getDefaultInstance()));
        Preconditions.checkState(action != null, "action not found");
        Command command = getUnchecked(expect(action.getCommandDigest(), Command.parser(), newDirectExecutorService(), RequestMetadata.getDefaultInstance()));
        Preconditions.checkState(command != null, "command not found");
        String operationName = operation.getName();
        DequeueMatchSettings settings = new DequeueMatchSettings();
        if (command == null) {
            cancelOperation(operationName);
        } else if (DequeueMatchEvaluator.shouldKeepOperation(settings, provisions, command)) {
            QueuedOperation queuedOperation = QueuedOperation.newBuilder().setAction(action).setCommand(command).setTree(getCompleteTree(action.getInputRootDigest())).build();
            ByteString queuedOperationBlob = queuedOperation.toByteString();
            Digest queuedOperationDigest = getDigestUtil().compute(queuedOperationBlob);
            // maybe do this elsewhere
            try {
                putBlob(this, queuedOperationDigest, queuedOperationBlob, 60, SECONDS, RequestMetadata.getDefaultInstance());
                QueueEntry queueEntry = QueueEntry.newBuilder().setExecuteEntry(ExecuteEntry.newBuilder().setOperationName(operationName).setActionDigest(metadata.getActionDigest()).setStdoutStreamName(metadata.getStdoutStreamName()).setStderrStreamName(metadata.getStderrStreamName()).setQueuedTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))).setQueuedOperationDigest(queuedOperationDigest).setPlatform(command.getPlatform()).build();
                matched = true;
                if (listener.onEntry(queueEntry)) {
                    onDispatched(operation);
                } else {
                    enqueueOperation(operation);
                }
            } catch (StatusException | IOException e) {
                logger.log(Level.SEVERE, format("could not emplace queued operation: %s", operationName), e);
            }
        } else {
            rejectedOperations.add(operation);
        }
    }
    for (Operation operation : rejectedOperations.build()) {
        requeueOperation(operation);
    }
    if (!matched) {
        synchronized (queue.workers) {
            listener.setOnCancelHandler(() -> queuedOperations.removeWorker(listener));
            listener.onWaitStart();
            queuedOperations.AddWorker(provisions, listener);
        }
    }
}

Example 3 with WorkerQueue

use of build.buildfarm.instance.queues.WorkerQueue in project bazel-buildfarm by bazelbuild.

the class MemoryInstance method matchOperation.

@Override
protected boolean matchOperation(Operation operation) throws InterruptedException {
    ExecuteOperationMetadata metadata = expectExecuteOperationMetadata(operation);
    Preconditions.checkState(metadata != null, "metadata not found");
    Action action = getUnchecked(expect(metadata.getActionDigest(), Action.parser(), newDirectExecutorService(), RequestMetadata.getDefaultInstance()));
    Preconditions.checkState(action != null, "action not found");
    Command command = getUnchecked(expect(action.getCommandDigest(), Command.parser(), newDirectExecutorService(), RequestMetadata.getDefaultInstance()));
    Preconditions.checkState(command != null, "command not found");
    Tree tree = getCompleteTree(action.getInputRootDigest());
    QueuedOperation queuedOperation = QueuedOperation.newBuilder().setAction(action).setCommand(command).setTree(tree).build();
    ByteString queuedOperationBlob = queuedOperation.toByteString();
    Digest queuedOperationDigest = getDigestUtil().compute(queuedOperationBlob);
    String operationName = operation.getName();
    try {
        putBlob(this, queuedOperationDigest, queuedOperationBlob, 60, SECONDS, RequestMetadata.getDefaultInstance());
    } catch (StatusException | IOException e) {
        logger.log(Level.SEVERE, format("could not emplace queued operation: %s", operationName), e);
        return false;
    }
    ImmutableList.Builder<Worker> rejectedWorkers = new ImmutableList.Builder<>();
    boolean dispatched = false;
    WorkerQueue queue = queuedOperations.MatchEligibleQueue(createProvisions(command.getPlatform()));
    DequeueMatchSettings settings = new DequeueMatchSettings();
    synchronized (queue.workers) {
        while (!dispatched && !queue.workers.isEmpty()) {
            Worker worker = queue.workers.remove(0);
            if (!DequeueMatchEvaluator.shouldKeepOperation(settings, worker.getProvisions(), command)) {
                rejectedWorkers.add(worker);
            } else {
                QueueEntry queueEntry = QueueEntry.newBuilder().setExecuteEntry(ExecuteEntry.newBuilder().setOperationName(operationName).setActionDigest(metadata.getActionDigest()).setStdoutStreamName(metadata.getStdoutStreamName()).setStderrStreamName(metadata.getStderrStreamName()).setQueuedTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))).setQueuedOperationDigest(queuedOperationDigest).setPlatform(command.getPlatform()).build();
                dispatched = worker.getListener().onEntry(queueEntry);
                if (dispatched) {
                    onDispatched(operation);
                }
            }
        }
        Iterables.addAll(queue.workers, rejectedWorkers.build());
    }
    return dispatched;
}