Examples with DispatchedOperation build.buildfarm.v1test.DispatchedOperation used on opensource projects

Example 1 with DispatchedOperation

use of build.buildfarm.v1test.DispatchedOperation in project bazel-buildfarm by bazelbuild.

the class RedisShardBackplane method dispatchOperation.

@Nullable
private QueueEntry dispatchOperation(JedisCluster jedis, List<Platform.Property> provisions) throws InterruptedException {
    String queueEntryJson = operationQueue.dequeue(jedis, provisions);
    if (queueEntryJson == null) {
        return null;
    }
    QueueEntry.Builder queueEntryBuilder = QueueEntry.newBuilder();
    try {
        JsonFormat.parser().merge(queueEntryJson, queueEntryBuilder);
    } catch (InvalidProtocolBufferException e) {
        logger.log(Level.SEVERE, "error parsing queue entry", e);
        return null;
    }
    QueueEntry queueEntry = queueEntryBuilder.build();
    String operationName = queueEntry.getExecuteEntry().getOperationName();
    Operation operation = keepaliveOperation(operationName);
    publishReset(jedis, operation);
    long requeueAt = System.currentTimeMillis() + config.getDispatchingTimeoutMillis();
    DispatchedOperation o = DispatchedOperation.newBuilder().setQueueEntry(queueEntry).setRequeueAt(requeueAt).build();
    boolean success = false;
    try {
        String dispatchedOperationJson = JsonFormat.printer().print(o);
        /* if the operation is already in the dispatch list, fail the dispatch */
        long result = jedis.hsetnx(config.getDispatchedOperationsHashName(), operationName, dispatchedOperationJson);
        success = result == 1;
    } catch (InvalidProtocolBufferException e) {
        logger.log(Level.SEVERE, "error printing dispatched operation", e);
    // very unlikely, printer would have to fail
    }
    if (success) {
        if (!operationQueue.removeFromDequeue(jedis, queueEntryJson)) {
            logger.log(Level.WARNING, format("operation %s was missing in %s, may be orphaned", operationName, operationQueue.getDequeueName()));
        }
        dispatchedOperations.remove(jedis, operationName);
        // Return an entry so that if it needs re-queued, it will have the correct "requeue attempts".
        return queueEntryBuilder.setRequeueAttempts(queueEntry.getRequeueAttempts() + 1).build();
    }
    return null;
}

Example 2 with DispatchedOperation

use of build.buildfarm.v1test.DispatchedOperation in project bazel-buildfarm by bazelbuild.

the class RedisShardBackplane method getDispatchedOperations.

@SuppressWarnings("ConstantConditions")
@Override
public ImmutableList<DispatchedOperation> getDispatchedOperations() throws IOException {
    ImmutableList.Builder<DispatchedOperation> builder = new ImmutableList.Builder<>();
    Map<String, String> dispatchedOperations = client.call(jedis -> jedis.hgetAll(config.getDispatchedOperationsHashName()));
    ImmutableList.Builder<String> invalidOperationNames = new ImmutableList.Builder<>();
    boolean hasInvalid = false;
    // executor work queue?
    for (Map.Entry<String, String> entry : dispatchedOperations.entrySet()) {
        try {
            DispatchedOperation.Builder dispatchedOperationBuilder = DispatchedOperation.newBuilder();
            JsonFormat.parser().merge(entry.getValue(), dispatchedOperationBuilder);
            builder.add(dispatchedOperationBuilder.build());
        } catch (InvalidProtocolBufferException e) {
            logger.log(Level.SEVERE, "RedisShardBackplane::getDispatchedOperations: removing invalid operation " + entry.getKey(), e);
            /* guess we don't want to spin on this */
            invalidOperationNames.add(entry.getKey());
            hasInvalid = true;
        }
    }
    if (hasInvalid) {
        client.run(jedis -> {
            JedisClusterPipeline p = jedis.pipelined();
            for (String invalidOperationName : invalidOperationNames.build()) {
                p.hdel(config.getDispatchedOperationsHashName(), invalidOperationName);
            }
            p.sync();
        });
    }
    return builder.build();
}

Example 3 with DispatchedOperation

use of build.buildfarm.v1test.DispatchedOperation in project bazel-buildfarm by bazelbuild.

the class RedisShardBackplaneTest method dispatchedOperationsShowProperRequeueAmount0to1.

@Test
public void dispatchedOperationsShowProperRequeueAmount0to1() throws IOException, InterruptedException {
    // ARRANGE
    int STARTING_REQUEUE_AMOUNT = 0;
    int REQUEUE_AMOUNT_WHEN_DISPATCHED = 0;
    int REQUEUE_AMOUNT_WHEN_READY_TO_REQUEUE = 1;
    // create a backplane
    RedisShardBackplaneConfig config = RedisShardBackplaneConfig.newBuilder().setDispatchedOperationsHashName("DispatchedOperations").setOperationChannelPrefix("OperationChannel").setQueuedOperationsListName("{hash}QueuedOperations").build();
    JedisCluster jedisCluster = mock(JedisCluster.class);
    when(mockJedisClusterFactory.get()).thenReturn(jedisCluster);
    backplane = new RedisShardBackplane(config, "requeue-operation-test", (o) -> o, (o) -> o, mockJedisClusterFactory);
    backplane.start("startTime/test:0000");
    // ARRANGE
    // Assume the operation queue is already populated with a first-time operation.
    // this means the operation's requeue amount will be 0.
    // The jedis cluser is also mocked to assume success on other operations.
    QueueEntry queueEntry = QueueEntry.newBuilder().setExecuteEntry(ExecuteEntry.newBuilder().setOperationName("op").build()).setRequeueAttempts(STARTING_REQUEUE_AMOUNT).build();
    String queueEntryJson = JsonFormat.printer().print(queueEntry);
    when(jedisCluster.brpoplpush(any(String.class), any(String.class), any(Integer.class))).thenReturn(queueEntryJson);
    // PRE-ASSERT
    when(jedisCluster.hsetnx(any(String.class), any(String.class), any(String.class))).thenAnswer(args -> {
        // Extract the operation that was dispatched
        String dispatchedOperationJson = args.getArgument(2);
        DispatchedOperation.Builder dispatchedOperationBuilder = DispatchedOperation.newBuilder();
        JsonFormat.parser().merge(dispatchedOperationJson, dispatchedOperationBuilder);
        DispatchedOperation dispatchedOperation = dispatchedOperationBuilder.build();
        assertThat(dispatchedOperation.getQueueEntry().getRequeueAttempts()).isEqualTo(REQUEUE_AMOUNT_WHEN_DISPATCHED);
        return 1L;
    });
    // ACT
    // dispatch the operation and test properties of the QueueEntry and internal jedis calls.
    List<Platform.Property> properties = new ArrayList<>();
    QueueEntry readyForRequeue = backplane.dispatchOperation(properties);
    // ASSERT
    assertThat(readyForRequeue.getRequeueAttempts()).isEqualTo(REQUEUE_AMOUNT_WHEN_READY_TO_REQUEUE);
}

Example 4 with DispatchedOperation

use of build.buildfarm.v1test.DispatchedOperation in project bazel-buildfarm by bazelbuild.

the class RedisShardBackplaneTest method dispatchedOperationsShowProperRequeueAmount1to2.

@Test
public void dispatchedOperationsShowProperRequeueAmount1to2() throws IOException, InterruptedException {
    // ARRANGE
    int STARTING_REQUEUE_AMOUNT = 1;
    int REQUEUE_AMOUNT_WHEN_DISPATCHED = 1;
    int REQUEUE_AMOUNT_WHEN_READY_TO_REQUEUE = 2;
    // create a backplane
    RedisShardBackplaneConfig config = RedisShardBackplaneConfig.newBuilder().setDispatchedOperationsHashName("DispatchedOperations").setOperationChannelPrefix("OperationChannel").setQueuedOperationsListName("{hash}QueuedOperations").build();
    JedisCluster jedisCluster = mock(JedisCluster.class);
    when(mockJedisClusterFactory.get()).thenReturn(jedisCluster);
    backplane = new RedisShardBackplane(config, "requeue-operation-test", (o) -> o, (o) -> o, mockJedisClusterFactory);
    backplane.start("startTime/test:0000");
    // Assume the operation queue is already populated from a first re-queue.
    // this means the operation's requeue amount will be 1.
    // The jedis cluser is also mocked to assume success on other operations.
    QueueEntry queueEntry = QueueEntry.newBuilder().setExecuteEntry(ExecuteEntry.newBuilder().setOperationName("op").build()).setRequeueAttempts(STARTING_REQUEUE_AMOUNT).build();
    String queueEntryJson = JsonFormat.printer().print(queueEntry);
    when(jedisCluster.brpoplpush(any(String.class), any(String.class), any(Integer.class))).thenReturn(queueEntryJson);
    // PRE-ASSERT
    when(jedisCluster.hsetnx(any(String.class), any(String.class), any(String.class))).thenAnswer(args -> {
        // Extract the operation that was dispatched
        String dispatchedOperationJson = args.getArgument(2);
        DispatchedOperation.Builder dispatchedOperationBuilder = DispatchedOperation.newBuilder();
        JsonFormat.parser().merge(dispatchedOperationJson, dispatchedOperationBuilder);
        DispatchedOperation dispatchedOperation = dispatchedOperationBuilder.build();
        assertThat(dispatchedOperation.getQueueEntry().getRequeueAttempts()).isEqualTo(REQUEUE_AMOUNT_WHEN_DISPATCHED);
        return 1L;
    });
    // ACT
    // dispatch the operation and test properties of the QueueEntry and internal jedis calls.
    List<Platform.Property> properties = new ArrayList<>();
    QueueEntry readyForRequeue = backplane.dispatchOperation(properties);
    // ASSERT
    assertThat(readyForRequeue.getRequeueAttempts()).isEqualTo(REQUEUE_AMOUNT_WHEN_READY_TO_REQUEUE);
}