Examples with KafkaFuture org.apache.kafka.common.KafkaFuture used on opensource projects

Example 76 with KafkaFuture

use of org.apache.kafka.common.KafkaFuture in project kafka by apache.

the class MockAdminClient method handleDescribeTopicsByNames.

private Map<String, KafkaFuture<TopicDescription>> handleDescribeTopicsByNames(Collection<String> topicNames, DescribeTopicsOptions options) {
    Map<String, KafkaFuture<TopicDescription>> topicDescriptions = new HashMap<>();
    if (timeoutNextRequests > 0) {
        for (String requestedTopic : topicNames) {
            KafkaFutureImpl<TopicDescription> future = new KafkaFutureImpl<>();
            future.completeExceptionally(new TimeoutException());
            topicDescriptions.put(requestedTopic, future);
        }
        --timeoutNextRequests;
        return topicDescriptions;
    }
    for (String requestedTopic : topicNames) {
        for (Map.Entry<String, TopicMetadata> topicDescription : allTopics.entrySet()) {
            String topicName = topicDescription.getKey();
            Uuid topicId = topicIds.getOrDefault(topicName, Uuid.ZERO_UUID);
            if (topicName.equals(requestedTopic) && !topicDescription.getValue().markedForDeletion) {
                if (topicDescription.getValue().fetchesRemainingUntilVisible > 0) {
                    topicDescription.getValue().fetchesRemainingUntilVisible--;
                } else {
                    TopicMetadata topicMetadata = topicDescription.getValue();
                    KafkaFutureImpl<TopicDescription> future = new KafkaFutureImpl<>();
                    future.complete(new TopicDescription(topicName, topicMetadata.isInternalTopic, topicMetadata.partitions, Collections.emptySet(), topicId));
                    topicDescriptions.put(topicName, future);
                    break;
                }
            }
        }
        if (!topicDescriptions.containsKey(requestedTopic)) {
            KafkaFutureImpl<TopicDescription> future = new KafkaFutureImpl<>();
            future.completeExceptionally(new UnknownTopicOrPartitionException("Topic " + requestedTopic + " not found."));
            topicDescriptions.put(requestedTopic, future);
        }
    }
    return topicDescriptions;
}

Example 77 with KafkaFuture

use of org.apache.kafka.common.KafkaFuture in project kafka by apache.

the class StreamsResetter method doDelete.

// visible for testing
public void doDelete(final List<String> topicsToDelete, final Admin adminClient) {
    boolean hasDeleteErrors = false;
    final DeleteTopicsResult deleteTopicsResult = adminClient.deleteTopics(topicsToDelete);
    final Map<String, KafkaFuture<Void>> results = deleteTopicsResult.topicNameValues();
    for (final Map.Entry<String, KafkaFuture<Void>> entry : results.entrySet()) {
        try {
            entry.getValue().get(30, TimeUnit.SECONDS);
        } catch (final Exception e) {
            System.err.println("ERROR: deleting topic " + entry.getKey());
            e.printStackTrace(System.err);
            hasDeleteErrors = true;
        }
    }
    if (hasDeleteErrors) {
        throw new RuntimeException("Encountered an error deleting one or more topics");
    }
}

Example 78 with KafkaFuture

use of org.apache.kafka.common.KafkaFuture in project kafka by apache.

the class WorkerManager method createWorker.

public KafkaFuture<String> createWorker(long workerId, String taskId, TaskSpec spec) throws Throwable {
    try (ShutdownManager.Reference ref = shutdownManager.takeReference()) {
        final Worker worker = stateChangeExecutor.submit(new CreateWorker(workerId, taskId, spec, time.milliseconds())).get();
        if (worker.doneFuture != null) {
            log.info("{}: Ignoring request to create worker {}, because there is already " + "a worker with that id.", nodeName, workerId);
            return worker.doneFuture;
        }
        worker.doneFuture = new KafkaFutureImpl<>();
        if (worker.spec.endMs() <= time.milliseconds()) {
            log.info("{}: Will not run worker {} as it has expired.", nodeName, worker);
            stateChangeExecutor.submit(new HandleWorkerHalting(worker, "worker expired", true));
            return worker.doneFuture;
        }
        KafkaFutureImpl<String> haltFuture = new KafkaFutureImpl<>();
        haltFuture.thenApply((KafkaFuture.BaseFunction<String, Void>) errorString -> {
            if (errorString == null)
                errorString = "";
            if (errorString.isEmpty()) {
                log.info("{}: Worker {} is halting.", nodeName, worker);
            } else {
                log.info("{}: Worker {} is halting with error {}", nodeName, worker, errorString);
            }
            stateChangeExecutor.submit(new HandleWorkerHalting(worker, errorString, false));
            return null;
        });
        try {
            worker.taskWorker.start(platform, worker.status, haltFuture);
        } catch (Exception e) {
            log.info("{}: Worker {} start() exception", nodeName, worker, e);
            stateChangeExecutor.submit(new HandleWorkerHalting(worker, "worker.start() exception: " + Utils.stackTrace(e), true));
        }
        stateChangeExecutor.submit(new FinishCreatingWorker(worker));
        return worker.doneFuture;
    } catch (ExecutionException e) {
        if (e.getCause() instanceof RequestConflictException) {
            log.info("{}: request conflict while creating worker {} for task {} with spec {}.", nodeName, workerId, taskId, spec);
        } else {
            log.info("{}: Error creating worker {} for task {} with spec {}", nodeName, workerId, taskId, spec, e);
        }
        throw e.getCause();
    }
}

Example 79 with KafkaFuture

use of org.apache.kafka.common.KafkaFuture in project cruise-control by linkedin.

the class PartitionProvisionerTest method assertProvisionPartitionIncreaseConstructsCorrectResponse.

private void assertProvisionPartitionIncreaseConstructsCorrectResponse(KafkaCruiseControlUtils.CompletionType partitionIncreaseCompletion, ProvisionerState.State expectedState, String expectedSummary, boolean hasBrokerRecommendation) throws ExecutionException, InterruptedException, TimeoutException {
    int recommendedPartitionCount = expectedSummary.contains("Ignored") ? MOCK_IGNORED_PARTITION_COUNT : MOCK_PARTITION_COUNT;
    ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numPartitions(recommendedPartitionCount).topicPattern(MOCK_TOPIC_PATTERN).build();
    Map<String, ProvisionRecommendation> provisionRecommendation;
    if (hasBrokerRecommendation) {
        provisionRecommendation = Map.of(RECOMMENDER_UP, recommendation, RECOMMENDER_TO_EXECUTE, BROKER_REC_TO_EXECUTE);
    } else {
        provisionRecommendation = Collections.singletonMap(RECOMMENDER_UP, recommendation);
    }
    Map<String, KafkaFuture<TopicDescription>> describeTopicsValues = Collections.singletonMap(MOCK_TOPIC, MOCK_TOPIC_DESCRIPTION_FUTURE);
    EasyMock.expect(MOCK_ADMIN_CLIENT.describeTopics(Collections.singletonList(MOCK_TOPIC))).andReturn(MOCK_DESCRIBE_TOPICS_RESULT);
    EasyMock.expect(MOCK_DESCRIBE_TOPICS_RESULT.values()).andReturn(describeTopicsValues);
    if (partitionIncreaseCompletion == COMPLETED) {
        EasyMock.expect(MOCK_TOPIC_DESCRIPTION_FUTURE.get(CLIENT_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS)).andReturn(MOCK_TOPIC_DESCRIPTION);
        // Create partitions: for this test, we ignore the fact that the mock cluster has one node -- i.e. in reality a request to increase
        // partition count to two would fail in a cluster with one node.
        EasyMock.expect(MOCK_ADMIN_CLIENT.createPartitions(Collections.singletonMap(MOCK_TOPIC, EasyMock.anyObject()))).andReturn(MOCK_CREATE_PARTITIONS_RESULT);
        Map<String, KafkaFuture<Void>> createPartitionsResultValues = Collections.singletonMap(MOCK_TOPIC, MOCK_CREATE_PARTITIONS_FUTURE);
        EasyMock.expect(MOCK_CREATE_PARTITIONS_RESULT.values()).andReturn(createPartitionsResultValues);
        EasyMock.expect(MOCK_CREATE_PARTITIONS_FUTURE.get(CLIENT_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS)).andReturn(null);
    } else if (partitionIncreaseCompletion == KafkaCruiseControlUtils.CompletionType.COMPLETED_WITH_ERROR) {
        EasyMock.expect(MOCK_TOPIC_DESCRIPTION_FUTURE.get(CLIENT_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS)).andThrow(new ExecutionException(new InvalidTopicException()));
    } else {
        EasyMock.expect(MOCK_TOPIC_DESCRIPTION_FUTURE.get(CLIENT_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS)).andReturn(MOCK_TOPIC_DESCRIPTION);
    }
    EasyMock.replay(MOCK_ADMIN_CLIENT, MOCK_DESCRIBE_TOPICS_RESULT, MOCK_TOPIC_DESCRIPTION_FUTURE, MOCK_CREATE_PARTITIONS_FUTURE, MOCK_CREATE_PARTITIONS_RESULT);
    ProvisionerState results = _provisioner.rightsize(provisionRecommendation, RIGHTSIZE_OPTIONS);
    assertEquals(expectedState, results.state());
    assertEquals(expectedSummary, results.summary());
}

Example 80 with KafkaFuture

use of org.apache.kafka.common.KafkaFuture in project cruise-control by linkedin.

the class ExecutionTaskPlannerTest method testGetIntraBrokerPartitionMovementTasks.

@Test
public void testGetIntraBrokerPartitionMovementTasks() {
    ReplicaPlacementInfo r0d0 = new ReplicaPlacementInfo(0, "d0");
    ReplicaPlacementInfo r0d1 = new ReplicaPlacementInfo(0, "d1");
    ReplicaPlacementInfo r1d0 = new ReplicaPlacementInfo(1, "d0");
    ReplicaPlacementInfo r1d1 = new ReplicaPlacementInfo(1, "d1");
    List<ExecutionProposal> proposals = Collections.singletonList(new ExecutionProposal(new TopicPartition(TOPIC2, 0), 4, r0d0, Arrays.asList(r0d0, r1d1), Arrays.asList(r1d0, r0d1)));
    TopicPartitionReplica tpr0 = new TopicPartitionReplica(TOPIC2, 0, 0);
    TopicPartitionReplica tpr1 = new TopicPartitionReplica(TOPIC2, 0, 1);
    // Mock adminClient
    AdminClient mockAdminClient = EasyMock.mock(AdminClient.class);
    try {
        // Reflectively set constructors from package private to public.
        Constructor<DescribeReplicaLogDirsResult> constructor1 = DescribeReplicaLogDirsResult.class.getDeclaredConstructor(Map.class);
        constructor1.setAccessible(true);
        Constructor<ReplicaLogDirInfo> constructor2 = ReplicaLogDirInfo.class.getDeclaredConstructor(String.class, long.class, String.class, long.class);
        constructor2.setAccessible(true);
        Map<TopicPartitionReplica, KafkaFuture<ReplicaLogDirInfo>> futureByReplica = new HashMap<>();
        futureByReplica.put(tpr0, completedFuture(constructor2.newInstance("d0", 0L, null, -1L)));
        futureByReplica.put(tpr1, completedFuture(constructor2.newInstance("d1", 0L, null, -1L)));
        EasyMock.expect(mockAdminClient.describeReplicaLogDirs(anyObject())).andReturn(constructor1.newInstance(futureByReplica)).anyTimes();
        EasyMock.replay(mockAdminClient);
    } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
    // Let it go.
    }
    Set<PartitionInfo> partitions = new HashSet<>();
    Node[] isrArray = generateExpectedReplicas(proposals.get(0));
    partitions.add(new PartitionInfo(proposals.get(0).topicPartition().topic(), proposals.get(0).topicPartition().partition(), isrArray[0], isrArray, isrArray));
    Cluster expectedCluster = new Cluster(null, _expectedNodes, partitions, Collections.emptySet(), Collections.emptySet());
    StrategyOptions strategyOptions = new StrategyOptions.Builder(expectedCluster).build();
    ExecutionTaskPlanner planner = new ExecutionTaskPlanner(mockAdminClient, new KafkaCruiseControlConfig(KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties()));
    planner.addExecutionProposals(proposals, strategyOptions, null);
    assertEquals(1, planner.remainingLeadershipMovements().size());
    assertEquals(2, planner.remainingIntraBrokerReplicaMovements().size());
    planner.clear();
    assertEquals(0, planner.remainingLeadershipMovements().size());
    assertEquals(0, planner.remainingIntraBrokerReplicaMovements().size());
    EasyMock.verify(mockAdminClient);
}