Examples with DeleteRecordsResponse org.apache.kafka.common.requests.DeleteRecordsResponse used on opensource projects

Example 1 with DeleteRecordsResponse

use of org.apache.kafka.common.requests.DeleteRecordsResponse in project apache-kafka-on-k8s by banzaicloud.

the class KafkaAdminClient method deleteRecords.

public DeleteRecordsResult deleteRecords(final Map<TopicPartition, RecordsToDelete> recordsToDelete, final DeleteRecordsOptions options) {
    // requests need to be sent to partitions leader nodes so ...
    // ... from the provided map it's needed to create more maps grouping topic/partition per leader
    final Map<TopicPartition, KafkaFutureImpl<DeletedRecords>> futures = new HashMap<>(recordsToDelete.size());
    for (TopicPartition topicPartition : recordsToDelete.keySet()) {
        futures.put(topicPartition, new KafkaFutureImpl<DeletedRecords>());
    }
    // preparing topics list for asking metadata about them
    final Set<String> topics = new HashSet<>();
    for (TopicPartition topicPartition : recordsToDelete.keySet()) {
        topics.add(topicPartition.topic());
    }
    final long nowMetadata = time.milliseconds();
    final long deadline = calcDeadlineMs(nowMetadata, options.timeoutMs());
    // asking for topics metadata for getting partitions leaders
    runnable.call(new Call("topicsMetadata", deadline, new LeastLoadedNodeProvider()) {

        @Override
        AbstractRequest.Builder createRequest(int timeoutMs) {
            return new MetadataRequest.Builder(new ArrayList<>(topics), false);
        }

        @Override
        void handleResponse(AbstractResponse abstractResponse) {
            MetadataResponse response = (MetadataResponse) abstractResponse;
            Map<String, Errors> errors = response.errors();
            Cluster cluster = response.cluster();
            // completing futures for topics with errors
            for (Map.Entry<String, Errors> topicError : errors.entrySet()) {
                for (Map.Entry<TopicPartition, KafkaFutureImpl<DeletedRecords>> future : futures.entrySet()) {
                    if (future.getKey().topic().equals(topicError.getKey())) {
                        future.getValue().completeExceptionally(topicError.getValue().exception());
                    }
                }
            }
            // grouping topic partitions per leader
            Map<Node, Map<TopicPartition, Long>> leaders = new HashMap<>();
            for (Map.Entry<TopicPartition, RecordsToDelete> entry : recordsToDelete.entrySet()) {
                // avoiding to send deletion request for topics with errors
                if (!errors.containsKey(entry.getKey().topic())) {
                    Node node = cluster.leaderFor(entry.getKey());
                    if (node != null) {
                        if (!leaders.containsKey(node))
                            leaders.put(node, new HashMap<TopicPartition, Long>());
                        leaders.get(node).put(entry.getKey(), entry.getValue().beforeOffset());
                    } else {
                        KafkaFutureImpl<DeletedRecords> future = futures.get(entry.getKey());
                        future.completeExceptionally(Errors.LEADER_NOT_AVAILABLE.exception());
                    }
                }
            }
            for (final Map.Entry<Node, Map<TopicPartition, Long>> entry : leaders.entrySet()) {
                final long nowDelete = time.milliseconds();
                final int brokerId = entry.getKey().id();
                runnable.call(new Call("deleteRecords", deadline, new ConstantNodeIdProvider(brokerId)) {

                    @Override
                    AbstractRequest.Builder createRequest(int timeoutMs) {
                        return new DeleteRecordsRequest.Builder(timeoutMs, entry.getValue());
                    }

                    @Override
                    void handleResponse(AbstractResponse abstractResponse) {
                        DeleteRecordsResponse response = (DeleteRecordsResponse) abstractResponse;
                        for (Map.Entry<TopicPartition, DeleteRecordsResponse.PartitionResponse> result : response.responses().entrySet()) {
                            KafkaFutureImpl<DeletedRecords> future = futures.get(result.getKey());
                            if (result.getValue().error == Errors.NONE) {
                                future.complete(new DeletedRecords(result.getValue().lowWatermark));
                            } else {
                                future.completeExceptionally(result.getValue().error.exception());
                            }
                        }
                    }

                    @Override
                    void handleFailure(Throwable throwable) {
                        completeAllExceptionally(futures.values(), throwable);
                    }
                }, nowDelete);
            }
        }

        @Override
        void handleFailure(Throwable throwable) {
            completeAllExceptionally(futures.values(), throwable);
        }
    }, nowMetadata);
    return new DeleteRecordsResult(new HashMap<TopicPartition, KafkaFuture<DeletedRecords>>(futures));
}

Example 2 with DeleteRecordsResponse

use of org.apache.kafka.common.requests.DeleteRecordsResponse in project apache-kafka-on-k8s by banzaicloud.

the class KafkaAdminClientTest method testDeleteRecords.

@Test
public void testDeleteRecords() throws Exception {
    HashMap<Integer, Node> nodes = new HashMap<>();
    nodes.put(0, new Node(0, "localhost", 8121));
    List<PartitionInfo> partitionInfos = new ArrayList<>();
    partitionInfos.add(new PartitionInfo("my_topic", 0, nodes.get(0), new Node[] { nodes.get(0) }, new Node[] { nodes.get(0) }));
    partitionInfos.add(new PartitionInfo("my_topic", 1, nodes.get(0), new Node[] { nodes.get(0) }, new Node[] { nodes.get(0) }));
    partitionInfos.add(new PartitionInfo("my_topic", 2, null, new Node[] { nodes.get(0) }, new Node[] { nodes.get(0) }));
    partitionInfos.add(new PartitionInfo("my_topic", 3, nodes.get(0), new Node[] { nodes.get(0) }, new Node[] { nodes.get(0) }));
    partitionInfos.add(new PartitionInfo("my_topic", 4, nodes.get(0), new Node[] { nodes.get(0) }, new Node[] { nodes.get(0) }));
    Cluster cluster = new Cluster("mockClusterId", nodes.values(), partitionInfos, Collections.<String>emptySet(), Collections.<String>emptySet(), nodes.get(0));
    TopicPartition myTopicPartition0 = new TopicPartition("my_topic", 0);
    TopicPartition myTopicPartition1 = new TopicPartition("my_topic", 1);
    TopicPartition myTopicPartition2 = new TopicPartition("my_topic", 2);
    TopicPartition myTopicPartition3 = new TopicPartition("my_topic", 3);
    TopicPartition myTopicPartition4 = new TopicPartition("my_topic", 4);
    try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) {
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
        env.kafkaClient().prepareMetadataUpdate(env.cluster(), Collections.<String>emptySet());
        env.kafkaClient().setNode(env.cluster().nodes().get(0));
        Map<TopicPartition, DeleteRecordsResponse.PartitionResponse> m = new HashMap<>();
        m.put(myTopicPartition0, new DeleteRecordsResponse.PartitionResponse(3, Errors.NONE));
        m.put(myTopicPartition1, new DeleteRecordsResponse.PartitionResponse(DeleteRecordsResponse.INVALID_LOW_WATERMARK, Errors.OFFSET_OUT_OF_RANGE));
        m.put(myTopicPartition3, new DeleteRecordsResponse.PartitionResponse(DeleteRecordsResponse.INVALID_LOW_WATERMARK, Errors.NOT_LEADER_FOR_PARTITION));
        m.put(myTopicPartition4, new DeleteRecordsResponse.PartitionResponse(DeleteRecordsResponse.INVALID_LOW_WATERMARK, Errors.UNKNOWN_TOPIC_OR_PARTITION));
        List<MetadataResponse.TopicMetadata> t = new ArrayList<>();
        List<MetadataResponse.PartitionMetadata> p = new ArrayList<>();
        p.add(new MetadataResponse.PartitionMetadata(Errors.NONE, 0, nodes.get(0), Collections.singletonList(nodes.get(0)), Collections.singletonList(nodes.get(0)), Collections.<Node>emptyList()));
        p.add(new MetadataResponse.PartitionMetadata(Errors.NONE, 1, nodes.get(0), Collections.singletonList(nodes.get(0)), Collections.singletonList(nodes.get(0)), Collections.<Node>emptyList()));
        p.add(new MetadataResponse.PartitionMetadata(Errors.LEADER_NOT_AVAILABLE, 2, null, Collections.singletonList(nodes.get(0)), Collections.singletonList(nodes.get(0)), Collections.<Node>emptyList()));
        p.add(new MetadataResponse.PartitionMetadata(Errors.NONE, 3, nodes.get(0), Collections.singletonList(nodes.get(0)), Collections.singletonList(nodes.get(0)), Collections.<Node>emptyList()));
        p.add(new MetadataResponse.PartitionMetadata(Errors.NONE, 4, nodes.get(0), Collections.singletonList(nodes.get(0)), Collections.singletonList(nodes.get(0)), Collections.<Node>emptyList()));
        t.add(new MetadataResponse.TopicMetadata(Errors.NONE, "my_topic", false, p));
        env.kafkaClient().prepareResponse(new MetadataResponse(cluster.nodes(), cluster.clusterResource().clusterId(), cluster.controller().id(), t));
        env.kafkaClient().prepareResponse(new DeleteRecordsResponse(0, m));
        Map<TopicPartition, RecordsToDelete> recordsToDelete = new HashMap<>();
        recordsToDelete.put(myTopicPartition0, RecordsToDelete.beforeOffset(3L));
        recordsToDelete.put(myTopicPartition1, RecordsToDelete.beforeOffset(10L));
        recordsToDelete.put(myTopicPartition2, RecordsToDelete.beforeOffset(10L));
        recordsToDelete.put(myTopicPartition3, RecordsToDelete.beforeOffset(10L));
        recordsToDelete.put(myTopicPartition4, RecordsToDelete.beforeOffset(10L));
        DeleteRecordsResult results = env.adminClient().deleteRecords(recordsToDelete);
        // success on records deletion for partition 0
        Map<TopicPartition, KafkaFuture<DeletedRecords>> values = results.lowWatermarks();
        KafkaFuture<DeletedRecords> myTopicPartition0Result = values.get(myTopicPartition0);
        long lowWatermark = myTopicPartition0Result.get().lowWatermark();
        assertEquals(lowWatermark, 3);
        // "offset out of range" failure on records deletion for partition 1
        KafkaFuture<DeletedRecords> myTopicPartition1Result = values.get(myTopicPartition1);
        try {
            myTopicPartition1Result.get();
            fail("get() should throw ExecutionException");
        } catch (ExecutionException e0) {
            assertTrue(e0.getCause() instanceof OffsetOutOfRangeException);
        }
        // "leader not available" failure on metadata request for partition 2
        KafkaFuture<DeletedRecords> myTopicPartition2Result = values.get(myTopicPartition2);
        try {
            myTopicPartition2Result.get();
            fail("get() should throw ExecutionException");
        } catch (ExecutionException e1) {
            assertTrue(e1.getCause() instanceof LeaderNotAvailableException);
        }
        // "not leader for partition" failure on records deletion for partition 3
        KafkaFuture<DeletedRecords> myTopicPartition3Result = values.get(myTopicPartition3);
        try {
            myTopicPartition3Result.get();
            fail("get() should throw ExecutionException");
        } catch (ExecutionException e1) {
            assertTrue(e1.getCause() instanceof NotLeaderForPartitionException);
        }
        // "unknown topic or partition" failure on records deletion for partition 4
        KafkaFuture<DeletedRecords> myTopicPartition4Result = values.get(myTopicPartition4);
        try {
            myTopicPartition4Result.get();
            fail("get() should throw ExecutionException");
        } catch (ExecutionException e1) {
            assertTrue(e1.getCause() instanceof UnknownTopicOrPartitionException);
        }
    }
}