Examples with BatchItem org.zalando.nakadi.domain.BatchItem used on opensource projects

Example 16 with BatchItem

use of org.zalando.nakadi.domain.BatchItem in project nakadi by zalando.

the class KafkaTopicRepositoryTest method whenPostEventOverflowsBufferThenUpdateItemStatus.

@Test
public void whenPostEventOverflowsBufferThenUpdateItemStatus() throws Exception {
    final BatchItem item = new BatchItem("{}", BatchItem.EmptyInjectionConfiguration.build(1, true), new BatchItem.InjectionConfiguration[BatchItem.Injection.values().length], Collections.emptyList());
    item.setPartition("1");
    final List<BatchItem> batch = new ArrayList<>();
    batch.add(item);
    when(kafkaProducer.partitionsFor(EXPECTED_PRODUCER_RECORD.topic())).thenReturn(ImmutableList.of(new PartitionInfo(EXPECTED_PRODUCER_RECORD.topic(), 1, new Node(1, "host", 9091), null, null)));
    Mockito.doThrow(BufferExhaustedException.class).when(kafkaProducer).send(any(), any());
    try {
        kafkaTopicRepository.syncPostBatch(EXPECTED_PRODUCER_RECORD.topic(), batch);
        fail();
    } catch (final EventPublishingException e) {
        assertThat(item.getResponse().getPublishingStatus(), equalTo(EventPublishingStatus.FAILED));
        assertThat(item.getResponse().getDetail(), equalTo("internal error"));
    }
}

Example 17 with BatchItem

use of org.zalando.nakadi.domain.BatchItem in project nakadi by zalando.

the class KafkaTopicRepositoryTest method whenKafkaPublishTimeoutThenCircuitIsOpened.

@Test
public void whenKafkaPublishTimeoutThenCircuitIsOpened() throws Exception {
    when(nakadiSettings.getKafkaSendTimeoutMs()).thenReturn(1000L);
    when(kafkaProducer.partitionsFor(EXPECTED_PRODUCER_RECORD.topic())).thenReturn(ImmutableList.of(new PartitionInfo(EXPECTED_PRODUCER_RECORD.topic(), 1, new Node(1, "host", 9091), null, null)));
    when(kafkaProducer.send(any(), any())).thenAnswer(invocation -> {
        final Callback callback = (Callback) invocation.getArguments()[1];
        callback.onCompletion(null, new TimeoutException());
        return null;
    });
    final List<BatchItem> batches = new LinkedList<>();
    for (int i = 0; i < 1000; i++) {
        try {
            final BatchItem batchItem = new BatchItem("{}", BatchItem.EmptyInjectionConfiguration.build(1, true), new BatchItem.InjectionConfiguration[BatchItem.Injection.values().length], Collections.emptyList());
            batchItem.setPartition("1");
            batches.add(batchItem);
            kafkaTopicRepository.syncPostBatch(EXPECTED_PRODUCER_RECORD.topic(), ImmutableList.of(batchItem));
            fail();
        } catch (final EventPublishingException e) {
        }
    }
    Assert.assertTrue(batches.stream().filter(item -> item.getResponse().getPublishingStatus() == EventPublishingStatus.FAILED && item.getResponse().getDetail().equals("short circuited")).count() >= 1);
}

Example 18 with BatchItem

use of org.zalando.nakadi.domain.BatchItem in project nakadi by zalando.

the class KafkaTopicRepositoryTest method whenKafkaPublishCallbackWithExceptionThenEventPublishingException.

@Test
public void whenKafkaPublishCallbackWithExceptionThenEventPublishingException() throws Exception {
    final BatchItem firstItem = new BatchItem("{}", BatchItem.EmptyInjectionConfiguration.build(1, true), new BatchItem.InjectionConfiguration[BatchItem.Injection.values().length], Collections.emptyList());
    firstItem.setPartition("1");
    final BatchItem secondItem = new BatchItem("{}", BatchItem.EmptyInjectionConfiguration.build(1, true), new BatchItem.InjectionConfiguration[BatchItem.Injection.values().length], Collections.emptyList());
    secondItem.setPartition("2");
    final List<BatchItem> batch = ImmutableList.of(firstItem, secondItem);
    when(kafkaProducer.partitionsFor(EXPECTED_PRODUCER_RECORD.topic())).thenReturn(ImmutableList.of(new PartitionInfo(EXPECTED_PRODUCER_RECORD.topic(), 1, new Node(1, "host", 9091), null, null), new PartitionInfo(EXPECTED_PRODUCER_RECORD.topic(), 2, new Node(1, "host", 9091), null, null)));
    when(kafkaProducer.send(any(), any())).thenAnswer(invocation -> {
        final ProducerRecord record = (ProducerRecord) invocation.getArguments()[0];
        final Callback callback = (Callback) invocation.getArguments()[1];
        if (record.partition() == 2) {
            // return exception only for second event
            callback.onCompletion(null, new Exception());
        } else {
            callback.onCompletion(null, null);
        }
        return null;
    });
    try {
        kafkaTopicRepository.syncPostBatch(EXPECTED_PRODUCER_RECORD.topic(), batch);
        fail();
    } catch (final EventPublishingException e) {
        assertThat(firstItem.getResponse().getPublishingStatus(), equalTo(EventPublishingStatus.SUBMITTED));
        assertThat(firstItem.getResponse().getDetail(), equalTo(""));
        assertThat(secondItem.getResponse().getPublishingStatus(), equalTo(EventPublishingStatus.FAILED));
        assertThat(secondItem.getResponse().getDetail(), equalTo("internal error"));
    }
}

Example 19 with BatchItem

use of org.zalando.nakadi.domain.BatchItem in project nakadi by zalando.

the class EventPublisherTest method createStringFromBatchItems.

private String createStringFromBatchItems(final List<BatchItem> batch) {
    final StringBuilder sb = new StringBuilder();
    sb.append("[");
    for (final BatchItem item : batch) {
        sb.append(item.getEvent().toString());
        sb.append(",");
    }
    sb.setCharAt(sb.length() - 1, ']');
    return sb.toString();
}

Example 20 with BatchItem

use of org.zalando.nakadi.domain.BatchItem in project nakadi by zalando.

the class KafkaTopicRepository method syncPostBatch.

@Override
public void syncPostBatch(final String topicId, final List<BatchItem> batch) throws EventPublishingException {
    final Producer<String, String> producer = kafkaFactory.takeProducer();
    try {
        final Map<String, String> partitionToBroker = producer.partitionsFor(topicId).stream().collect(Collectors.toMap(p -> String.valueOf(p.partition()), p -> String.valueOf(p.leader().id())));
        batch.forEach(item -> {
            Preconditions.checkNotNull(item.getPartition(), "BatchItem partition can't be null at the moment of publishing!");
            item.setBrokerId(partitionToBroker.get(item.getPartition()));
        });
        int shortCircuited = 0;
        final Map<BatchItem, CompletableFuture<Exception>> sendFutures = new HashMap<>();
        for (final BatchItem item : batch) {
            item.setStep(EventPublishingStep.PUBLISHING);
            final HystrixKafkaCircuitBreaker circuitBreaker = circuitBreakers.computeIfAbsent(item.getBrokerId(), brokerId -> new HystrixKafkaCircuitBreaker(brokerId));
            if (circuitBreaker.allowRequest()) {
                sendFutures.put(item, publishItem(producer, topicId, item, circuitBreaker));
            } else {
                shortCircuited++;
                item.updateStatusAndDetail(EventPublishingStatus.FAILED, "short circuited");
            }
        }
        if (shortCircuited > 0) {
            LOG.warn("Short circuiting request to Kafka {} time(s) due to timeout for topic {}", shortCircuited, topicId);
        }
        final CompletableFuture<Void> multiFuture = CompletableFuture.allOf(sendFutures.values().toArray(new CompletableFuture<?>[sendFutures.size()]));
        multiFuture.get(createSendTimeout(), TimeUnit.MILLISECONDS);
        // Now lets check for errors
        final Optional<Exception> needReset = sendFutures.entrySet().stream().filter(entry -> isExceptionShouldLeadToReset(entry.getValue().getNow(null))).map(entry -> entry.getValue().getNow(null)).findAny();
        if (needReset.isPresent()) {
            LOG.info("Terminating producer while publishing to topic {} because of unrecoverable exception", topicId, needReset.get());
            kafkaFactory.terminateProducer(producer);
        }
    } catch (final TimeoutException ex) {
        failUnpublished(batch, "timed out");
        throw new EventPublishingException("Error publishing message to kafka", ex);
    } catch (final ExecutionException ex) {
        failUnpublished(batch, "internal error");
        throw new EventPublishingException("Error publishing message to kafka", ex);
    } catch (final InterruptedException ex) {
        Thread.currentThread().interrupt();
        failUnpublished(batch, "interrupted");
        throw new EventPublishingException("Error publishing message to kafka", ex);
    } finally {
        kafkaFactory.releaseProducer(producer);
    }
    final boolean atLeastOneFailed = batch.stream().anyMatch(item -> item.getResponse().getPublishingStatus() == EventPublishingStatus.FAILED);
    if (atLeastOneFailed) {
        failUnpublished(batch, "internal error");
        throw new EventPublishingException("Error publishing message to kafka");
    }
}