Examples with WakeupException org.apache.kafka.common.errors.WakeupException used on opensource projects

Example 11 with WakeupException

use of org.apache.kafka.common.errors.WakeupException in project flink by apache.

the class KafkaConsumerThread method run.

// ------------------------------------------------------------------------
@Override
public void run() {
    // early exit check
    if (!running) {
        return;
    }
    // this is the means to talk to FlinkKafkaConsumer's main thread
    final Handover handover = this.handover;
    // This method initializes the KafkaConsumer and guarantees it is torn down properly.
    // This is important, because the consumer has multi-threading issues,
    // including concurrent 'close()' calls.
    final KafkaConsumer<byte[], byte[]> consumer;
    try {
        consumer = new KafkaConsumer<>(kafkaProperties);
    } catch (Throwable t) {
        handover.reportError(t);
        return;
    }
    // from here on, the consumer is guaranteed to be closed properly
    try {
        // The callback invoked by Kafka once an offset commit is complete
        final OffsetCommitCallback offsetCommitCallback = new CommitCallback();
        // tell the consumer which partitions to work with
        consumerCallBridge.assignPartitions(consumer, convertKafkaPartitions(subscribedPartitionStates));
        // register Kafka's very own metrics in Flink's metric reporters
        if (useMetrics) {
            // register Kafka metrics to Flink
            Map<MetricName, ? extends Metric> metrics = consumer.metrics();
            if (metrics == null) {
                // MapR's Kafka implementation returns null here.
                log.info("Consumer implementation does not support metrics");
            } else {
                // we have Kafka metrics, register them
                for (Map.Entry<MetricName, ? extends Metric> metric : metrics.entrySet()) {
                    kafkaMetricGroup.gauge(metric.getKey().name(), new KafkaMetricWrapper(metric.getValue()));
                }
            }
        }
        // early exit check
        if (!running) {
            return;
        }
        // values yet; replace those with actual offsets, according to what the sentinel value represent.
        for (KafkaTopicPartitionState<TopicPartition> partition : subscribedPartitionStates) {
            if (partition.getOffset() == KafkaTopicPartitionStateSentinel.EARLIEST_OFFSET) {
                consumerCallBridge.seekPartitionToBeginning(consumer, partition.getKafkaPartitionHandle());
                partition.setOffset(consumer.position(partition.getKafkaPartitionHandle()) - 1);
            } else if (partition.getOffset() == KafkaTopicPartitionStateSentinel.LATEST_OFFSET) {
                consumerCallBridge.seekPartitionToEnd(consumer, partition.getKafkaPartitionHandle());
                partition.setOffset(consumer.position(partition.getKafkaPartitionHandle()) - 1);
            } else if (partition.getOffset() == KafkaTopicPartitionStateSentinel.GROUP_OFFSET) {
                // the KafkaConsumer by default will automatically seek the consumer position
                // to the committed group offset, so we do not need to do it.
                partition.setOffset(consumer.position(partition.getKafkaPartitionHandle()) - 1);
            } else {
                consumer.seek(partition.getKafkaPartitionHandle(), partition.getOffset() + 1);
            }
        }
        // from now on, external operations may call the consumer
        this.consumer = consumer;
        // the latest bulk of records. may carry across the loop if the thread is woken up
        // from blocking on the handover
        ConsumerRecords<byte[], byte[]> records = null;
        // main fetch loop
        while (running) {
            // check if there is something to commit
            if (!commitInProgress) {
                // get and reset the work-to-be committed, so we don't repeatedly commit the same
                final Map<TopicPartition, OffsetAndMetadata> toCommit = nextOffsetsToCommit.getAndSet(null);
                if (toCommit != null) {
                    log.debug("Sending async offset commit request to Kafka broker");
                    // also record that a commit is already in progress
                    // the order here matters! first set the flag, then send the commit command.
                    commitInProgress = true;
                    consumer.commitAsync(toCommit, offsetCommitCallback);
                }
            }
            // get the next batch of records, unless we did not manage to hand the old batch over
            if (records == null) {
                try {
                    records = consumer.poll(pollTimeout);
                } catch (WakeupException we) {
                    continue;
                }
            }
            try {
                handover.produce(records);
                records = null;
            } catch (Handover.WakeupException e) {
            // fall through the loop
            }
        }
    // end main fetch loop
    } catch (Throwable t) {
        // let the main thread know and exit
        // it may be that this exception comes because the main thread closed the handover, in
        // which case the below reporting is irrelevant, but does not hurt either
        handover.reportError(t);
    } finally {
        // make sure the handover is closed if it is not already closed or has an error
        handover.close();
        // make sure the KafkaConsumer is closed
        try {
            consumer.close();
        } catch (Throwable t) {
            log.warn("Error while closing Kafka consumer", t);
        }
    }
}

Example 12 with WakeupException

use of org.apache.kafka.common.errors.WakeupException in project kafka by apache.

the class KafkaConsumerTest method testWakeupWithFetchDataAvailable.

@Test
public void testWakeupWithFetchDataAvailable() {
    int rebalanceTimeoutMs = 60000;
    int sessionTimeoutMs = 30000;
    int heartbeatIntervalMs = 3000;
    // adjust auto commit interval lower than heartbeat so we don't need to deal with
    // a concurrent heartbeat request
    int autoCommitIntervalMs = 1000;
    Time time = new MockTime();
    Cluster cluster = TestUtils.singletonCluster(topic, 1);
    Node node = cluster.nodes().get(0);
    Metadata metadata = new Metadata(0, Long.MAX_VALUE);
    metadata.update(cluster, Collections.<String>emptySet(), time.milliseconds());
    MockClient client = new MockClient(time, metadata);
    client.setNode(node);
    PartitionAssignor assignor = new RoundRobinAssignor();
    final KafkaConsumer<String, String> consumer = newConsumer(time, client, metadata, assignor, rebalanceTimeoutMs, sessionTimeoutMs, heartbeatIntervalMs, true, autoCommitIntervalMs);
    consumer.subscribe(Arrays.asList(topic), getConsumerRebalanceListener(consumer));
    prepareRebalance(client, node, assignor, Arrays.asList(tp0), null);
    consumer.poll(0);
    // respond to the outstanding fetch so that we have data available on the next poll
    client.respondFrom(fetchResponse(tp0, 0, 5), node);
    client.poll(0, time.milliseconds());
    consumer.wakeup();
    try {
        consumer.poll(0);
        fail();
    } catch (WakeupException e) {
    }
    // make sure the position hasn't been updated
    assertEquals(0, consumer.position(tp0));
    // the next poll should return the completed fetch
    ConsumerRecords<String, String> records = consumer.poll(0);
    assertEquals(5, records.count());
}

Example 13 with WakeupException

use of org.apache.kafka.common.errors.WakeupException in project kafka by apache.

the class AbstractCoordinatorTest method testWakeupAfterJoinGroupSentExternalCompletion.

@Test
public void testWakeupAfterJoinGroupSentExternalCompletion() throws Exception {
    mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    mockClient.prepareResponse(new MockClient.RequestMatcher() {

        private int invocations = 0;

        @Override
        public boolean matches(AbstractRequest body) {
            invocations++;
            boolean isJoinGroupRequest = body instanceof JoinGroupRequest;
            if (isJoinGroupRequest && invocations == 1)
                // simulate wakeup before the request returns
                throw new WakeupException();
            return isJoinGroupRequest;
        }
    }, joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
    mockClient.prepareResponse(syncGroupResponse(Errors.NONE));
    AtomicBoolean heartbeatReceived = prepareFirstHeartbeat();
    try {
        coordinator.ensureActiveGroup();
        fail("Should have woken up from ensureActiveGroup()");
    } catch (WakeupException e) {
    }
    assertEquals(1, coordinator.onJoinPrepareInvokes);
    assertEquals(0, coordinator.onJoinCompleteInvokes);
    assertFalse(heartbeatReceived.get());
    // the join group completes in this poll()
    consumerClient.poll(0);
    coordinator.ensureActiveGroup();
    assertEquals(1, coordinator.onJoinPrepareInvokes);
    assertEquals(1, coordinator.onJoinCompleteInvokes);
    awaitFirstHeartbeat(heartbeatReceived);
}

Example 14 with WakeupException

use of org.apache.kafka.common.errors.WakeupException in project kafka by apache.

the class AbstractCoordinatorTest method testWakeupAfterSyncGroupReceived.

@Test
public void testWakeupAfterSyncGroupReceived() throws Exception {
    mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    mockClient.prepareResponse(joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
    mockClient.prepareResponse(new MockClient.RequestMatcher() {

        @Override
        public boolean matches(AbstractRequest body) {
            boolean isSyncGroupRequest = body instanceof SyncGroupRequest;
            if (isSyncGroupRequest)
                // wakeup after the request returns
                consumerClient.wakeup();
            return isSyncGroupRequest;
        }
    }, syncGroupResponse(Errors.NONE));
    AtomicBoolean heartbeatReceived = prepareFirstHeartbeat();
    try {
        coordinator.ensureActiveGroup();
        fail("Should have woken up from ensureActiveGroup()");
    } catch (WakeupException e) {
    }
    assertEquals(1, coordinator.onJoinPrepareInvokes);
    assertEquals(0, coordinator.onJoinCompleteInvokes);
    assertFalse(heartbeatReceived.get());
    coordinator.ensureActiveGroup();
    assertEquals(1, coordinator.onJoinPrepareInvokes);
    assertEquals(1, coordinator.onJoinCompleteInvokes);
    awaitFirstHeartbeat(heartbeatReceived);
}

Example 15 with WakeupException

use of org.apache.kafka.common.errors.WakeupException in project kafka by apache.

the class AbstractCoordinatorTest method testWakeupAfterJoinGroupReceived.

@Test
public void testWakeupAfterJoinGroupReceived() throws Exception {
    mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    mockClient.prepareResponse(new MockClient.RequestMatcher() {

        @Override
        public boolean matches(AbstractRequest body) {
            boolean isJoinGroupRequest = body instanceof JoinGroupRequest;
            if (isJoinGroupRequest)
                // wakeup after the request returns
                consumerClient.wakeup();
            return isJoinGroupRequest;
        }
    }, joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
    mockClient.prepareResponse(syncGroupResponse(Errors.NONE));
    AtomicBoolean heartbeatReceived = prepareFirstHeartbeat();
    try {
        coordinator.ensureActiveGroup();
        fail("Should have woken up from ensureActiveGroup()");
    } catch (WakeupException e) {
    }
    assertEquals(1, coordinator.onJoinPrepareInvokes);
    assertEquals(0, coordinator.onJoinCompleteInvokes);
    assertFalse(heartbeatReceived.get());
    coordinator.ensureActiveGroup();
    assertEquals(1, coordinator.onJoinPrepareInvokes);
    assertEquals(1, coordinator.onJoinCompleteInvokes);
    awaitFirstHeartbeat(heartbeatReceived);
}