Examples with WakeupException org.apache.kafka.common.errors.WakeupException used on opensource projects

Example 36 with WakeupException

use of org.apache.kafka.common.errors.WakeupException in project apache-kafka-on-k8s by banzaicloud.

the class AbstractTask method updateOffsetLimits.

protected void updateOffsetLimits() {
    for (final TopicPartition partition : partitions) {
        try {
            // TODO: batch API?
            final OffsetAndMetadata metadata = consumer.committed(partition);
            final long offset = metadata != null ? metadata.offset() : 0L;
            stateMgr.putOffsetLimit(partition, offset);
            if (log.isTraceEnabled()) {
                log.trace("Updating store offset limits {} for changelog {}", offset, partition);
            }
        } catch (final AuthorizationException e) {
            throw new ProcessorStateException(String.format("task [%s] AuthorizationException when initializing offsets for %s", id, partition), e);
        } catch (final WakeupException e) {
            throw e;
        } catch (final KafkaException e) {
            throw new ProcessorStateException(String.format("task [%s] Failed to initialize offsets for %s", id, partition), e);
        }
    }
}

Example 37 with WakeupException

use of org.apache.kafka.common.errors.WakeupException in project apache-kafka-on-k8s by banzaicloud.

the class WorkerSinkTaskTest method testWakeupInCommitSyncCausesRetry.

@Test
public void testWakeupInCommitSyncCausesRetry() throws Exception {
    createTask(initialState);
    expectInitializeTask();
    expectPollInitialAssignment();
    expectConsumerPoll(1);
    expectConversionAndTransformation(1);
    sinkTask.put(EasyMock.<Collection<SinkRecord>>anyObject());
    EasyMock.expectLastCall();
    final List<TopicPartition> partitions = asList(TOPIC_PARTITION, TOPIC_PARTITION2);
    final Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
    offsets.put(TOPIC_PARTITION, new OffsetAndMetadata(FIRST_OFFSET + 1));
    offsets.put(TOPIC_PARTITION2, new OffsetAndMetadata(FIRST_OFFSET));
    sinkTask.preCommit(offsets);
    EasyMock.expectLastCall().andReturn(offsets);
    // first one raises wakeup
    consumer.commitSync(EasyMock.<Map<TopicPartition, OffsetAndMetadata>>anyObject());
    EasyMock.expectLastCall().andThrow(new WakeupException());
    // we should retry and complete the commit
    consumer.commitSync(EasyMock.<Map<TopicPartition, OffsetAndMetadata>>anyObject());
    EasyMock.expectLastCall();
    sinkTask.close(new HashSet<>(partitions));
    EasyMock.expectLastCall();
    EasyMock.expect(consumer.position(TOPIC_PARTITION)).andReturn(FIRST_OFFSET);
    EasyMock.expect(consumer.position(TOPIC_PARTITION2)).andReturn(FIRST_OFFSET);
    sinkTask.open(partitions);
    EasyMock.expectLastCall();
    EasyMock.expect(consumer.poll(EasyMock.anyLong())).andAnswer(new IAnswer<ConsumerRecords<byte[], byte[]>>() {

        @Override
        public ConsumerRecords<byte[], byte[]> answer() throws Throwable {
            rebalanceListener.getValue().onPartitionsRevoked(partitions);
            rebalanceListener.getValue().onPartitionsAssigned(partitions);
            return ConsumerRecords.empty();
        }
    });
    EasyMock.expect(consumer.assignment()).andReturn(new HashSet<>(partitions));
    consumer.resume(Collections.singleton(TOPIC_PARTITION));
    EasyMock.expectLastCall();
    consumer.resume(Collections.singleton(TOPIC_PARTITION2));
    EasyMock.expectLastCall();
    statusListener.onResume(taskId);
    EasyMock.expectLastCall();
    PowerMock.replayAll();
    workerTask.initialize(TASK_CONFIG);
    time.sleep(30000L);
    workerTask.initializeAndStart();
    time.sleep(30000L);
    // poll for initial assignment
    workerTask.iteration();
    time.sleep(30000L);
    // first record delivered
    workerTask.iteration();
    // now rebalance with the wakeup triggered
    workerTask.iteration();
    time.sleep(30000L);
    assertSinkMetricValue("partition-count", 2);
    assertSinkMetricValue("sink-record-read-total", 1.0);
    assertSinkMetricValue("sink-record-send-total", 1.0);
    assertSinkMetricValue("sink-record-active-count", 0.0);
    assertSinkMetricValue("sink-record-active-count-max", 1.0);
    assertSinkMetricValue("sink-record-active-count-avg", 0.33333);
    assertSinkMetricValue("offset-commit-seq-no", 1.0);
    assertSinkMetricValue("offset-commit-completion-total", 1.0);
    assertSinkMetricValue("offset-commit-skip-total", 0.0);
    assertTaskMetricValue("status", "running");
    assertTaskMetricValue("running-ratio", 1.0);
    assertTaskMetricValue("pause-ratio", 0.0);
    assertTaskMetricValue("batch-size-max", 1.0);
    assertTaskMetricValue("batch-size-avg", 1.0);
    assertTaskMetricValue("offset-commit-max-time-ms", 0.0);
    assertTaskMetricValue("offset-commit-avg-time-ms", 0.0);
    assertTaskMetricValue("offset-commit-failure-percentage", 0.0);
    assertTaskMetricValue("offset-commit-success-percentage", 1.0);
    PowerMock.verifyAll();
}

Example 38 with WakeupException

use of org.apache.kafka.common.errors.WakeupException in project apache-kafka-on-k8s by banzaicloud.

the class ConsumerCoordinator method onJoinComplete.

@Override
protected void onJoinComplete(int generation, String memberId, String assignmentStrategy, ByteBuffer assignmentBuffer) {
    // only the leader is responsible for monitoring for metadata changes (i.e. partition changes)
    if (!isLeader)
        assignmentSnapshot = null;
    PartitionAssignor assignor = lookupAssignor(assignmentStrategy);
    if (assignor == null)
        throw new IllegalStateException("Coordinator selected invalid assignment protocol: " + assignmentStrategy);
    Assignment assignment = ConsumerProtocol.deserializeAssignment(assignmentBuffer);
    subscriptions.assignFromSubscribed(assignment.partitions());
    // check if the assignment contains some topics that were not in the original
    // subscription, if yes we will obey what leader has decided and add these topics
    // into the subscriptions as long as they still match the subscribed pattern
    // 
    // TODO this part of the logic should be removed once we allow regex on leader assign
    Set<String> addedTopics = new HashSet<>();
    for (TopicPartition tp : subscriptions.assignedPartitions()) {
        if (!joinedSubscription.contains(tp.topic()))
            addedTopics.add(tp.topic());
    }
    if (!addedTopics.isEmpty()) {
        Set<String> newSubscription = new HashSet<>(subscriptions.subscription());
        Set<String> newJoinedSubscription = new HashSet<>(joinedSubscription);
        newSubscription.addAll(addedTopics);
        newJoinedSubscription.addAll(addedTopics);
        this.subscriptions.subscribeFromPattern(newSubscription);
        this.joinedSubscription = newJoinedSubscription;
    }
    // update the metadata and enforce a refresh to make sure the fetcher can start
    // fetching data in the next iteration
    this.metadata.setTopics(subscriptions.groupSubscription());
    client.ensureFreshMetadata();
    // give the assignor a chance to update internal state based on the received assignment
    assignor.onAssignment(assignment);
    // reschedule the auto commit starting from now
    this.nextAutoCommitDeadline = time.milliseconds() + autoCommitIntervalMs;
    // execute the user's callback after rebalance
    ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
    log.info("Setting newly assigned partitions {}", subscriptions.assignedPartitions());
    try {
        Set<TopicPartition> assigned = new HashSet<>(subscriptions.assignedPartitions());
        listener.onPartitionsAssigned(assigned);
    } catch (WakeupException | InterruptException e) {
        throw e;
    } catch (Exception e) {
        log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e);
    }
}

Example 39 with WakeupException

use of org.apache.kafka.common.errors.WakeupException in project apache-kafka-on-k8s by banzaicloud.

the class MockConsumer method poll.

@Override
public synchronized ConsumerRecords<K, V> poll(long timeout) {
    ensureNotClosed();
    // the callback
    synchronized (pollTasks) {
        Runnable task = pollTasks.poll();
        if (task != null)
            task.run();
    }
    if (wakeup.get()) {
        wakeup.set(false);
        throw new WakeupException();
    }
    if (exception != null) {
        RuntimeException exception = this.exception;
        this.exception = null;
        throw exception;
    }
    // Handle seeks that need to wait for a poll() call to be processed
    for (TopicPartition tp : subscriptions.assignedPartitions()) if (!subscriptions.hasValidPosition(tp))
        updateFetchPosition(tp);
    // update the consumed offset
    final Map<TopicPartition, List<ConsumerRecord<K, V>>> results = new HashMap<>();
    for (final TopicPartition topicPartition : records.keySet()) {
        results.put(topicPartition, new ArrayList<ConsumerRecord<K, V>>());
    }
    for (Map.Entry<TopicPartition, List<ConsumerRecord<K, V>>> entry : this.records.entrySet()) {
        if (!subscriptions.isPaused(entry.getKey())) {
            final List<ConsumerRecord<K, V>> recs = entry.getValue();
            for (final ConsumerRecord<K, V> rec : recs) {
                if (assignment().contains(entry.getKey()) && rec.offset() >= subscriptions.position(entry.getKey())) {
                    results.get(entry.getKey()).add(rec);
                    subscriptions.position(entry.getKey(), rec.offset() + 1);
                }
            }
        }
    }
    this.records.clear();
    return new ConsumerRecords<>(results);
}

Example 40 with WakeupException

use of org.apache.kafka.common.errors.WakeupException in project apache-kafka-on-k8s by banzaicloud.

the class AbstractCoordinatorTest method testWakeupAfterSyncGroupSent.

@Test
public void testWakeupAfterSyncGroupSent() throws Exception {
    setupCoordinator(RETRY_BACKOFF_MS);
    mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    mockClient.prepareResponse(joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
    mockClient.prepareResponse(new MockClient.RequestMatcher() {

        private int invocations = 0;

        @Override
        public boolean matches(AbstractRequest body) {
            invocations++;
            boolean isSyncGroupRequest = body instanceof SyncGroupRequest;
            if (isSyncGroupRequest && invocations == 1)
                // simulate wakeup after the request sent
                throw new WakeupException();
            return isSyncGroupRequest;
        }
    }, syncGroupResponse(Errors.NONE));
    AtomicBoolean heartbeatReceived = prepareFirstHeartbeat();
    try {
        coordinator.ensureActiveGroup();
        fail("Should have woken up from ensureActiveGroup()");
    } catch (WakeupException e) {
    }
    assertEquals(1, coordinator.onJoinPrepareInvokes);
    assertEquals(0, coordinator.onJoinCompleteInvokes);
    assertFalse(heartbeatReceived.get());
    coordinator.ensureActiveGroup();
    assertEquals(1, coordinator.onJoinPrepareInvokes);
    assertEquals(1, coordinator.onJoinCompleteInvokes);
    awaitFirstHeartbeat(heartbeatReceived);
}