Search in sources :

Example 1 with EasyMock.expectLastCall

use of org.easymock.EasyMock.expectLastCall in project kafka by apache.

the class TaskManagerTest method shouldReviveCorruptTasks.

@Test
public void shouldReviveCorruptTasks() {
    final ProcessorStateManager stateManager = EasyMock.createStrictMock(ProcessorStateManager.class);
    stateManager.markChangelogAsCorrupted(taskId00Partitions);
    EasyMock.expectLastCall().once();
    replay(stateManager);
    final AtomicBoolean enforcedCheckpoint = new AtomicBoolean(false);
    final StateMachineTask task00 = new StateMachineTask(taskId00, taskId00Partitions, true, stateManager) {

        @Override
        public void postCommit(final boolean enforceCheckpoint) {
            if (enforceCheckpoint) {
                enforcedCheckpoint.set(true);
            }
            super.postCommit(enforceCheckpoint);
        }
    };
    // `handleAssignment`
    expectRestoreToBeCompleted(consumer, changeLogReader);
    expect(activeTaskCreator.createTasks(anyObject(), eq(taskId00Assignment))).andStubReturn(singletonList(task00));
    topologyBuilder.addSubscribedTopicsFromAssignment(anyObject(), anyString());
    expectLastCall().anyTimes();
    expect(consumer.assignment()).andReturn(taskId00Partitions);
    replay(activeTaskCreator, topologyBuilder, consumer, changeLogReader);
    taskManager.handleAssignment(taskId00Assignment, emptyMap());
    assertThat(taskManager.tryToCompleteRestoration(time.milliseconds(), tp -> assertThat(tp, is(empty()))), is(true));
    assertThat(task00.state(), is(Task.State.RUNNING));
    task00.setChangelogOffsets(singletonMap(t1p0, 0L));
    taskManager.handleCorruption(singleton(taskId00));
    assertThat(task00.commitPrepared, is(true));
    assertThat(task00.state(), is(Task.State.CREATED));
    assertThat(task00.partitionsForOffsetReset, equalTo(taskId00Partitions));
    assertThat(enforcedCheckpoint.get(), is(true));
    assertThat(taskManager.activeTaskMap(), is(singletonMap(taskId00, task00)));
    assertThat(taskManager.standbyTaskMap(), Matchers.anEmptyMap());
    verify(stateManager);
    verify(consumer);
}
Also used : MockTime(org.apache.kafka.common.utils.MockTime) Arrays(java.util.Arrays) CoreMatchers.hasItem(org.hamcrest.CoreMatchers.hasItem) ConsumerGroupMetadata(org.apache.kafka.clients.consumer.ConsumerGroupMetadata) Mock(org.easymock.Mock) IsEqual.equalTo(org.hamcrest.core.IsEqual.equalTo) KafkaException(org.apache.kafka.common.KafkaException) StreamsException(org.apache.kafka.streams.errors.StreamsException) Collections.singletonList(java.util.Collections.singletonList) Utils.mkMap(org.apache.kafka.common.utils.Utils.mkMap) DeleteRecordsResult(org.apache.kafka.clients.admin.DeleteRecordsResult) Collections.singleton(java.util.Collections.singleton) EasyMock.reset(org.easymock.EasyMock.reset) Arrays.asList(java.util.Arrays.asList) Map(java.util.Map) Metric(org.apache.kafka.common.Metric) MetricName(org.apache.kafka.common.MetricName) StreamsMetricsImpl(org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl) Assert.fail(org.junit.Assert.fail) EasyMock.eq(org.easymock.EasyMock.eq) Consumer(org.apache.kafka.clients.consumer.Consumer) TopicPartition(org.apache.kafka.common.TopicPartition) StreamsConfigUtils(org.apache.kafka.streams.internals.StreamsConfigUtils) Time(org.apache.kafka.common.utils.Time) Collections.emptyList(java.util.Collections.emptyList) State(org.apache.kafka.streams.processor.internals.Task.State) Collection(java.util.Collection) Utils.mkSet(org.apache.kafka.common.utils.Utils.mkSet) Set(java.util.Set) Measurable(org.apache.kafka.common.metrics.Measurable) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) List(java.util.List) Metrics(org.apache.kafka.common.metrics.Metrics) Utils.mkEntry(org.apache.kafka.common.utils.Utils.mkEntry) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) Assert.assertFalse(org.junit.Assert.assertFalse) OffsetAndMetadata(org.apache.kafka.clients.consumer.OffsetAndMetadata) Optional(java.util.Optional) KafkaMetric(org.apache.kafka.common.metrics.KafkaMetric) LockException(org.apache.kafka.streams.errors.LockException) Matchers.is(org.hamcrest.Matchers.is) MockType(org.easymock.MockType) StreamsConfig(org.apache.kafka.streams.StreamsConfig) TaskId(org.apache.kafka.streams.processor.TaskId) Assert.assertThrows(org.junit.Assert.assertThrows) RunWith(org.junit.runner.RunWith) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) EasyMock.mock(org.easymock.EasyMock.mock) HashMap(java.util.HashMap) Deque(java.util.Deque) AtomicReference(java.util.concurrent.atomic.AtomicReference) RecordsToDelete(org.apache.kafka.clients.admin.RecordsToDelete) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) TaskMigratedException(org.apache.kafka.streams.errors.TaskMigratedException) EasyMock.resetToStrict(org.easymock.EasyMock.resetToStrict) TaskDirectory(org.apache.kafka.streams.processor.internals.StateDirectory.TaskDirectory) Admin(org.apache.kafka.clients.admin.Admin) EasyMockRunner(org.easymock.EasyMockRunner) Collections.singletonMap(java.util.Collections.singletonMap) EasyMock.replay(org.easymock.EasyMock.replay) KafkaFutureImpl(org.apache.kafka.common.internals.KafkaFutureImpl) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) LinkedList(java.util.LinkedList) TaskCorruptedException(org.apache.kafka.streams.errors.TaskCorruptedException) Before(org.junit.Before) EasyMock.anyObject(org.easymock.EasyMock.anyObject) Collections.emptyMap(java.util.Collections.emptyMap) TimeoutException(org.apache.kafka.common.errors.TimeoutException) Matchers.empty(org.hamcrest.Matchers.empty) Collections.emptySet(java.util.Collections.emptySet) EasyMock.anyString(org.easymock.EasyMock.anyString) Matchers(org.hamcrest.Matchers) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) ProcessingMode(org.apache.kafka.streams.internals.StreamsConfigUtils.ProcessingMode) EasyMock(org.easymock.EasyMock) EasyMock.expect(org.easymock.EasyMock.expect) File(java.io.File) Utils.union(org.apache.kafka.common.utils.Utils.union) EasyMock.expectLastCall(org.easymock.EasyMock.expectLastCall) Rule(org.junit.Rule) Assert.assertNull(org.junit.Assert.assertNull) StateStore(org.apache.kafka.streams.processor.StateStore) LogCaptureAppender(org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender) CommitFailedException(org.apache.kafka.clients.consumer.CommitFailedException) OffsetCheckpoint(org.apache.kafka.streams.state.internals.OffsetCheckpoint) EasyMock.verify(org.easymock.EasyMock.verify) DummyStreamsConfig(org.apache.kafka.streams.processor.internals.testutil.DummyStreamsConfig) Collections(java.util.Collections) DeletedRecords(org.apache.kafka.clients.admin.DeletedRecords) TemporaryFolder(org.junit.rules.TemporaryFolder) Assert.assertEquals(org.junit.Assert.assertEquals) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Test(org.junit.Test)

Example 2 with EasyMock.expectLastCall

use of org.easymock.EasyMock.expectLastCall in project kafka by apache.

the class StoreChangelogReaderTest method shouldRequestCommittedOffsetsAndHandleTimeoutException.

@Test
public void shouldRequestCommittedOffsetsAndHandleTimeoutException() {
    final TaskId taskId = new TaskId(0, 0);
    final Task mockTask = mock(Task.class);
    if (type == ACTIVE) {
        mockTask.clearTaskTimeout();
    }
    mockTask.maybeInitTaskTimeoutOrThrow(anyLong(), anyObject());
    EasyMock.expectLastCall();
    EasyMock.expect(stateManager.changelogAsSource(tp)).andReturn(true).anyTimes();
    EasyMock.expect(storeMetadata.offset()).andReturn(5L).anyTimes();
    EasyMock.expect(stateManager.changelogOffsets()).andReturn(singletonMap(tp, 5L));
    EasyMock.expect(stateManager.taskId()).andReturn(taskId).anyTimes();
    EasyMock.replay(mockTask, stateManager, storeMetadata, store);
    final AtomicBoolean functionCalled = new AtomicBoolean(false);
    final MockConsumer<byte[], byte[]> consumer = new MockConsumer<byte[], byte[]>(OffsetResetStrategy.EARLIEST) {

        @Override
        public Map<TopicPartition, OffsetAndMetadata> committed(final Set<TopicPartition> partitions) {
            if (functionCalled.get()) {
                return partitions.stream().collect(Collectors.toMap(Function.identity(), partition -> new OffsetAndMetadata(10L)));
            } else {
                functionCalled.set(true);
                throw new TimeoutException("KABOOM!");
            }
        }
    };
    adminClient.updateEndOffsets(Collections.singletonMap(tp, 20L));
    final StoreChangelogReader changelogReader = new StoreChangelogReader(time, config, logContext, adminClient, consumer, callback);
    changelogReader.setMainConsumer(consumer);
    changelogReader.register(tp, stateManager);
    changelogReader.restore(Collections.singletonMap(taskId, mockTask));
    assertEquals(type == ACTIVE ? StoreChangelogReader.ChangelogState.REGISTERED : StoreChangelogReader.ChangelogState.RESTORING, changelogReader.changelogMetadata(tp).state());
    if (type == ACTIVE) {
        assertNull(changelogReader.changelogMetadata(tp).endOffset());
    } else {
        assertEquals(0L, (long) changelogReader.changelogMetadata(tp).endOffset());
    }
    assertTrue(functionCalled.get());
    verify(mockTask);
    resetToDefault(mockTask);
    if (type == ACTIVE) {
        mockTask.clearTaskTimeout();
        mockTask.clearTaskTimeout();
        expectLastCall();
    }
    replay(mockTask);
    changelogReader.restore(Collections.singletonMap(taskId, mockTask));
    assertEquals(StoreChangelogReader.ChangelogState.RESTORING, changelogReader.changelogMetadata(tp).state());
    assertEquals(type == ACTIVE ? 10L : 0L, (long) changelogReader.changelogMetadata(tp).endOffset());
    assertEquals(6L, consumer.position(tp));
    verify(mockTask);
}
Also used : MockTime(org.apache.kafka.common.utils.MockTime) MockConsumer(org.apache.kafka.clients.consumer.MockConsumer) Mock(org.easymock.Mock) KafkaException(org.apache.kafka.common.KafkaException) StreamsException(org.apache.kafka.streams.errors.StreamsException) OffsetResetStrategy(org.apache.kafka.clients.consumer.OffsetResetStrategy) ACTIVE(org.apache.kafka.streams.processor.internals.Task.TaskType.ACTIVE) ListOffsetsResult(org.apache.kafka.clients.admin.ListOffsetsResult) Utils.mkMap(org.apache.kafka.common.utils.Utils.mkMap) LogContext(org.apache.kafka.common.utils.LogContext) After(org.junit.After) Duration(java.time.Duration) Map(java.util.Map) Parameterized(org.junit.runners.Parameterized) EasyMockSupport(org.easymock.EasyMockSupport) TopicPartition(org.apache.kafka.common.TopicPartition) RESTORE_END(org.apache.kafka.test.MockStateRestoreListener.RESTORE_END) Utils.mkSet(org.apache.kafka.common.utils.Utils.mkSet) Set(java.util.Set) PartitionInfo(org.apache.kafka.common.PartitionInfo) Collectors(java.util.stream.Collectors) MockAdminClient(org.apache.kafka.clients.admin.MockAdminClient) EasyMock.resetToDefault(org.easymock.EasyMock.resetToDefault) Utils.mkEntry(org.apache.kafka.common.utils.Utils.mkEntry) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) STANDBY_UPDATING(org.apache.kafka.streams.processor.internals.StoreChangelogReader.ChangelogReaderState.STANDBY_UPDATING) Assert.assertFalse(org.junit.Assert.assertFalse) Matchers.equalTo(org.hamcrest.Matchers.equalTo) OffsetAndMetadata(org.apache.kafka.clients.consumer.OffsetAndMetadata) MockType(org.easymock.MockType) RESTORE_START(org.apache.kafka.test.MockStateRestoreListener.RESTORE_START) StreamsConfig(org.apache.kafka.streams.StreamsConfig) TaskId(org.apache.kafka.streams.processor.TaskId) Assert.assertThrows(org.junit.Assert.assertThrows) RunWith(org.junit.runner.RunWith) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Function(java.util.function.Function) Collections.singletonMap(java.util.Collections.singletonMap) EasyMock.replay(org.easymock.EasyMock.replay) EasyMockRule(org.easymock.EasyMockRule) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) STANDBY(org.apache.kafka.streams.processor.internals.Task.TaskType.STANDBY) RESTORE_BATCH(org.apache.kafka.test.MockStateRestoreListener.RESTORE_BATCH) ACTIVE_RESTORING(org.apache.kafka.streams.processor.internals.StoreChangelogReader.ChangelogReaderState.ACTIVE_RESTORING) Before(org.junit.Before) EasyMock.anyObject(org.easymock.EasyMock.anyObject) StateStoreMetadata(org.apache.kafka.streams.processor.internals.ProcessorStateManager.StateStoreMetadata) TimeoutException(org.apache.kafka.common.errors.TimeoutException) Properties(java.util.Properties) MockStateRestoreListener(org.apache.kafka.test.MockStateRestoreListener) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) EasyMock(org.easymock.EasyMock) OffsetSpec(org.apache.kafka.clients.admin.OffsetSpec) EasyMock.expectLastCall(org.easymock.EasyMock.expectLastCall) AtomicLong(java.util.concurrent.atomic.AtomicLong) Rule(org.junit.Rule) Matchers.hasItem(org.hamcrest.Matchers.hasItem) Assert.assertNull(org.junit.Assert.assertNull) StateStore(org.apache.kafka.streams.processor.StateStore) EasyMock.anyLong(org.easymock.EasyMock.anyLong) LogCaptureAppender(org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender) ListOffsetsOptions(org.apache.kafka.clients.admin.ListOffsetsOptions) StreamsTestUtils(org.apache.kafka.test.StreamsTestUtils) EasyMock.verify(org.easymock.EasyMock.verify) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) TaskId(org.apache.kafka.streams.processor.TaskId) Utils.mkSet(org.apache.kafka.common.utils.Utils.mkSet) Set(java.util.Set) TopicPartition(org.apache.kafka.common.TopicPartition) OffsetAndMetadata(org.apache.kafka.clients.consumer.OffsetAndMetadata) MockConsumer(org.apache.kafka.clients.consumer.MockConsumer) TimeoutException(org.apache.kafka.common.errors.TimeoutException) Test(org.junit.Test)

Aggregations

Collections (java.util.Collections)2 Collections.singletonMap (java.util.Collections.singletonMap)2 Map (java.util.Map)2 Set (java.util.Set)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 Collectors (java.util.stream.Collectors)2 ConsumerRecord (org.apache.kafka.clients.consumer.ConsumerRecord)2 OffsetAndMetadata (org.apache.kafka.clients.consumer.OffsetAndMetadata)2 KafkaException (org.apache.kafka.common.KafkaException)2 TopicPartition (org.apache.kafka.common.TopicPartition)2 TimeoutException (org.apache.kafka.common.errors.TimeoutException)2 File (java.io.File)1 Duration (java.time.Duration)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Arrays.asList (java.util.Arrays.asList)1 Collection (java.util.Collection)1 Collections.emptyList (java.util.Collections.emptyList)1 Collections.emptyMap (java.util.Collections.emptyMap)1 Collections.emptySet (java.util.Collections.emptySet)1