Search in sources :

Example 56 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestTransactionalStateTaskRestoreManager method testGetCurrentChangelogOffsets.

@Test
public void testGetCurrentChangelogOffsets() {
    // test gets metadata for all and only task store changelog SSPs
    // test all changelogs have same partition
    // test does not change returned ssp metadata
    TaskModel mockTaskModel = mock(TaskModel.class);
    when(mockTaskModel.getTaskName()).thenReturn(new TaskName("Partition 0"));
    Partition taskChangelogPartition = new Partition(0);
    when(mockTaskModel.getChangelogPartition()).thenReturn(taskChangelogPartition);
    String store1Name = "store1";
    String changelog1SystemName = "system1";
    String changelog1StreamName = "store1Changelog";
    String store2Name = "store2";
    String changelog2SystemName = "system2";
    String changelog2StreamName = "store2Changelog";
    SystemStream changelog1SystemStream = new SystemStream(changelog1SystemName, changelog1StreamName);
    SystemStream changelog2SystemStream = new SystemStream(changelog2SystemName, changelog2StreamName);
    Map<String, SystemStream> mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream, store2Name, changelog2SystemStream);
    SSPMetadataCache mockSSPMetadataCache = mock(SSPMetadataCache.class);
    SystemStreamPartition changelog1SSP = new SystemStreamPartition(changelog1SystemStream, taskChangelogPartition);
    SystemStreamPartitionMetadata changelog1SSPMetadata = new SystemStreamPartitionMetadata("0", "1", "2");
    when(mockSSPMetadataCache.getMetadata(eq(changelog1SSP))).thenReturn(changelog1SSPMetadata);
    SystemStreamPartition changelog2SSP = new SystemStreamPartition(changelog2SystemStream, taskChangelogPartition);
    SystemStreamPartitionMetadata changelog2SSPMetadata = new SystemStreamPartitionMetadata("1", "2", "3");
    when(mockSSPMetadataCache.getMetadata(eq(changelog2SSP))).thenReturn(changelog2SSPMetadata);
    Map<SystemStreamPartition, SystemStreamPartitionMetadata> currentChangelogOffsets = TransactionalStateTaskRestoreManager.getCurrentChangelogOffsets(mockTaskModel, mockStoreChangelogs, mockSSPMetadataCache);
    verify(mockSSPMetadataCache, times(1)).getMetadata(changelog1SSP);
    verify(mockSSPMetadataCache, times(1)).getMetadata(changelog2SSP);
    verifyNoMoreInteractions(mockSSPMetadataCache);
    assertEquals(2, currentChangelogOffsets.size());
    assertEquals(changelog1SSPMetadata, currentChangelogOffsets.get(changelog1SSP));
    assertEquals(changelog2SSPMetadata, currentChangelogOffsets.get(changelog2SSP));
}
Also used : SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Partition(org.apache.samza.Partition) TaskName(org.apache.samza.container.TaskName) SystemStream(org.apache.samza.system.SystemStream) SSPMetadataCache(org.apache.samza.system.SSPMetadataCache) Matchers.anyString(org.mockito.Matchers.anyString) SystemStreamPartitionMetadata(org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata) TaskModel(org.apache.samza.job.model.TaskModel) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Test(org.junit.Test)

Example 57 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestTransactionalStateTaskRestoreManager method testStoreDeletedWhenCleanDirsFlagSet.

@Test
public void testStoreDeletedWhenCleanDirsFlagSet() {
    TaskModel mockTaskModel = mock(TaskModel.class);
    TaskName taskName = new TaskName("Partition 0");
    when(mockTaskModel.getTaskName()).thenReturn(taskName);
    Partition taskChangelogPartition = new Partition(0);
    when(mockTaskModel.getChangelogPartition()).thenReturn(taskChangelogPartition);
    when(mockTaskModel.getTaskMode()).thenReturn(TaskMode.Active);
    String store1Name = "store1";
    StorageEngine store1Engine = mock(StorageEngine.class);
    StoreProperties mockStore1Properties = mock(StoreProperties.class);
    when(store1Engine.getStoreProperties()).thenReturn(mockStore1Properties);
    when(mockStore1Properties.isLoggedStore()).thenReturn(true);
    when(mockStore1Properties.isPersistedToDisk()).thenReturn(true);
    Map<String, StorageEngine> mockStoreEngines = ImmutableMap.of(store1Name, store1Engine);
    String changelog1SystemName = "system1";
    String changelog1StreamName = "store1Changelog";
    SystemStream changelog1SystemStream = new SystemStream(changelog1SystemName, changelog1StreamName);
    SystemStreamPartition changelog1SSP = new SystemStreamPartition(changelog1SystemStream, taskChangelogPartition);
    SystemStreamPartitionMetadata changelog1SSPMetadata = new SystemStreamPartitionMetadata("0", "10", "11");
    Map<String, SystemStream> mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream);
    String changelog1CheckpointedOffset = "5";
    CheckpointId checkpointId = CheckpointId.create();
    KafkaStateCheckpointMarker kafkaStateCheckpointMarker = new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset);
    ImmutableMap<String, KafkaStateCheckpointMarker> mockCheckpointedChangelogOffset = ImmutableMap.of(store1Name, kafkaStateCheckpointMarker);
    Map<SystemStreamPartition, SystemStreamPartitionMetadata> mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata);
    SystemAdmins mockSystemAdmins = mock(SystemAdmins.class);
    SystemAdmin mockSystemAdmin = mock(SystemAdmin.class);
    when(mockSystemAdmins.getSystemAdmin(changelog1SSP.getSystem())).thenReturn(mockSystemAdmin);
    StorageManagerUtil mockStorageManagerUtil = mock(StorageManagerUtil.class);
    File mockLoggedStoreBaseDir = mock(File.class);
    File mockNonLoggedStoreBaseDir = mock(File.class);
    // set the clean.on.container.start config set on the store
    Config mockConfig = new MapConfig(Collections.singletonMap("stores.store1.clean.on.container.start", "true"));
    Clock mockClock = mock(Clock.class);
    Mockito.when(mockSystemAdmin.offsetComparator(anyString(), anyString())).thenAnswer((Answer<Integer>) invocation -> {
        String offset1 = (String) invocation.getArguments()[0];
        String offset2 = (String) invocation.getArguments()[1];
        return Long.valueOf(offset1).compareTo(Long.valueOf(offset2));
    });
    File dummyCurrentDir = new File("currentDir");
    File dummyCheckpointDir = new File("checkpointDir1");
    when(mockStorageManagerUtil.getTaskStoreDir(mockLoggedStoreBaseDir, store1Name, taskName, TaskMode.Active)).thenReturn(dummyCurrentDir);
    when(mockStorageManagerUtil.getTaskStoreCheckpointDirs(mockLoggedStoreBaseDir, store1Name, taskName, TaskMode.Active)).thenReturn(ImmutableList.of(dummyCheckpointDir));
    StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions(mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock);
    // ensure that current and checkpoint directories are marked for deletion
    assertEquals(2, storeActions.storeDirsToDelete.size());
    assertTrue(storeActions.storeDirsToDelete.containsValue(dummyCheckpointDir));
    assertTrue(storeActions.storeDirsToDelete.containsValue(dummyCurrentDir));
    // ensure that we restore from the oldest changelog offset to checkpointed changelog offset
    assertEquals("0", storeActions.storesToRestore.get(store1Name).startingOffset);
    assertEquals(changelog1CheckpointedOffset, storeActions.storesToRestore.get(store1Name).endingOffset);
}
Also used : ArrayListMultimap(com.google.common.collect.ArrayListMultimap) ListMultimap(com.google.common.collect.ListMultimap) SSPMetadataCache(org.apache.samza.system.SSPMetadataCache) HashMap(java.util.HashMap) TaskModel(org.apache.samza.job.model.TaskModel) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Matchers.anyString(org.mockito.Matchers.anyString) FileUtil(org.apache.samza.util.FileUtil) Answer(org.mockito.stubbing.Answer) ImmutableList(com.google.common.collect.ImmutableList) SystemConsumer(org.apache.samza.system.SystemConsumer) Mockito.verifyNoMoreInteractions(org.mockito.Mockito.verifyNoMoreInteractions) SystemStream(org.apache.samza.system.SystemStream) Matchers.eq(org.mockito.Matchers.eq) Map(java.util.Map) Assert.fail(org.junit.Assert.fail) Path(java.nio.file.Path) MapConfig(org.apache.samza.config.MapConfig) ImmutableSet(com.google.common.collect.ImmutableSet) TaskName(org.apache.samza.container.TaskName) ImmutableMap(com.google.common.collect.ImmutableMap) TaskConfig(org.apache.samza.config.TaskConfig) Assert.assertNotNull(org.junit.Assert.assertNotNull) Partition(org.apache.samza.Partition) Set(java.util.Set) Assert.assertTrue(org.junit.Assert.assertTrue) Clock(org.apache.samza.util.Clock) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) Mockito.when(org.mockito.Mockito.when) StoreActions(org.apache.samza.storage.TransactionalStateTaskRestoreManager.StoreActions) File(java.io.File) CheckpointId(org.apache.samza.checkpoint.CheckpointId) Mockito.verify(org.mockito.Mockito.verify) Matchers.any(org.mockito.Matchers.any) Mockito(org.mockito.Mockito) TaskMode(org.apache.samza.job.model.TaskMode) Mockito.never(org.mockito.Mockito.never) Assert.assertNull(org.junit.Assert.assertNull) RestoreOffsets(org.apache.samza.storage.TransactionalStateTaskRestoreManager.RestoreOffsets) SystemAdmin(org.apache.samza.system.SystemAdmin) SystemStreamPartitionMetadata(org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata) KafkaStateCheckpointMarker(org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker) Config(org.apache.samza.config.Config) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) SystemAdmins(org.apache.samza.system.SystemAdmins) Mockito.mock(org.mockito.Mockito.mock) MapConfig(org.apache.samza.config.MapConfig) TaskConfig(org.apache.samza.config.TaskConfig) Config(org.apache.samza.config.Config) Matchers.anyString(org.mockito.Matchers.anyString) SystemStreamPartitionMetadata(org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata) Clock(org.apache.samza.util.Clock) MapConfig(org.apache.samza.config.MapConfig) SystemAdmins(org.apache.samza.system.SystemAdmins) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Partition(org.apache.samza.Partition) SystemStream(org.apache.samza.system.SystemStream) StoreActions(org.apache.samza.storage.TransactionalStateTaskRestoreManager.StoreActions) TaskName(org.apache.samza.container.TaskName) CheckpointId(org.apache.samza.checkpoint.CheckpointId) SystemAdmin(org.apache.samza.system.SystemAdmin) File(java.io.File) TaskModel(org.apache.samza.job.model.TaskModel) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) KafkaStateCheckpointMarker(org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker) Test(org.junit.Test)

Example 58 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestTransactionalStateTaskRestoreManager method testRegisterStartingOffsetsThrowsIfStartingGreaterThanEnding.

@Test(expected = IllegalStateException.class)
public void testRegisterStartingOffsetsThrowsIfStartingGreaterThanEnding() {
    TaskModel mockTaskModel = mock(TaskModel.class);
    TaskName taskName = new TaskName("Partition 0");
    when(mockTaskModel.getTaskName()).thenReturn(taskName);
    Partition taskChangelogPartition = new Partition(0);
    when(mockTaskModel.getChangelogPartition()).thenReturn(taskChangelogPartition);
    // tests starting offset > ending offset
    Map<String, RestoreOffsets> mockRestoreOffsets = ImmutableMap.of("store1", new RestoreOffsets("5", "0"));
    StoreActions mockStoreActions = new StoreActions(ImmutableMap.of(), ArrayListMultimap.create(), mockRestoreOffsets);
    String store1Name = "store1";
    String changelogSystemName = "system";
    String changelog1StreamName = "store1Changelog";
    SystemStream changelog1SystemStream = new SystemStream(changelogSystemName, changelog1StreamName);
    Map<String, SystemStream> mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream);
    SystemStreamPartition changelog1SSP = new SystemStreamPartition(changelog1SystemStream, taskChangelogPartition);
    SystemStreamPartitionMetadata changelog1SSPMetadata = new SystemStreamPartitionMetadata("0", "10", "11");
    Map<SystemStreamPartition, SystemStreamPartitionMetadata> mockCurrentChangelogSSPMetadata = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata);
    SystemAdmins mockSystemAdmins = mock(SystemAdmins.class);
    SystemAdmin mockSystemAdmin = mock(SystemAdmin.class);
    when(mockSystemAdmins.getSystemAdmin(eq(changelogSystemName))).thenReturn(mockSystemAdmin);
    Mockito.when(mockSystemAdmin.offsetComparator(anyString(), anyString())).thenAnswer((Answer<Integer>) invocation -> {
        String offset1 = (String) invocation.getArguments()[0];
        String offset2 = (String) invocation.getArguments()[1];
        return Long.valueOf(offset1).compareTo(Long.valueOf(offset2));
    });
    Mockito.when(mockSystemAdmin.getOffsetsAfter(any())).thenAnswer((Answer<Map<SystemStreamPartition, String>>) invocation -> {
        Map<SystemStreamPartition, String> offsets = (Map<SystemStreamPartition, String>) invocation.getArguments()[0];
        Map<SystemStreamPartition, String> nextOffsets = new HashMap<>();
        offsets.forEach((ssp, offset) -> nextOffsets.put(ssp, Long.toString(Long.valueOf(offset) + 1)));
        return nextOffsets;
    });
    SystemConsumer mockSystemConsumer = mock(SystemConsumer.class);
    Map<String, SystemConsumer> mockStoreConsumers = ImmutableMap.of("store1", mockSystemConsumer);
    TransactionalStateTaskRestoreManager.registerStartingOffsets(mockTaskModel, mockStoreActions, mockStoreChangelogs, mockSystemAdmins, mockStoreConsumers, mockCurrentChangelogSSPMetadata);
    fail("Should have thrown an exception since starting offset > ending offset");
}
Also used : ArrayListMultimap(com.google.common.collect.ArrayListMultimap) ListMultimap(com.google.common.collect.ListMultimap) SSPMetadataCache(org.apache.samza.system.SSPMetadataCache) HashMap(java.util.HashMap) TaskModel(org.apache.samza.job.model.TaskModel) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Matchers.anyString(org.mockito.Matchers.anyString) FileUtil(org.apache.samza.util.FileUtil) Answer(org.mockito.stubbing.Answer) ImmutableList(com.google.common.collect.ImmutableList) SystemConsumer(org.apache.samza.system.SystemConsumer) Mockito.verifyNoMoreInteractions(org.mockito.Mockito.verifyNoMoreInteractions) SystemStream(org.apache.samza.system.SystemStream) Matchers.eq(org.mockito.Matchers.eq) Map(java.util.Map) Assert.fail(org.junit.Assert.fail) Path(java.nio.file.Path) MapConfig(org.apache.samza.config.MapConfig) ImmutableSet(com.google.common.collect.ImmutableSet) TaskName(org.apache.samza.container.TaskName) ImmutableMap(com.google.common.collect.ImmutableMap) TaskConfig(org.apache.samza.config.TaskConfig) Assert.assertNotNull(org.junit.Assert.assertNotNull) Partition(org.apache.samza.Partition) Set(java.util.Set) Assert.assertTrue(org.junit.Assert.assertTrue) Clock(org.apache.samza.util.Clock) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) Mockito.when(org.mockito.Mockito.when) StoreActions(org.apache.samza.storage.TransactionalStateTaskRestoreManager.StoreActions) File(java.io.File) CheckpointId(org.apache.samza.checkpoint.CheckpointId) Mockito.verify(org.mockito.Mockito.verify) Matchers.any(org.mockito.Matchers.any) Mockito(org.mockito.Mockito) TaskMode(org.apache.samza.job.model.TaskMode) Mockito.never(org.mockito.Mockito.never) Assert.assertNull(org.junit.Assert.assertNull) RestoreOffsets(org.apache.samza.storage.TransactionalStateTaskRestoreManager.RestoreOffsets) SystemAdmin(org.apache.samza.system.SystemAdmin) SystemStreamPartitionMetadata(org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata) KafkaStateCheckpointMarker(org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker) Config(org.apache.samza.config.Config) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) SystemAdmins(org.apache.samza.system.SystemAdmins) Mockito.mock(org.mockito.Mockito.mock) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Partition(org.apache.samza.Partition) SystemConsumer(org.apache.samza.system.SystemConsumer) StoreActions(org.apache.samza.storage.TransactionalStateTaskRestoreManager.StoreActions) SystemStream(org.apache.samza.system.SystemStream) Matchers.anyString(org.mockito.Matchers.anyString) SystemStreamPartitionMetadata(org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata) RestoreOffsets(org.apache.samza.storage.TransactionalStateTaskRestoreManager.RestoreOffsets) TaskName(org.apache.samza.container.TaskName) SystemAdmin(org.apache.samza.system.SystemAdmin) SystemAdmins(org.apache.samza.system.SystemAdmins) HashMap(java.util.HashMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) TaskModel(org.apache.samza.job.model.TaskModel) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Test(org.junit.Test)

Example 59 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestTransactionalStateTaskRestoreManager method testGetStoreActionsForLoggedPersistentStore_FullRestoreIfEqualCheckpointedOldestAndNewestOffset.

/**
 * This is the case when the changelog topic is empty but not new. E.g., if wrote 100 messages,
 * then deleted 100 messages, and after compaction oldest == newest == checkpointed. In this case
 * full restore does not do anything since there is nothing to restore or trim, but the code path will
 * leave us in a consistent state with the appropriate stores deleted and retained.
 */
@Test
public void testGetStoreActionsForLoggedPersistentStore_FullRestoreIfEqualCheckpointedOldestAndNewestOffset() {
    TaskModel mockTaskModel = mock(TaskModel.class);
    TaskName taskName = new TaskName("Partition 0");
    when(mockTaskModel.getTaskName()).thenReturn(taskName);
    Partition taskChangelogPartition = new Partition(0);
    when(mockTaskModel.getChangelogPartition()).thenReturn(taskChangelogPartition);
    String store1Name = "store1";
    StorageEngine store1Engine = mock(StorageEngine.class);
    StoreProperties mockStore1Properties = mock(StoreProperties.class);
    when(store1Engine.getStoreProperties()).thenReturn(mockStore1Properties);
    when(mockStore1Properties.isLoggedStore()).thenReturn(true);
    when(mockStore1Properties.isPersistedToDisk()).thenReturn(true);
    Map<String, StorageEngine> mockStoreEngines = ImmutableMap.of(store1Name, store1Engine);
    String changelog1SystemName = "system1";
    String changelog1StreamName = "store1Changelog";
    SystemStream changelog1SystemStream = new SystemStream(changelog1SystemName, changelog1StreamName);
    SystemStreamPartition changelog1SSP = new SystemStreamPartition(changelog1SystemStream, taskChangelogPartition);
    SystemStreamPartitionMetadata changelog1SSPMetadata = new SystemStreamPartitionMetadata("5", "5", "6");
    Map<String, SystemStream> mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream);
    String changelog1CheckpointedOffset = "5";
    CheckpointId checkpointId = CheckpointId.create();
    KafkaStateCheckpointMarker kafkaStateCheckpointMarker = new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset);
    Map<String, KafkaStateCheckpointMarker> mockCheckpointedChangelogOffset = new HashMap<String, KafkaStateCheckpointMarker>() {

        {
            put(store1Name, kafkaStateCheckpointMarker);
        }
    };
    Map<SystemStreamPartition, SystemStreamPartitionMetadata> mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata);
    SystemAdmins mockSystemAdmins = mock(SystemAdmins.class);
    SystemAdmin mockSystemAdmin = mock(SystemAdmin.class);
    when(mockSystemAdmins.getSystemAdmin(changelog1SSP.getSystem())).thenReturn(mockSystemAdmin);
    StorageManagerUtil mockStorageManagerUtil = mock(StorageManagerUtil.class);
    File mockLoggedStoreBaseDir = mock(File.class);
    File mockNonLoggedStoreBaseDir = mock(File.class);
    HashMap<String, String> configMap = new HashMap<>();
    // should not matter
    configMap.put(TaskConfig.TRANSACTIONAL_STATE_RETAIN_EXISTING_STATE, "true");
    Config mockConfig = new MapConfig(configMap);
    Clock mockClock = mock(Clock.class);
    File mockCurrentStoreDir = mock(File.class);
    File mockStoreCheckpointDir = mock(File.class);
    String checkpointDirLocalOffset = "5";
    when(mockStorageManagerUtil.getTaskStoreDir(eq(mockLoggedStoreBaseDir), eq(store1Name), eq(taskName), any())).thenReturn(mockCurrentStoreDir);
    when(mockStorageManagerUtil.getTaskStoreCheckpointDirs(eq(mockLoggedStoreBaseDir), eq(store1Name), eq(taskName), any())).thenReturn(ImmutableList.of(mockStoreCheckpointDir));
    when(mockStorageManagerUtil.isLoggedStoreValid(eq(store1Name), eq(mockStoreCheckpointDir), any(), eq(mockStoreChangelogs), eq(mockTaskModel), any(), eq(mockStoreEngines))).thenReturn(true);
    Set<SystemStreamPartition> mockChangelogSSPs = ImmutableSet.of(changelog1SSP);
    when(mockStorageManagerUtil.readOffsetFile(eq(mockStoreCheckpointDir), eq(mockChangelogSSPs), eq(false))).thenReturn(ImmutableMap.of(changelog1SSP, checkpointDirLocalOffset));
    Mockito.when(mockSystemAdmin.offsetComparator(anyString(), anyString())).thenAnswer((Answer<Integer>) invocation -> {
        String offset1 = (String) invocation.getArguments()[0];
        String offset2 = (String) invocation.getArguments()[1];
        if (offset1 == null || offset2 == null) {
            return -1;
        }
        return Long.valueOf(offset1).compareTo(Long.valueOf(offset2));
    });
    StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions(mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock);
    // ensure that current and old checkpoint dirs are marked for deletion
    assertEquals(1, storeActions.storeDirsToDelete.get(store1Name).size());
    assertTrue(storeActions.storeDirsToDelete.get(store1Name).contains(mockCurrentStoreDir));
    // ensure that latest checkpoint dir is retained
    assertEquals(mockStoreCheckpointDir, storeActions.storeDirsToRetain.get(store1Name));
    // ensure that we do a full restore (on the empty topic)
    assertEquals("5", storeActions.storesToRestore.get(store1Name).startingOffset);
    assertEquals("5", storeActions.storesToRestore.get(store1Name).endingOffset);
}
Also used : ArrayListMultimap(com.google.common.collect.ArrayListMultimap) ListMultimap(com.google.common.collect.ListMultimap) SSPMetadataCache(org.apache.samza.system.SSPMetadataCache) HashMap(java.util.HashMap) TaskModel(org.apache.samza.job.model.TaskModel) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Matchers.anyString(org.mockito.Matchers.anyString) FileUtil(org.apache.samza.util.FileUtil) Answer(org.mockito.stubbing.Answer) ImmutableList(com.google.common.collect.ImmutableList) SystemConsumer(org.apache.samza.system.SystemConsumer) Mockito.verifyNoMoreInteractions(org.mockito.Mockito.verifyNoMoreInteractions) SystemStream(org.apache.samza.system.SystemStream) Matchers.eq(org.mockito.Matchers.eq) Map(java.util.Map) Assert.fail(org.junit.Assert.fail) Path(java.nio.file.Path) MapConfig(org.apache.samza.config.MapConfig) ImmutableSet(com.google.common.collect.ImmutableSet) TaskName(org.apache.samza.container.TaskName) ImmutableMap(com.google.common.collect.ImmutableMap) TaskConfig(org.apache.samza.config.TaskConfig) Assert.assertNotNull(org.junit.Assert.assertNotNull) Partition(org.apache.samza.Partition) Set(java.util.Set) Assert.assertTrue(org.junit.Assert.assertTrue) Clock(org.apache.samza.util.Clock) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) Mockito.when(org.mockito.Mockito.when) StoreActions(org.apache.samza.storage.TransactionalStateTaskRestoreManager.StoreActions) File(java.io.File) CheckpointId(org.apache.samza.checkpoint.CheckpointId) Mockito.verify(org.mockito.Mockito.verify) Matchers.any(org.mockito.Matchers.any) Mockito(org.mockito.Mockito) TaskMode(org.apache.samza.job.model.TaskMode) Mockito.never(org.mockito.Mockito.never) Assert.assertNull(org.junit.Assert.assertNull) RestoreOffsets(org.apache.samza.storage.TransactionalStateTaskRestoreManager.RestoreOffsets) SystemAdmin(org.apache.samza.system.SystemAdmin) SystemStreamPartitionMetadata(org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata) KafkaStateCheckpointMarker(org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker) Config(org.apache.samza.config.Config) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) SystemAdmins(org.apache.samza.system.SystemAdmins) Mockito.mock(org.mockito.Mockito.mock) HashMap(java.util.HashMap) MapConfig(org.apache.samza.config.MapConfig) TaskConfig(org.apache.samza.config.TaskConfig) Config(org.apache.samza.config.Config) Matchers.anyString(org.mockito.Matchers.anyString) SystemStreamPartitionMetadata(org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata) Clock(org.apache.samza.util.Clock) MapConfig(org.apache.samza.config.MapConfig) SystemAdmins(org.apache.samza.system.SystemAdmins) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Partition(org.apache.samza.Partition) SystemStream(org.apache.samza.system.SystemStream) StoreActions(org.apache.samza.storage.TransactionalStateTaskRestoreManager.StoreActions) TaskName(org.apache.samza.container.TaskName) CheckpointId(org.apache.samza.checkpoint.CheckpointId) SystemAdmin(org.apache.samza.system.SystemAdmin) File(java.io.File) TaskModel(org.apache.samza.job.model.TaskModel) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) KafkaStateCheckpointMarker(org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker) Test(org.junit.Test)

Example 60 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestStartpointManager method testFanOutWithStartpointResolutions.

@Test
public void testFanOutWithStartpointResolutions() throws IOException {
    SystemStreamPartition sspBroadcast = new SystemStreamPartition("mockSystem1", "mockStream1", new Partition(2));
    SystemStreamPartition sspSingle = new SystemStreamPartition("mockSystem2", "mockStream2", new Partition(3));
    List<TaskName> tasks = ImmutableList.of(new TaskName("t0"), new TaskName("t1"), new TaskName("t2"), new TaskName("t3"), new TaskName("t4"));
    TaskName taskWithNonBroadcast = tasks.get(1);
    TaskName taskBroadcastInPast = tasks.get(2);
    TaskName taskBroadcastInFuture = tasks.get(3);
    Map<TaskName, Set<SystemStreamPartition>> taskToSSPs = tasks.stream().collect(Collectors.toMap(task -> task, task -> task.equals(taskWithNonBroadcast) ? ImmutableSet.of(sspBroadcast, sspSingle) : ImmutableSet.of(sspBroadcast)));
    Instant now = Instant.now();
    StartpointMock startpointPast = new StartpointMock(now.minusMillis(10000L).toEpochMilli());
    StartpointMock startpointPresent = new StartpointMock(now.toEpochMilli());
    StartpointMock startpointFuture = new StartpointMock(now.plusMillis(10000L).toEpochMilli());
    startpointManager.getObjectMapper().registerSubtypes(StartpointMock.class);
    startpointManager.writeStartpoint(sspSingle, startpointPast);
    startpointManager.writeStartpoint(sspSingle, startpointPresent);
    startpointManager.writeStartpoint(sspBroadcast, startpointPresent);
    startpointManager.writeStartpoint(sspBroadcast, taskBroadcastInPast, startpointPast);
    startpointManager.writeStartpoint(sspBroadcast, taskBroadcastInFuture, startpointFuture);
    Map<TaskName, Map<SystemStreamPartition, Startpoint>> fannedOut = startpointManager.fanOut(taskToSSPs);
    Assert.assertEquals(tasks.size(), fannedOut.size());
    Assert.assertTrue(fannedOut.keySet().containsAll(tasks));
    Assert.assertFalse("Should be deleted after fan out", startpointManager.readStartpoint(sspBroadcast).isPresent());
    for (TaskName taskName : fannedOut.keySet()) {
        Assert.assertFalse("Should be deleted after fan out for task: " + taskName.getTaskName(), startpointManager.readStartpoint(sspBroadcast, taskName).isPresent());
    }
    Assert.assertFalse("Should be deleted after fan out", startpointManager.readStartpoint(sspSingle).isPresent());
    for (TaskName taskName : tasks) {
        Map<SystemStreamPartition, Startpoint> fanOutForTask = startpointManager.getFanOutForTask(taskName);
        if (taskName.equals(taskWithNonBroadcast)) {
            Assert.assertEquals(startpointPresent, fanOutForTask.get(sspSingle));
            Assert.assertEquals(startpointPresent, fanOutForTask.get(sspBroadcast));
        } else if (taskName.equals(taskBroadcastInPast)) {
            Assert.assertNull(fanOutForTask.get(sspSingle));
            Assert.assertEquals(startpointPresent, fanOutForTask.get(sspBroadcast));
        } else if (taskName.equals(taskBroadcastInFuture)) {
            Assert.assertNull(fanOutForTask.get(sspSingle));
            Assert.assertEquals(startpointFuture, fanOutForTask.get(sspBroadcast));
        } else {
            Assert.assertNull(fanOutForTask.get(sspSingle));
            Assert.assertEquals(startpointPresent, fanOutForTask.get(sspBroadcast));
        }
        startpointManager.removeFanOutForTask(taskName);
        Assert.assertTrue(startpointManager.getFanOutForTask(taskName).isEmpty());
    }
}
Also used : NamespaceAwareCoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.NamespaceAwareCoordinatorStreamStore) CoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore) ImmutableSet(com.google.common.collect.ImmutableSet) TaskName(org.apache.samza.container.TaskName) ImmutableMap(com.google.common.collect.ImmutableMap) Partition(org.apache.samza.Partition) Set(java.util.Set) IOException(java.io.IOException) HashMap(java.util.HashMap) Test(org.junit.Test) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) Map(java.util.Map) CoordinatorStreamStoreTestUtil(org.apache.samza.coordinator.metadatastore.CoordinatorStreamStoreTestUtil) After(org.junit.After) Config(org.apache.samza.config.Config) Assert(org.junit.Assert) MapConfig(org.apache.samza.config.MapConfig) Before(org.junit.Before) Partition(org.apache.samza.Partition) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) ImmutableSet(com.google.common.collect.ImmutableSet) Set(java.util.Set) Instant(java.time.Instant) TaskName(org.apache.samza.container.TaskName) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap) Map(java.util.Map) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Test(org.junit.Test)

Aggregations

TaskName (org.apache.samza.container.TaskName)212 HashMap (java.util.HashMap)136 Test (org.junit.Test)133 SystemStreamPartition (org.apache.samza.system.SystemStreamPartition)117 Partition (org.apache.samza.Partition)106 MapConfig (org.apache.samza.config.MapConfig)92 TaskModel (org.apache.samza.job.model.TaskModel)90 Map (java.util.Map)75 Set (java.util.Set)73 Config (org.apache.samza.config.Config)71 ContainerModel (org.apache.samza.job.model.ContainerModel)63 ImmutableMap (com.google.common.collect.ImmutableMap)53 File (java.io.File)53 SystemStream (org.apache.samza.system.SystemStream)52 ImmutableSet (com.google.common.collect.ImmutableSet)50 TaskMode (org.apache.samza.job.model.TaskMode)46 TaskConfig (org.apache.samza.config.TaskConfig)43 ImmutableList (com.google.common.collect.ImmutableList)42 Collections (java.util.Collections)41 CheckpointId (org.apache.samza.checkpoint.CheckpointId)41