Search in sources :

Example 11 with SystemAdmins

use of org.apache.samza.system.SystemAdmins in project samza by apache.

the class TestTransactionalStateTaskRestoreManager method testRegisterStartingOffsets.

@Test
public void testRegisterStartingOffsets() {
    TaskModel mockTaskModel = mock(TaskModel.class);
    TaskName taskName = new TaskName("Partition 0");
    when(mockTaskModel.getTaskName()).thenReturn(taskName);
    Partition taskChangelogPartition = new Partition(0);
    when(mockTaskModel.getChangelogPartition()).thenReturn(taskChangelogPartition);
    String store1Name = "store1";
    String changelogSystemName = "system";
    String changelog1StreamName = "store1Changelog";
    String store2Name = "store2";
    String changelog2StreamName = "store2Changelog";
    String store3Name = "store3";
    String changelog3StreamName = "store3Changelog";
    String store4Name = "store4";
    String changelog4StreamName = "store4Changelog";
    // tests restore for store 1 and store 2 but not store 3
    Map<String, RestoreOffsets> mockRestoreOffsets = ImmutableMap.of(// tests starting offset == oldest (i.e. restore is inclusive)
    store1Name, // tests starting offset == oldest (i.e. restore is inclusive)
    new RestoreOffsets("0", "5"), // tests starting offset != oldest (i.e. restore from next offset)
    store2Name, // tests starting offset != oldest (i.e. restore from next offset)
    new RestoreOffsets("15", "20"), store4Name, // tests that null ending offsets are OK (should trim)
    new RestoreOffsets("31", null));
    StoreActions mockStoreActions = new StoreActions(ImmutableMap.of(), ArrayListMultimap.create(), mockRestoreOffsets);
    SystemStream changelog1SystemStream = new SystemStream(changelogSystemName, changelog1StreamName);
    SystemStream changelog2SystemStream = new SystemStream(changelogSystemName, changelog2StreamName);
    SystemStream changelog3SystemStream = new SystemStream(changelogSystemName, changelog3StreamName);
    SystemStream changelog4SystemStream = new SystemStream(changelogSystemName, changelog4StreamName);
    Map<String, SystemStream> mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream, store2Name, changelog2SystemStream, store3Name, changelog3SystemStream, store4Name, changelog4SystemStream);
    SystemStreamPartition changelog1SSP = new SystemStreamPartition(changelog1SystemStream, taskChangelogPartition);
    SystemStreamPartitionMetadata changelog1SSPMetadata = new SystemStreamPartitionMetadata("0", "10", "11");
    SystemStreamPartition changelog2SSP = new SystemStreamPartition(changelog2SystemStream, taskChangelogPartition);
    SystemStreamPartitionMetadata changelog2SSPMetadata = new SystemStreamPartitionMetadata("11", "20", "21");
    SystemStreamPartition changelog3SSP = new SystemStreamPartition(changelog3SystemStream, taskChangelogPartition);
    SystemStreamPartitionMetadata changelog3SSPMetadata = new SystemStreamPartitionMetadata("21", "30", "31");
    SystemStreamPartition changelog4SSP = new SystemStreamPartition(changelog4SystemStream, taskChangelogPartition);
    SystemStreamPartitionMetadata changelog4SSPMetadata = new SystemStreamPartitionMetadata("31", "40", "41");
    Map<SystemStreamPartition, SystemStreamPartitionMetadata> mockCurrentChangelogSSPMetadata = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata, changelog2SSP, changelog2SSPMetadata, changelog3SSP, changelog3SSPMetadata, changelog4SSP, changelog4SSPMetadata);
    SystemAdmins mockSystemAdmins = mock(SystemAdmins.class);
    SystemAdmin mockSystemAdmin = mock(SystemAdmin.class);
    when(mockSystemAdmins.getSystemAdmin(eq(changelogSystemName))).thenReturn(mockSystemAdmin);
    Mockito.when(mockSystemAdmin.offsetComparator(anyString(), anyString())).thenAnswer((Answer<Integer>) invocation -> {
        String offset1 = (String) invocation.getArguments()[0];
        String offset2 = (String) invocation.getArguments()[1];
        return Long.valueOf(offset1).compareTo(Long.valueOf(offset2));
    });
    Mockito.when(mockSystemAdmin.getOffsetsAfter(any())).thenAnswer((Answer<Map<SystemStreamPartition, String>>) invocation -> {
        Map<SystemStreamPartition, String> offsets = (Map<SystemStreamPartition, String>) invocation.getArguments()[0];
        Map<SystemStreamPartition, String> nextOffsets = new HashMap<>();
        offsets.forEach((ssp, offset) -> nextOffsets.put(ssp, Long.toString(Long.valueOf(offset) + 1)));
        return nextOffsets;
    });
    SystemConsumer mockSystemConsumer = mock(SystemConsumer.class);
    Map<String, SystemConsumer> mockStoreConsumers = ImmutableMap.of(store1Name, mockSystemConsumer, store2Name, mockSystemConsumer, store3Name, mockSystemConsumer, store4Name, mockSystemConsumer);
    TransactionalStateTaskRestoreManager.registerStartingOffsets(mockTaskModel, mockStoreActions, mockStoreChangelogs, mockSystemAdmins, mockStoreConsumers, mockCurrentChangelogSSPMetadata);
    // verify that we first register upcoming offsets for each changelog ssp
    verify(mockSystemConsumer, times(1)).register(changelog1SSP, "11");
    verify(mockSystemConsumer, times(1)).register(changelog2SSP, "21");
    verify(mockSystemConsumer, times(1)).register(changelog3SSP, "31");
    verify(mockSystemConsumer, times(1)).register(changelog4SSP, "41");
    // then verify that we override the starting offsets for changelog 1 and 2
    // ensure that starting offset is inclusive if oldest
    verify(mockSystemConsumer, times(1)).register(changelog1SSP, "0");
    // and that it is next offset if not oldest
    verify(mockSystemConsumer, times(1)).register(changelog2SSP, "16");
    // and that null ending offset is ok
    verify(mockSystemConsumer, times(1)).register(changelog4SSP, "31");
    verifyNoMoreInteractions(mockSystemConsumer);
}
Also used : ArrayListMultimap(com.google.common.collect.ArrayListMultimap) ListMultimap(com.google.common.collect.ListMultimap) SSPMetadataCache(org.apache.samza.system.SSPMetadataCache) HashMap(java.util.HashMap) TaskModel(org.apache.samza.job.model.TaskModel) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Matchers.anyString(org.mockito.Matchers.anyString) FileUtil(org.apache.samza.util.FileUtil) Answer(org.mockito.stubbing.Answer) ImmutableList(com.google.common.collect.ImmutableList) SystemConsumer(org.apache.samza.system.SystemConsumer) Mockito.verifyNoMoreInteractions(org.mockito.Mockito.verifyNoMoreInteractions) SystemStream(org.apache.samza.system.SystemStream) Matchers.eq(org.mockito.Matchers.eq) Map(java.util.Map) Assert.fail(org.junit.Assert.fail) Path(java.nio.file.Path) MapConfig(org.apache.samza.config.MapConfig) ImmutableSet(com.google.common.collect.ImmutableSet) TaskName(org.apache.samza.container.TaskName) ImmutableMap(com.google.common.collect.ImmutableMap) TaskConfig(org.apache.samza.config.TaskConfig) Assert.assertNotNull(org.junit.Assert.assertNotNull) Partition(org.apache.samza.Partition) Set(java.util.Set) Assert.assertTrue(org.junit.Assert.assertTrue) Clock(org.apache.samza.util.Clock) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) Mockito.when(org.mockito.Mockito.when) StoreActions(org.apache.samza.storage.TransactionalStateTaskRestoreManager.StoreActions) File(java.io.File) CheckpointId(org.apache.samza.checkpoint.CheckpointId) Mockito.verify(org.mockito.Mockito.verify) Matchers.any(org.mockito.Matchers.any) Mockito(org.mockito.Mockito) TaskMode(org.apache.samza.job.model.TaskMode) Mockito.never(org.mockito.Mockito.never) Assert.assertNull(org.junit.Assert.assertNull) RestoreOffsets(org.apache.samza.storage.TransactionalStateTaskRestoreManager.RestoreOffsets) SystemAdmin(org.apache.samza.system.SystemAdmin) SystemStreamPartitionMetadata(org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata) KafkaStateCheckpointMarker(org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker) Config(org.apache.samza.config.Config) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) SystemAdmins(org.apache.samza.system.SystemAdmins) Mockito.mock(org.mockito.Mockito.mock) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Partition(org.apache.samza.Partition) SystemConsumer(org.apache.samza.system.SystemConsumer) StoreActions(org.apache.samza.storage.TransactionalStateTaskRestoreManager.StoreActions) SystemStream(org.apache.samza.system.SystemStream) Matchers.anyString(org.mockito.Matchers.anyString) SystemStreamPartitionMetadata(org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata) RestoreOffsets(org.apache.samza.storage.TransactionalStateTaskRestoreManager.RestoreOffsets) TaskName(org.apache.samza.container.TaskName) SystemAdmin(org.apache.samza.system.SystemAdmin) SystemAdmins(org.apache.samza.system.SystemAdmins) HashMap(java.util.HashMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) TaskModel(org.apache.samza.job.model.TaskModel) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Test(org.junit.Test)

Example 12 with SystemAdmins

use of org.apache.samza.system.SystemAdmins in project samza by apache.

the class TestTransactionalStateTaskRestoreManager method testGetStoreActionsForNonLoggedPersistentStore_AlwaysClearStore.

@Test
public void testGetStoreActionsForNonLoggedPersistentStore_AlwaysClearStore() {
    TaskModel mockTaskModel = mock(TaskModel.class);
    TaskName taskName = new TaskName("Partition 0");
    when(mockTaskModel.getTaskName()).thenReturn(taskName);
    Partition taskChangelogPartition = new Partition(0);
    when(mockTaskModel.getChangelogPartition()).thenReturn(taskChangelogPartition);
    String store1Name = "store1";
    StorageEngine store1Engine = mock(StorageEngine.class);
    StoreProperties mockStore1Properties = mock(StoreProperties.class);
    when(store1Engine.getStoreProperties()).thenReturn(mockStore1Properties);
    when(mockStore1Properties.isLoggedStore()).thenReturn(false);
    when(mockStore1Properties.isPersistedToDisk()).thenReturn(true);
    Map<String, StorageEngine> mockStoreEngines = ImmutableMap.of(store1Name, store1Engine);
    Map<String, SystemStream> mockStoreChangelogs = ImmutableMap.of();
    Map<String, KafkaStateCheckpointMarker> mockCheckpointedChangelogOffset = ImmutableMap.of();
    Map<SystemStreamPartition, SystemStreamPartitionMetadata> mockCurrentChangelogOffsets = ImmutableMap.of();
    SystemAdmins mockSystemAdmins = mock(SystemAdmins.class);
    StorageManagerUtil mockStorageManagerUtil = mock(StorageManagerUtil.class);
    File mockLoggedStoreBaseDir = mock(File.class);
    File mockNonLoggedStoreBaseDir = mock(File.class);
    Config mockConfig = mock(Config.class);
    Clock mockClock = mock(Clock.class);
    File mockCurrentStoreDir = mock(File.class);
    when(mockStorageManagerUtil.getTaskStoreDir(eq(mockNonLoggedStoreBaseDir), eq(store1Name), eq(taskName), any())).thenReturn(mockCurrentStoreDir);
    StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions(mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, null, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock);
    assertEquals(1, storeActions.storeDirsToDelete.get(store1Name).size());
    assertTrue(storeActions.storeDirsToDelete.get(store1Name).contains(mockCurrentStoreDir));
    // ensure that there is nothing to retain or restore.
    assertEquals(0, storeActions.storeDirsToRetain.size());
    assertEquals(0, storeActions.storesToRestore.size());
}
Also used : SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Partition(org.apache.samza.Partition) SystemStream(org.apache.samza.system.SystemStream) MapConfig(org.apache.samza.config.MapConfig) TaskConfig(org.apache.samza.config.TaskConfig) Config(org.apache.samza.config.Config) StoreActions(org.apache.samza.storage.TransactionalStateTaskRestoreManager.StoreActions) Matchers.anyString(org.mockito.Matchers.anyString) SystemStreamPartitionMetadata(org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata) Clock(org.apache.samza.util.Clock) TaskName(org.apache.samza.container.TaskName) SystemAdmins(org.apache.samza.system.SystemAdmins) File(java.io.File) TaskModel(org.apache.samza.job.model.TaskModel) KafkaStateCheckpointMarker(org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Test(org.junit.Test)

Example 13 with SystemAdmins

use of org.apache.samza.system.SystemAdmins in project samza by apache.

the class TestTransactionalStateTaskRestoreManager method testStoreDeletedWhenCleanDirsFlagSet.

@Test
public void testStoreDeletedWhenCleanDirsFlagSet() {
    TaskModel mockTaskModel = mock(TaskModel.class);
    TaskName taskName = new TaskName("Partition 0");
    when(mockTaskModel.getTaskName()).thenReturn(taskName);
    Partition taskChangelogPartition = new Partition(0);
    when(mockTaskModel.getChangelogPartition()).thenReturn(taskChangelogPartition);
    when(mockTaskModel.getTaskMode()).thenReturn(TaskMode.Active);
    String store1Name = "store1";
    StorageEngine store1Engine = mock(StorageEngine.class);
    StoreProperties mockStore1Properties = mock(StoreProperties.class);
    when(store1Engine.getStoreProperties()).thenReturn(mockStore1Properties);
    when(mockStore1Properties.isLoggedStore()).thenReturn(true);
    when(mockStore1Properties.isPersistedToDisk()).thenReturn(true);
    Map<String, StorageEngine> mockStoreEngines = ImmutableMap.of(store1Name, store1Engine);
    String changelog1SystemName = "system1";
    String changelog1StreamName = "store1Changelog";
    SystemStream changelog1SystemStream = new SystemStream(changelog1SystemName, changelog1StreamName);
    SystemStreamPartition changelog1SSP = new SystemStreamPartition(changelog1SystemStream, taskChangelogPartition);
    SystemStreamPartitionMetadata changelog1SSPMetadata = new SystemStreamPartitionMetadata("0", "10", "11");
    Map<String, SystemStream> mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream);
    String changelog1CheckpointedOffset = "5";
    CheckpointId checkpointId = CheckpointId.create();
    KafkaStateCheckpointMarker kafkaStateCheckpointMarker = new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset);
    ImmutableMap<String, KafkaStateCheckpointMarker> mockCheckpointedChangelogOffset = ImmutableMap.of(store1Name, kafkaStateCheckpointMarker);
    Map<SystemStreamPartition, SystemStreamPartitionMetadata> mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata);
    SystemAdmins mockSystemAdmins = mock(SystemAdmins.class);
    SystemAdmin mockSystemAdmin = mock(SystemAdmin.class);
    when(mockSystemAdmins.getSystemAdmin(changelog1SSP.getSystem())).thenReturn(mockSystemAdmin);
    StorageManagerUtil mockStorageManagerUtil = mock(StorageManagerUtil.class);
    File mockLoggedStoreBaseDir = mock(File.class);
    File mockNonLoggedStoreBaseDir = mock(File.class);
    // set the clean.on.container.start config set on the store
    Config mockConfig = new MapConfig(Collections.singletonMap("stores.store1.clean.on.container.start", "true"));
    Clock mockClock = mock(Clock.class);
    Mockito.when(mockSystemAdmin.offsetComparator(anyString(), anyString())).thenAnswer((Answer<Integer>) invocation -> {
        String offset1 = (String) invocation.getArguments()[0];
        String offset2 = (String) invocation.getArguments()[1];
        return Long.valueOf(offset1).compareTo(Long.valueOf(offset2));
    });
    File dummyCurrentDir = new File("currentDir");
    File dummyCheckpointDir = new File("checkpointDir1");
    when(mockStorageManagerUtil.getTaskStoreDir(mockLoggedStoreBaseDir, store1Name, taskName, TaskMode.Active)).thenReturn(dummyCurrentDir);
    when(mockStorageManagerUtil.getTaskStoreCheckpointDirs(mockLoggedStoreBaseDir, store1Name, taskName, TaskMode.Active)).thenReturn(ImmutableList.of(dummyCheckpointDir));
    StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions(mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock);
    // ensure that current and checkpoint directories are marked for deletion
    assertEquals(2, storeActions.storeDirsToDelete.size());
    assertTrue(storeActions.storeDirsToDelete.containsValue(dummyCheckpointDir));
    assertTrue(storeActions.storeDirsToDelete.containsValue(dummyCurrentDir));
    // ensure that we restore from the oldest changelog offset to checkpointed changelog offset
    assertEquals("0", storeActions.storesToRestore.get(store1Name).startingOffset);
    assertEquals(changelog1CheckpointedOffset, storeActions.storesToRestore.get(store1Name).endingOffset);
}
Also used : ArrayListMultimap(com.google.common.collect.ArrayListMultimap) ListMultimap(com.google.common.collect.ListMultimap) SSPMetadataCache(org.apache.samza.system.SSPMetadataCache) HashMap(java.util.HashMap) TaskModel(org.apache.samza.job.model.TaskModel) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Matchers.anyString(org.mockito.Matchers.anyString) FileUtil(org.apache.samza.util.FileUtil) Answer(org.mockito.stubbing.Answer) ImmutableList(com.google.common.collect.ImmutableList) SystemConsumer(org.apache.samza.system.SystemConsumer) Mockito.verifyNoMoreInteractions(org.mockito.Mockito.verifyNoMoreInteractions) SystemStream(org.apache.samza.system.SystemStream) Matchers.eq(org.mockito.Matchers.eq) Map(java.util.Map) Assert.fail(org.junit.Assert.fail) Path(java.nio.file.Path) MapConfig(org.apache.samza.config.MapConfig) ImmutableSet(com.google.common.collect.ImmutableSet) TaskName(org.apache.samza.container.TaskName) ImmutableMap(com.google.common.collect.ImmutableMap) TaskConfig(org.apache.samza.config.TaskConfig) Assert.assertNotNull(org.junit.Assert.assertNotNull) Partition(org.apache.samza.Partition) Set(java.util.Set) Assert.assertTrue(org.junit.Assert.assertTrue) Clock(org.apache.samza.util.Clock) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) Mockito.when(org.mockito.Mockito.when) StoreActions(org.apache.samza.storage.TransactionalStateTaskRestoreManager.StoreActions) File(java.io.File) CheckpointId(org.apache.samza.checkpoint.CheckpointId) Mockito.verify(org.mockito.Mockito.verify) Matchers.any(org.mockito.Matchers.any) Mockito(org.mockito.Mockito) TaskMode(org.apache.samza.job.model.TaskMode) Mockito.never(org.mockito.Mockito.never) Assert.assertNull(org.junit.Assert.assertNull) RestoreOffsets(org.apache.samza.storage.TransactionalStateTaskRestoreManager.RestoreOffsets) SystemAdmin(org.apache.samza.system.SystemAdmin) SystemStreamPartitionMetadata(org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata) KafkaStateCheckpointMarker(org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker) Config(org.apache.samza.config.Config) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) SystemAdmins(org.apache.samza.system.SystemAdmins) Mockito.mock(org.mockito.Mockito.mock) MapConfig(org.apache.samza.config.MapConfig) TaskConfig(org.apache.samza.config.TaskConfig) Config(org.apache.samza.config.Config) Matchers.anyString(org.mockito.Matchers.anyString) SystemStreamPartitionMetadata(org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata) Clock(org.apache.samza.util.Clock) MapConfig(org.apache.samza.config.MapConfig) SystemAdmins(org.apache.samza.system.SystemAdmins) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Partition(org.apache.samza.Partition) SystemStream(org.apache.samza.system.SystemStream) StoreActions(org.apache.samza.storage.TransactionalStateTaskRestoreManager.StoreActions) TaskName(org.apache.samza.container.TaskName) CheckpointId(org.apache.samza.checkpoint.CheckpointId) SystemAdmin(org.apache.samza.system.SystemAdmin) File(java.io.File) TaskModel(org.apache.samza.job.model.TaskModel) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) KafkaStateCheckpointMarker(org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker) Test(org.junit.Test)

Example 14 with SystemAdmins

use of org.apache.samza.system.SystemAdmins in project samza by apache.

the class TestTransactionalStateTaskRestoreManager method testRegisterStartingOffsetsThrowsIfStartingGreaterThanEnding.

@Test(expected = IllegalStateException.class)
public void testRegisterStartingOffsetsThrowsIfStartingGreaterThanEnding() {
    TaskModel mockTaskModel = mock(TaskModel.class);
    TaskName taskName = new TaskName("Partition 0");
    when(mockTaskModel.getTaskName()).thenReturn(taskName);
    Partition taskChangelogPartition = new Partition(0);
    when(mockTaskModel.getChangelogPartition()).thenReturn(taskChangelogPartition);
    // tests starting offset > ending offset
    Map<String, RestoreOffsets> mockRestoreOffsets = ImmutableMap.of("store1", new RestoreOffsets("5", "0"));
    StoreActions mockStoreActions = new StoreActions(ImmutableMap.of(), ArrayListMultimap.create(), mockRestoreOffsets);
    String store1Name = "store1";
    String changelogSystemName = "system";
    String changelog1StreamName = "store1Changelog";
    SystemStream changelog1SystemStream = new SystemStream(changelogSystemName, changelog1StreamName);
    Map<String, SystemStream> mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream);
    SystemStreamPartition changelog1SSP = new SystemStreamPartition(changelog1SystemStream, taskChangelogPartition);
    SystemStreamPartitionMetadata changelog1SSPMetadata = new SystemStreamPartitionMetadata("0", "10", "11");
    Map<SystemStreamPartition, SystemStreamPartitionMetadata> mockCurrentChangelogSSPMetadata = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata);
    SystemAdmins mockSystemAdmins = mock(SystemAdmins.class);
    SystemAdmin mockSystemAdmin = mock(SystemAdmin.class);
    when(mockSystemAdmins.getSystemAdmin(eq(changelogSystemName))).thenReturn(mockSystemAdmin);
    Mockito.when(mockSystemAdmin.offsetComparator(anyString(), anyString())).thenAnswer((Answer<Integer>) invocation -> {
        String offset1 = (String) invocation.getArguments()[0];
        String offset2 = (String) invocation.getArguments()[1];
        return Long.valueOf(offset1).compareTo(Long.valueOf(offset2));
    });
    Mockito.when(mockSystemAdmin.getOffsetsAfter(any())).thenAnswer((Answer<Map<SystemStreamPartition, String>>) invocation -> {
        Map<SystemStreamPartition, String> offsets = (Map<SystemStreamPartition, String>) invocation.getArguments()[0];
        Map<SystemStreamPartition, String> nextOffsets = new HashMap<>();
        offsets.forEach((ssp, offset) -> nextOffsets.put(ssp, Long.toString(Long.valueOf(offset) + 1)));
        return nextOffsets;
    });
    SystemConsumer mockSystemConsumer = mock(SystemConsumer.class);
    Map<String, SystemConsumer> mockStoreConsumers = ImmutableMap.of("store1", mockSystemConsumer);
    TransactionalStateTaskRestoreManager.registerStartingOffsets(mockTaskModel, mockStoreActions, mockStoreChangelogs, mockSystemAdmins, mockStoreConsumers, mockCurrentChangelogSSPMetadata);
    fail("Should have thrown an exception since starting offset > ending offset");
}
Also used : ArrayListMultimap(com.google.common.collect.ArrayListMultimap) ListMultimap(com.google.common.collect.ListMultimap) SSPMetadataCache(org.apache.samza.system.SSPMetadataCache) HashMap(java.util.HashMap) TaskModel(org.apache.samza.job.model.TaskModel) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Matchers.anyString(org.mockito.Matchers.anyString) FileUtil(org.apache.samza.util.FileUtil) Answer(org.mockito.stubbing.Answer) ImmutableList(com.google.common.collect.ImmutableList) SystemConsumer(org.apache.samza.system.SystemConsumer) Mockito.verifyNoMoreInteractions(org.mockito.Mockito.verifyNoMoreInteractions) SystemStream(org.apache.samza.system.SystemStream) Matchers.eq(org.mockito.Matchers.eq) Map(java.util.Map) Assert.fail(org.junit.Assert.fail) Path(java.nio.file.Path) MapConfig(org.apache.samza.config.MapConfig) ImmutableSet(com.google.common.collect.ImmutableSet) TaskName(org.apache.samza.container.TaskName) ImmutableMap(com.google.common.collect.ImmutableMap) TaskConfig(org.apache.samza.config.TaskConfig) Assert.assertNotNull(org.junit.Assert.assertNotNull) Partition(org.apache.samza.Partition) Set(java.util.Set) Assert.assertTrue(org.junit.Assert.assertTrue) Clock(org.apache.samza.util.Clock) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) Mockito.when(org.mockito.Mockito.when) StoreActions(org.apache.samza.storage.TransactionalStateTaskRestoreManager.StoreActions) File(java.io.File) CheckpointId(org.apache.samza.checkpoint.CheckpointId) Mockito.verify(org.mockito.Mockito.verify) Matchers.any(org.mockito.Matchers.any) Mockito(org.mockito.Mockito) TaskMode(org.apache.samza.job.model.TaskMode) Mockito.never(org.mockito.Mockito.never) Assert.assertNull(org.junit.Assert.assertNull) RestoreOffsets(org.apache.samza.storage.TransactionalStateTaskRestoreManager.RestoreOffsets) SystemAdmin(org.apache.samza.system.SystemAdmin) SystemStreamPartitionMetadata(org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata) KafkaStateCheckpointMarker(org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker) Config(org.apache.samza.config.Config) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) SystemAdmins(org.apache.samza.system.SystemAdmins) Mockito.mock(org.mockito.Mockito.mock) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Partition(org.apache.samza.Partition) SystemConsumer(org.apache.samza.system.SystemConsumer) StoreActions(org.apache.samza.storage.TransactionalStateTaskRestoreManager.StoreActions) SystemStream(org.apache.samza.system.SystemStream) Matchers.anyString(org.mockito.Matchers.anyString) SystemStreamPartitionMetadata(org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata) RestoreOffsets(org.apache.samza.storage.TransactionalStateTaskRestoreManager.RestoreOffsets) TaskName(org.apache.samza.container.TaskName) SystemAdmin(org.apache.samza.system.SystemAdmin) SystemAdmins(org.apache.samza.system.SystemAdmins) HashMap(java.util.HashMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) TaskModel(org.apache.samza.job.model.TaskModel) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Test(org.junit.Test)

Example 15 with SystemAdmins

use of org.apache.samza.system.SystemAdmins in project samza by apache.

the class TestTransactionalStateTaskRestoreManager method testGetStoreActionsForLoggedPersistentStore_FullRestoreIfEqualCheckpointedOldestAndNewestOffset.

/**
 * This is the case when the changelog topic is empty but not new. E.g., if wrote 100 messages,
 * then deleted 100 messages, and after compaction oldest == newest == checkpointed. In this case
 * full restore does not do anything since there is nothing to restore or trim, but the code path will
 * leave us in a consistent state with the appropriate stores deleted and retained.
 */
@Test
public void testGetStoreActionsForLoggedPersistentStore_FullRestoreIfEqualCheckpointedOldestAndNewestOffset() {
    TaskModel mockTaskModel = mock(TaskModel.class);
    TaskName taskName = new TaskName("Partition 0");
    when(mockTaskModel.getTaskName()).thenReturn(taskName);
    Partition taskChangelogPartition = new Partition(0);
    when(mockTaskModel.getChangelogPartition()).thenReturn(taskChangelogPartition);
    String store1Name = "store1";
    StorageEngine store1Engine = mock(StorageEngine.class);
    StoreProperties mockStore1Properties = mock(StoreProperties.class);
    when(store1Engine.getStoreProperties()).thenReturn(mockStore1Properties);
    when(mockStore1Properties.isLoggedStore()).thenReturn(true);
    when(mockStore1Properties.isPersistedToDisk()).thenReturn(true);
    Map<String, StorageEngine> mockStoreEngines = ImmutableMap.of(store1Name, store1Engine);
    String changelog1SystemName = "system1";
    String changelog1StreamName = "store1Changelog";
    SystemStream changelog1SystemStream = new SystemStream(changelog1SystemName, changelog1StreamName);
    SystemStreamPartition changelog1SSP = new SystemStreamPartition(changelog1SystemStream, taskChangelogPartition);
    SystemStreamPartitionMetadata changelog1SSPMetadata = new SystemStreamPartitionMetadata("5", "5", "6");
    Map<String, SystemStream> mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream);
    String changelog1CheckpointedOffset = "5";
    CheckpointId checkpointId = CheckpointId.create();
    KafkaStateCheckpointMarker kafkaStateCheckpointMarker = new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset);
    Map<String, KafkaStateCheckpointMarker> mockCheckpointedChangelogOffset = new HashMap<String, KafkaStateCheckpointMarker>() {

        {
            put(store1Name, kafkaStateCheckpointMarker);
        }
    };
    Map<SystemStreamPartition, SystemStreamPartitionMetadata> mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata);
    SystemAdmins mockSystemAdmins = mock(SystemAdmins.class);
    SystemAdmin mockSystemAdmin = mock(SystemAdmin.class);
    when(mockSystemAdmins.getSystemAdmin(changelog1SSP.getSystem())).thenReturn(mockSystemAdmin);
    StorageManagerUtil mockStorageManagerUtil = mock(StorageManagerUtil.class);
    File mockLoggedStoreBaseDir = mock(File.class);
    File mockNonLoggedStoreBaseDir = mock(File.class);
    HashMap<String, String> configMap = new HashMap<>();
    // should not matter
    configMap.put(TaskConfig.TRANSACTIONAL_STATE_RETAIN_EXISTING_STATE, "true");
    Config mockConfig = new MapConfig(configMap);
    Clock mockClock = mock(Clock.class);
    File mockCurrentStoreDir = mock(File.class);
    File mockStoreCheckpointDir = mock(File.class);
    String checkpointDirLocalOffset = "5";
    when(mockStorageManagerUtil.getTaskStoreDir(eq(mockLoggedStoreBaseDir), eq(store1Name), eq(taskName), any())).thenReturn(mockCurrentStoreDir);
    when(mockStorageManagerUtil.getTaskStoreCheckpointDirs(eq(mockLoggedStoreBaseDir), eq(store1Name), eq(taskName), any())).thenReturn(ImmutableList.of(mockStoreCheckpointDir));
    when(mockStorageManagerUtil.isLoggedStoreValid(eq(store1Name), eq(mockStoreCheckpointDir), any(), eq(mockStoreChangelogs), eq(mockTaskModel), any(), eq(mockStoreEngines))).thenReturn(true);
    Set<SystemStreamPartition> mockChangelogSSPs = ImmutableSet.of(changelog1SSP);
    when(mockStorageManagerUtil.readOffsetFile(eq(mockStoreCheckpointDir), eq(mockChangelogSSPs), eq(false))).thenReturn(ImmutableMap.of(changelog1SSP, checkpointDirLocalOffset));
    Mockito.when(mockSystemAdmin.offsetComparator(anyString(), anyString())).thenAnswer((Answer<Integer>) invocation -> {
        String offset1 = (String) invocation.getArguments()[0];
        String offset2 = (String) invocation.getArguments()[1];
        if (offset1 == null || offset2 == null) {
            return -1;
        }
        return Long.valueOf(offset1).compareTo(Long.valueOf(offset2));
    });
    StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions(mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock);
    // ensure that current and old checkpoint dirs are marked for deletion
    assertEquals(1, storeActions.storeDirsToDelete.get(store1Name).size());
    assertTrue(storeActions.storeDirsToDelete.get(store1Name).contains(mockCurrentStoreDir));
    // ensure that latest checkpoint dir is retained
    assertEquals(mockStoreCheckpointDir, storeActions.storeDirsToRetain.get(store1Name));
    // ensure that we do a full restore (on the empty topic)
    assertEquals("5", storeActions.storesToRestore.get(store1Name).startingOffset);
    assertEquals("5", storeActions.storesToRestore.get(store1Name).endingOffset);
}
Also used : ArrayListMultimap(com.google.common.collect.ArrayListMultimap) ListMultimap(com.google.common.collect.ListMultimap) SSPMetadataCache(org.apache.samza.system.SSPMetadataCache) HashMap(java.util.HashMap) TaskModel(org.apache.samza.job.model.TaskModel) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Matchers.anyString(org.mockito.Matchers.anyString) FileUtil(org.apache.samza.util.FileUtil) Answer(org.mockito.stubbing.Answer) ImmutableList(com.google.common.collect.ImmutableList) SystemConsumer(org.apache.samza.system.SystemConsumer) Mockito.verifyNoMoreInteractions(org.mockito.Mockito.verifyNoMoreInteractions) SystemStream(org.apache.samza.system.SystemStream) Matchers.eq(org.mockito.Matchers.eq) Map(java.util.Map) Assert.fail(org.junit.Assert.fail) Path(java.nio.file.Path) MapConfig(org.apache.samza.config.MapConfig) ImmutableSet(com.google.common.collect.ImmutableSet) TaskName(org.apache.samza.container.TaskName) ImmutableMap(com.google.common.collect.ImmutableMap) TaskConfig(org.apache.samza.config.TaskConfig) Assert.assertNotNull(org.junit.Assert.assertNotNull) Partition(org.apache.samza.Partition) Set(java.util.Set) Assert.assertTrue(org.junit.Assert.assertTrue) Clock(org.apache.samza.util.Clock) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) Mockito.when(org.mockito.Mockito.when) StoreActions(org.apache.samza.storage.TransactionalStateTaskRestoreManager.StoreActions) File(java.io.File) CheckpointId(org.apache.samza.checkpoint.CheckpointId) Mockito.verify(org.mockito.Mockito.verify) Matchers.any(org.mockito.Matchers.any) Mockito(org.mockito.Mockito) TaskMode(org.apache.samza.job.model.TaskMode) Mockito.never(org.mockito.Mockito.never) Assert.assertNull(org.junit.Assert.assertNull) RestoreOffsets(org.apache.samza.storage.TransactionalStateTaskRestoreManager.RestoreOffsets) SystemAdmin(org.apache.samza.system.SystemAdmin) SystemStreamPartitionMetadata(org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata) KafkaStateCheckpointMarker(org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker) Config(org.apache.samza.config.Config) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) SystemAdmins(org.apache.samza.system.SystemAdmins) Mockito.mock(org.mockito.Mockito.mock) HashMap(java.util.HashMap) MapConfig(org.apache.samza.config.MapConfig) TaskConfig(org.apache.samza.config.TaskConfig) Config(org.apache.samza.config.Config) Matchers.anyString(org.mockito.Matchers.anyString) SystemStreamPartitionMetadata(org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata) Clock(org.apache.samza.util.Clock) MapConfig(org.apache.samza.config.MapConfig) SystemAdmins(org.apache.samza.system.SystemAdmins) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Partition(org.apache.samza.Partition) SystemStream(org.apache.samza.system.SystemStream) StoreActions(org.apache.samza.storage.TransactionalStateTaskRestoreManager.StoreActions) TaskName(org.apache.samza.container.TaskName) CheckpointId(org.apache.samza.checkpoint.CheckpointId) SystemAdmin(org.apache.samza.system.SystemAdmin) File(java.io.File) TaskModel(org.apache.samza.job.model.TaskModel) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) KafkaStateCheckpointMarker(org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker) Test(org.junit.Test)

Aggregations

SystemAdmins (org.apache.samza.system.SystemAdmins)47 SystemAdmin (org.apache.samza.system.SystemAdmin)37 SystemStream (org.apache.samza.system.SystemStream)36 Test (org.junit.Test)35 HashMap (java.util.HashMap)34 Partition (org.apache.samza.Partition)31 TaskName (org.apache.samza.container.TaskName)31 SystemStreamPartition (org.apache.samza.system.SystemStreamPartition)31 Config (org.apache.samza.config.Config)30 MapConfig (org.apache.samza.config.MapConfig)29 SystemStreamPartitionMetadata (org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata)29 Map (java.util.Map)28 TaskConfig (org.apache.samza.config.TaskConfig)28 Set (java.util.Set)27 File (java.io.File)26 KafkaStateCheckpointMarker (org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker)26 TaskModel (org.apache.samza.job.model.TaskModel)26 Clock (org.apache.samza.util.Clock)26 Collections (java.util.Collections)25 SSPMetadataCache (org.apache.samza.system.SSPMetadataCache)25