Examples with KafkaStateCheckpointMarker org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker used on opensource projects

Example 6 with KafkaStateCheckpointMarker

use of org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker in project samza by apache.

the class TestTransactionalStateTaskRestoreManager method testGetStoreActionsForNonLoggedPersistentStore_AlwaysClearStore.

@Test
public void testGetStoreActionsForNonLoggedPersistentStore_AlwaysClearStore() {
    TaskModel mockTaskModel = mock(TaskModel.class);
    TaskName taskName = new TaskName("Partition 0");
    when(mockTaskModel.getTaskName()).thenReturn(taskName);
    Partition taskChangelogPartition = new Partition(0);
    when(mockTaskModel.getChangelogPartition()).thenReturn(taskChangelogPartition);
    String store1Name = "store1";
    StorageEngine store1Engine = mock(StorageEngine.class);
    StoreProperties mockStore1Properties = mock(StoreProperties.class);
    when(store1Engine.getStoreProperties()).thenReturn(mockStore1Properties);
    when(mockStore1Properties.isLoggedStore()).thenReturn(false);
    when(mockStore1Properties.isPersistedToDisk()).thenReturn(true);
    Map<String, StorageEngine> mockStoreEngines = ImmutableMap.of(store1Name, store1Engine);
    Map<String, SystemStream> mockStoreChangelogs = ImmutableMap.of();
    Map<String, KafkaStateCheckpointMarker> mockCheckpointedChangelogOffset = ImmutableMap.of();
    Map<SystemStreamPartition, SystemStreamPartitionMetadata> mockCurrentChangelogOffsets = ImmutableMap.of();
    SystemAdmins mockSystemAdmins = mock(SystemAdmins.class);
    StorageManagerUtil mockStorageManagerUtil = mock(StorageManagerUtil.class);
    File mockLoggedStoreBaseDir = mock(File.class);
    File mockNonLoggedStoreBaseDir = mock(File.class);
    Config mockConfig = mock(Config.class);
    Clock mockClock = mock(Clock.class);
    File mockCurrentStoreDir = mock(File.class);
    when(mockStorageManagerUtil.getTaskStoreDir(eq(mockNonLoggedStoreBaseDir), eq(store1Name), eq(taskName), any())).thenReturn(mockCurrentStoreDir);
    StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions(mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, null, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock);
    assertEquals(1, storeActions.storeDirsToDelete.get(store1Name).size());
    assertTrue(storeActions.storeDirsToDelete.get(store1Name).contains(mockCurrentStoreDir));
    // ensure that there is nothing to retain or restore.
    assertEquals(0, storeActions.storeDirsToRetain.size());
    assertEquals(0, storeActions.storesToRestore.size());
}

Example 7 with KafkaStateCheckpointMarker

use of org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker in project samza by apache.

the class TestTransactionalStateTaskRestoreManager method testStoreDeletedWhenCleanDirsFlagSet.

@Test
public void testStoreDeletedWhenCleanDirsFlagSet() {
    TaskModel mockTaskModel = mock(TaskModel.class);
    TaskName taskName = new TaskName("Partition 0");
    when(mockTaskModel.getTaskName()).thenReturn(taskName);
    Partition taskChangelogPartition = new Partition(0);
    when(mockTaskModel.getChangelogPartition()).thenReturn(taskChangelogPartition);
    when(mockTaskModel.getTaskMode()).thenReturn(TaskMode.Active);
    String store1Name = "store1";
    StorageEngine store1Engine = mock(StorageEngine.class);
    StoreProperties mockStore1Properties = mock(StoreProperties.class);
    when(store1Engine.getStoreProperties()).thenReturn(mockStore1Properties);
    when(mockStore1Properties.isLoggedStore()).thenReturn(true);
    when(mockStore1Properties.isPersistedToDisk()).thenReturn(true);
    Map<String, StorageEngine> mockStoreEngines = ImmutableMap.of(store1Name, store1Engine);
    String changelog1SystemName = "system1";
    String changelog1StreamName = "store1Changelog";
    SystemStream changelog1SystemStream = new SystemStream(changelog1SystemName, changelog1StreamName);
    SystemStreamPartition changelog1SSP = new SystemStreamPartition(changelog1SystemStream, taskChangelogPartition);
    SystemStreamPartitionMetadata changelog1SSPMetadata = new SystemStreamPartitionMetadata("0", "10", "11");
    Map<String, SystemStream> mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream);
    String changelog1CheckpointedOffset = "5";
    CheckpointId checkpointId = CheckpointId.create();
    KafkaStateCheckpointMarker kafkaStateCheckpointMarker = new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset);
    ImmutableMap<String, KafkaStateCheckpointMarker> mockCheckpointedChangelogOffset = ImmutableMap.of(store1Name, kafkaStateCheckpointMarker);
    Map<SystemStreamPartition, SystemStreamPartitionMetadata> mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata);
    SystemAdmins mockSystemAdmins = mock(SystemAdmins.class);
    SystemAdmin mockSystemAdmin = mock(SystemAdmin.class);
    when(mockSystemAdmins.getSystemAdmin(changelog1SSP.getSystem())).thenReturn(mockSystemAdmin);
    StorageManagerUtil mockStorageManagerUtil = mock(StorageManagerUtil.class);
    File mockLoggedStoreBaseDir = mock(File.class);
    File mockNonLoggedStoreBaseDir = mock(File.class);
    // set the clean.on.container.start config set on the store
    Config mockConfig = new MapConfig(Collections.singletonMap("stores.store1.clean.on.container.start", "true"));
    Clock mockClock = mock(Clock.class);
    Mockito.when(mockSystemAdmin.offsetComparator(anyString(), anyString())).thenAnswer((Answer<Integer>) invocation -> {
        String offset1 = (String) invocation.getArguments()[0];
        String offset2 = (String) invocation.getArguments()[1];
        return Long.valueOf(offset1).compareTo(Long.valueOf(offset2));
    });
    File dummyCurrentDir = new File("currentDir");
    File dummyCheckpointDir = new File("checkpointDir1");
    when(mockStorageManagerUtil.getTaskStoreDir(mockLoggedStoreBaseDir, store1Name, taskName, TaskMode.Active)).thenReturn(dummyCurrentDir);
    when(mockStorageManagerUtil.getTaskStoreCheckpointDirs(mockLoggedStoreBaseDir, store1Name, taskName, TaskMode.Active)).thenReturn(ImmutableList.of(dummyCheckpointDir));
    StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions(mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock);
    // ensure that current and checkpoint directories are marked for deletion
    assertEquals(2, storeActions.storeDirsToDelete.size());
    assertTrue(storeActions.storeDirsToDelete.containsValue(dummyCheckpointDir));
    assertTrue(storeActions.storeDirsToDelete.containsValue(dummyCurrentDir));
    // ensure that we restore from the oldest changelog offset to checkpointed changelog offset
    assertEquals("0", storeActions.storesToRestore.get(store1Name).startingOffset);
    assertEquals(changelog1CheckpointedOffset, storeActions.storesToRestore.get(store1Name).endingOffset);
}

Example 8 with KafkaStateCheckpointMarker

use of org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker in project samza by apache.

the class TestTransactionalStateTaskRestoreManager method testGetStoreActionsForLoggedPersistentStore_FullRestoreIfEqualCheckpointedOldestAndNewestOffset.

/**
 * This is the case when the changelog topic is empty but not new. E.g., if wrote 100 messages,
 * then deleted 100 messages, and after compaction oldest == newest == checkpointed. In this case
 * full restore does not do anything since there is nothing to restore or trim, but the code path will
 * leave us in a consistent state with the appropriate stores deleted and retained.
 */
@Test
public void testGetStoreActionsForLoggedPersistentStore_FullRestoreIfEqualCheckpointedOldestAndNewestOffset() {
    TaskModel mockTaskModel = mock(TaskModel.class);
    TaskName taskName = new TaskName("Partition 0");
    when(mockTaskModel.getTaskName()).thenReturn(taskName);
    Partition taskChangelogPartition = new Partition(0);
    when(mockTaskModel.getChangelogPartition()).thenReturn(taskChangelogPartition);
    String store1Name = "store1";
    StorageEngine store1Engine = mock(StorageEngine.class);
    StoreProperties mockStore1Properties = mock(StoreProperties.class);
    when(store1Engine.getStoreProperties()).thenReturn(mockStore1Properties);
    when(mockStore1Properties.isLoggedStore()).thenReturn(true);
    when(mockStore1Properties.isPersistedToDisk()).thenReturn(true);
    Map<String, StorageEngine> mockStoreEngines = ImmutableMap.of(store1Name, store1Engine);
    String changelog1SystemName = "system1";
    String changelog1StreamName = "store1Changelog";
    SystemStream changelog1SystemStream = new SystemStream(changelog1SystemName, changelog1StreamName);
    SystemStreamPartition changelog1SSP = new SystemStreamPartition(changelog1SystemStream, taskChangelogPartition);
    SystemStreamPartitionMetadata changelog1SSPMetadata = new SystemStreamPartitionMetadata("5", "5", "6");
    Map<String, SystemStream> mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream);
    String changelog1CheckpointedOffset = "5";
    CheckpointId checkpointId = CheckpointId.create();
    KafkaStateCheckpointMarker kafkaStateCheckpointMarker = new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset);
    Map<String, KafkaStateCheckpointMarker> mockCheckpointedChangelogOffset = new HashMap<String, KafkaStateCheckpointMarker>() {

        {
            put(store1Name, kafkaStateCheckpointMarker);
        }
    };
    Map<SystemStreamPartition, SystemStreamPartitionMetadata> mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata);
    SystemAdmins mockSystemAdmins = mock(SystemAdmins.class);
    SystemAdmin mockSystemAdmin = mock(SystemAdmin.class);
    when(mockSystemAdmins.getSystemAdmin(changelog1SSP.getSystem())).thenReturn(mockSystemAdmin);
    StorageManagerUtil mockStorageManagerUtil = mock(StorageManagerUtil.class);
    File mockLoggedStoreBaseDir = mock(File.class);
    File mockNonLoggedStoreBaseDir = mock(File.class);
    HashMap<String, String> configMap = new HashMap<>();
    // should not matter
    configMap.put(TaskConfig.TRANSACTIONAL_STATE_RETAIN_EXISTING_STATE, "true");
    Config mockConfig = new MapConfig(configMap);
    Clock mockClock = mock(Clock.class);
    File mockCurrentStoreDir = mock(File.class);
    File mockStoreCheckpointDir = mock(File.class);
    String checkpointDirLocalOffset = "5";
    when(mockStorageManagerUtil.getTaskStoreDir(eq(mockLoggedStoreBaseDir), eq(store1Name), eq(taskName), any())).thenReturn(mockCurrentStoreDir);
    when(mockStorageManagerUtil.getTaskStoreCheckpointDirs(eq(mockLoggedStoreBaseDir), eq(store1Name), eq(taskName), any())).thenReturn(ImmutableList.of(mockStoreCheckpointDir));
    when(mockStorageManagerUtil.isLoggedStoreValid(eq(store1Name), eq(mockStoreCheckpointDir), any(), eq(mockStoreChangelogs), eq(mockTaskModel), any(), eq(mockStoreEngines))).thenReturn(true);
    Set<SystemStreamPartition> mockChangelogSSPs = ImmutableSet.of(changelog1SSP);
    when(mockStorageManagerUtil.readOffsetFile(eq(mockStoreCheckpointDir), eq(mockChangelogSSPs), eq(false))).thenReturn(ImmutableMap.of(changelog1SSP, checkpointDirLocalOffset));
    Mockito.when(mockSystemAdmin.offsetComparator(anyString(), anyString())).thenAnswer((Answer<Integer>) invocation -> {
        String offset1 = (String) invocation.getArguments()[0];
        String offset2 = (String) invocation.getArguments()[1];
        if (offset1 == null || offset2 == null) {
            return -1;
        }
        return Long.valueOf(offset1).compareTo(Long.valueOf(offset2));
    });
    StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions(mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock);
    // ensure that current and old checkpoint dirs are marked for deletion
    assertEquals(1, storeActions.storeDirsToDelete.get(store1Name).size());
    assertTrue(storeActions.storeDirsToDelete.get(store1Name).contains(mockCurrentStoreDir));
    // ensure that latest checkpoint dir is retained
    assertEquals(mockStoreCheckpointDir, storeActions.storeDirsToRetain.get(store1Name));
    // ensure that we do a full restore (on the empty topic)
    assertEquals("5", storeActions.storesToRestore.get(store1Name).startingOffset);
    assertEquals("5", storeActions.storesToRestore.get(store1Name).endingOffset);
}

Example 9 with KafkaStateCheckpointMarker

use of org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker in project samza by apache.

the class TestKafkaStateCheckpointMarker method testSerializeDeserialize.

@Test
public void testSerializeDeserialize() {
    SystemStreamPartition ssp = new SystemStreamPartition("system", "stream", new Partition(1));
    KafkaStateCheckpointMarker marker = new KafkaStateCheckpointMarker(ssp, "offset");
    KafkaStateCheckpointMarker deserializedMarker = KafkaStateCheckpointMarker.deserialize(KafkaStateCheckpointMarker.serialize(marker));
    assertEquals(MARKER_VERSION, deserializedMarker.getVersion());
    assertEquals(marker.getChangelogOffset(), deserializedMarker.getChangelogOffset());
    assertEquals(marker.getChangelogSSP(), deserializedMarker.getChangelogSSP());
    assertEquals(marker, deserializedMarker);
}

Example 10 with KafkaStateCheckpointMarker

use of org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker in project samza by apache.

the class TestKafkaStateCheckpointMarker method testSerializeDeserializeNullOffsets.

@Test
public void testSerializeDeserializeNullOffsets() {
    SystemStreamPartition ssp = new SystemStreamPartition("system", "stream", new Partition(1));
    KafkaStateCheckpointMarker marker = new KafkaStateCheckpointMarker(ssp, null);
    KafkaStateCheckpointMarker deserializedMarker = KafkaStateCheckpointMarker.deserialize(KafkaStateCheckpointMarker.serialize(marker));
    assertEquals(MARKER_VERSION, deserializedMarker.getVersion());
    assertNull(deserializedMarker.getChangelogOffset());
    assertEquals(marker.getChangelogSSP(), deserializedMarker.getChangelogSSP());
    assertEquals(marker, deserializedMarker);
}