use of org.apache.samza.metrics.Timer in project samza by apache.
the class TestTaskStorageCommitManager method testSnapshotFailsIfErrorCreatingCheckpoint.
@Test(expected = IllegalStateException.class)
public void testSnapshotFailsIfErrorCreatingCheckpoint() {
CheckpointManager checkpointManager = mock(CheckpointManager.class);
TaskBackupManager taskBackupManager1 = mock(TaskBackupManager.class);
TaskBackupManager taskBackupManager2 = mock(TaskBackupManager.class);
ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class);
StorageEngine mockLPStore = mock(StorageEngine.class);
StoreProperties lpStoreProps = mock(StoreProperties.class);
when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps);
when(lpStoreProps.isPersistedToDisk()).thenReturn(true);
when(lpStoreProps.isDurableStore()).thenReturn(true);
when(mockLPStore.checkpoint(any())).thenThrow(new IllegalStateException());
TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class);
Timer checkpointTimer = mock(Timer.class);
when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer);
TaskName taskName = new TaskName("task1");
Map<String, TaskBackupManager> backupManagers = ImmutableMap.of("factory1", taskBackupManager1, "factory2", taskBackupManager2);
Map<String, StorageEngine> storageEngines = ImmutableMap.of("storeLP", mockLPStore);
TaskStorageCommitManager cm = new TaskStorageCommitManager(taskName, backupManagers, containerStorageManager, Collections.emptyMap(), new Partition(1), checkpointManager, new MapConfig(), ForkJoinPool.commonPool(), new StorageManagerUtil(), null, metrics);
when(containerStorageManager.getAllStores(taskName)).thenReturn(storageEngines);
CheckpointId newCheckpointId = CheckpointId.create();
cm.init();
cm.snapshot(newCheckpointId);
// Assert stores where flushed
verify(mockLPStore).flush();
// only logged and persisted stores are checkpointed
verify(mockLPStore).checkpoint(newCheckpointId);
verify(taskBackupManager1, never()).snapshot(any());
verify(taskBackupManager2, never()).snapshot(any());
verify(taskBackupManager1, never()).upload(any(), any());
verify(taskBackupManager2, never()).upload(any(), any());
fail("Should have thrown an exception when the storageEngine#checkpoint did not succeed");
}
use of org.apache.samza.metrics.Timer in project samza by apache.
the class TestTaskStorageCommitManager method testFlushAndCheckpointOnSnapshot.
@Test
public void testFlushAndCheckpointOnSnapshot() {
CheckpointManager checkpointManager = mock(CheckpointManager.class);
TaskBackupManager taskBackupManager1 = mock(TaskBackupManager.class);
TaskBackupManager taskBackupManager2 = mock(TaskBackupManager.class);
ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class);
Checkpoint checkpoint = mock(Checkpoint.class);
StorageEngine mockLPStore = mock(StorageEngine.class);
StoreProperties lpStoreProps = mock(StoreProperties.class);
when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps);
when(lpStoreProps.isPersistedToDisk()).thenReturn(true);
when(lpStoreProps.isDurableStore()).thenReturn(true);
Path mockPath = mock(Path.class);
when(mockLPStore.checkpoint(any())).thenReturn(Optional.of(mockPath));
StorageEngine mockPStore = mock(StorageEngine.class);
StoreProperties pStoreProps = mock(StoreProperties.class);
when(mockPStore.getStoreProperties()).thenReturn(pStoreProps);
when(pStoreProps.isPersistedToDisk()).thenReturn(true);
when(pStoreProps.isDurableStore()).thenReturn(false);
StorageEngine mockLIStore = mock(StorageEngine.class);
StoreProperties liStoreProps = mock(StoreProperties.class);
when(mockLIStore.getStoreProperties()).thenReturn(liStoreProps);
when(liStoreProps.isPersistedToDisk()).thenReturn(false);
when(liStoreProps.isDurableStore()).thenReturn(true);
StorageEngine mockIStore = mock(StorageEngine.class);
StoreProperties iStoreProps = mock(StoreProperties.class);
when(mockIStore.getStoreProperties()).thenReturn(iStoreProps);
when(iStoreProps.isPersistedToDisk()).thenReturn(false);
when(iStoreProps.isDurableStore()).thenReturn(false);
TaskName taskName = new TaskName("task1");
Map<String, TaskBackupManager> backupManagers = ImmutableMap.of("factory1", taskBackupManager1, "factory2", taskBackupManager2);
Map<String, StorageEngine> storageEngines = ImmutableMap.of("storeLP", mockLPStore, "storeP", mockPStore, "storeLI", mockLIStore, "storeI", mockIStore);
TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class);
Timer checkpointTimer = mock(Timer.class);
when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer);
TaskStorageCommitManager cm = new TaskStorageCommitManager(taskName, backupManagers, containerStorageManager, Collections.emptyMap(), new Partition(1), checkpointManager, new MapConfig(), ForkJoinPool.commonPool(), new StorageManagerUtil(), null, metrics);
when(checkpointManager.readLastCheckpoint(taskName)).thenReturn(checkpoint);
cm.init();
verify(taskBackupManager1).init(eq(checkpoint));
verify(taskBackupManager2).init(eq(checkpoint));
CheckpointId newCheckpointId = CheckpointId.create();
Map<String, String> factory1Checkpoints = ImmutableMap.of("store1", "system;stream;1", "store2", "system;stream;2");
Map<String, String> factory2Checkpoints = ImmutableMap.of("store1", "blobId1", "store2", "blobId2");
when(containerStorageManager.getAllStores(taskName)).thenReturn(storageEngines);
when(taskBackupManager1.snapshot(newCheckpointId)).thenReturn(factory1Checkpoints);
when(taskBackupManager1.upload(newCheckpointId, factory1Checkpoints)).thenReturn(CompletableFuture.completedFuture(factory1Checkpoints));
when(taskBackupManager2.snapshot(newCheckpointId)).thenReturn(factory2Checkpoints);
when(taskBackupManager2.upload(newCheckpointId, factory2Checkpoints)).thenReturn(CompletableFuture.completedFuture(factory2Checkpoints));
when(mockLIStore.checkpoint(newCheckpointId)).thenReturn(Optional.empty());
cm.init();
cm.snapshot(newCheckpointId);
// Assert stores where flushed
verify(mockIStore).flush();
verify(mockPStore).flush();
verify(mockLIStore).flush();
verify(mockLPStore).flush();
// only logged and persisted stores are checkpointed
verify(mockLPStore).checkpoint(newCheckpointId);
// ensure that checkpoint is never called for non-logged persistent stores since they're
// always cleared on restart.
verify(mockPStore, never()).checkpoint(any());
// ensure that checkpoint is never called for non-persistent stores
verify(mockIStore, never()).checkpoint(any());
verify(mockLIStore, never()).checkpoint(any());
verify(checkpointTimer).update(anyLong());
}
use of org.apache.samza.metrics.Timer in project samza by apache.
the class TestTaskStorageCommitManager method testWriteChangelogOffsetFilesWithEmptyChangelogTopic.
@Test
public void testWriteChangelogOffsetFilesWithEmptyChangelogTopic() throws IOException {
Map<String, Map<SystemStreamPartition, String>> mockFileSystem = new HashMap<>();
ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class);
StorageEngine mockLPStore = mock(StorageEngine.class);
StoreProperties lpStoreProps = mock(StoreProperties.class);
when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps);
when(lpStoreProps.isPersistedToDisk()).thenReturn(true);
when(lpStoreProps.isDurableStore()).thenReturn(true);
Path mockPath = mock(Path.class);
when(mockLPStore.checkpoint(any())).thenReturn(Optional.of(mockPath));
TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class);
Timer checkpointTimer = mock(Timer.class);
when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer);
java.util.Map<String, StorageEngine> taskStores = ImmutableMap.of("loggedPersistentStore", mockLPStore);
Partition changelogPartition = new Partition(0);
SystemStream changelogSystemStream = new SystemStream("changelogSystem", "changelogStream");
SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition);
java.util.Map<String, SystemStream> storeChangelogsStreams = ImmutableMap.of("loggedPersistentStore", changelogSystemStream);
StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class);
File tmpTestPath = new File("store-checkpoint-test");
when(storageManagerUtil.getTaskStoreDir(eq(tmpTestPath), any(), any(), any())).thenReturn(tmpTestPath);
TaskName taskName = new TaskName("task");
when(containerStorageManager.getAllStores(taskName)).thenReturn(taskStores);
TaskStorageCommitManager commitManager = spy(new TaskStorageCommitManager(taskName, Collections.emptyMap(), containerStorageManager, storeChangelogsStreams, changelogPartition, null, null, ForkJoinPool.commonPool(), storageManagerUtil, tmpTestPath, metrics));
doAnswer(invocation -> {
String storeName = invocation.getArgumentAt(0, String.class);
String fileDir = invocation.getArgumentAt(3, File.class).getName();
String mockKey = storeName + fileDir;
SystemStreamPartition ssp = invocation.getArgumentAt(1, SystemStreamPartition.class);
String offset = invocation.getArgumentAt(2, String.class);
if (mockFileSystem.containsKey(mockKey)) {
mockFileSystem.get(mockKey).put(ssp, offset);
} else {
Map<SystemStreamPartition, String> sspOffsets = new HashMap<>();
sspOffsets.put(ssp, offset);
mockFileSystem.put(mockKey, sspOffsets);
}
return null;
}).when(commitManager).writeChangelogOffsetFile(any(), any(), any(), any());
CheckpointId newCheckpointId = CheckpointId.create();
String newestOffset = null;
KafkaChangelogSSPOffset kafkaChangelogSSPOffset = new KafkaChangelogSSPOffset(newCheckpointId, newestOffset);
java.util.Map<SystemStreamPartition, String> offsetsJava = ImmutableMap.of(changelogSSP, kafkaChangelogSSPOffset.toString());
commitManager.init();
// invoke persist to file system for v2 checkpoint
commitManager.writeCheckpointToStoreDirectories(new CheckpointV1(offsetsJava));
assertTrue(mockFileSystem.isEmpty());
// verify that delete was called on current store dir offset file
verify(storageManagerUtil, times(1)).deleteOffsetFile(eq(tmpTestPath));
}
use of org.apache.samza.metrics.Timer in project samza by apache.
the class TestTaskStorageCommitManager method testWriteChangelogOffsetFilesV1.
@Test
public void testWriteChangelogOffsetFilesV1() throws IOException {
Map<String, Map<SystemStreamPartition, String>> mockFileSystem = new HashMap<>();
ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class);
StorageEngine mockLPStore = mock(StorageEngine.class);
StoreProperties lpStoreProps = mock(StoreProperties.class);
when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps);
when(lpStoreProps.isPersistedToDisk()).thenReturn(true);
when(lpStoreProps.isDurableStore()).thenReturn(true);
Path mockPath = mock(Path.class);
when(mockLPStore.checkpoint(any())).thenReturn(Optional.of(mockPath));
TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class);
Timer checkpointTimer = mock(Timer.class);
when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer);
java.util.Map<String, StorageEngine> taskStores = ImmutableMap.of("loggedPersistentStore", mockLPStore);
Partition changelogPartition = new Partition(0);
SystemStream changelogSystemStream = new SystemStream("changelogSystem", "changelogStream");
SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition);
java.util.Map<String, SystemStream> storeChangelogsStreams = ImmutableMap.of("loggedPersistentStore", changelogSystemStream);
StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class);
File tmpTestPath = new File("store-checkpoint-test");
when(storageManagerUtil.getTaskStoreDir(eq(tmpTestPath), eq("loggedPersistentStore"), any(), any())).thenReturn(tmpTestPath);
TaskName taskName = new TaskName("task");
when(containerStorageManager.getAllStores(taskName)).thenReturn(taskStores);
TaskStorageCommitManager commitManager = spy(new TaskStorageCommitManager(taskName, Collections.emptyMap(), containerStorageManager, storeChangelogsStreams, changelogPartition, null, null, ForkJoinPool.commonPool(), storageManagerUtil, tmpTestPath, metrics));
when(storageManagerUtil.getStoreCheckpointDir(any(File.class), any(CheckpointId.class))).thenAnswer((Answer<String>) invocation -> {
File file = invocation.getArgumentAt(0, File.class);
CheckpointId checkpointId = invocation.getArgumentAt(1, CheckpointId.class);
return file + "-" + checkpointId;
});
doAnswer(invocation -> {
String fileDir = invocation.getArgumentAt(3, File.class).getName();
SystemStreamPartition ssp = invocation.getArgumentAt(1, SystemStreamPartition.class);
String offset = invocation.getArgumentAt(2, String.class);
if (mockFileSystem.containsKey(fileDir)) {
mockFileSystem.get(fileDir).put(ssp, offset);
} else {
Map<SystemStreamPartition, String> sspOffsets = new HashMap<>();
sspOffsets.put(ssp, offset);
mockFileSystem.put(fileDir, sspOffsets);
}
return null;
}).when(commitManager).writeChangelogOffsetFile(any(), any(), any(), any());
CheckpointId newCheckpointId = CheckpointId.create();
String newestOffset = "1";
KafkaChangelogSSPOffset kafkaChangelogSSPOffset = new KafkaChangelogSSPOffset(newCheckpointId, newestOffset);
java.util.Map<SystemStreamPartition, String> offsetsJava = ImmutableMap.of(changelogSSP, kafkaChangelogSSPOffset.toString());
commitManager.init();
// invoke persist to file system for v2 checkpoint
commitManager.writeCheckpointToStoreDirectories(new CheckpointV1(offsetsJava));
assertEquals(2, mockFileSystem.size());
// check if v2 offsets are written correctly
String v2FilePath = storageManagerUtil.getStoreCheckpointDir(tmpTestPath, newCheckpointId);
assertTrue(mockFileSystem.containsKey(v2FilePath));
assertTrue(mockFileSystem.get(v2FilePath).containsKey(changelogSSP));
assertEquals(1, mockFileSystem.get(v2FilePath).size());
assertEquals(newestOffset, mockFileSystem.get(v2FilePath).get(changelogSSP));
// check if v1 offsets are written correctly
String v1FilePath = tmpTestPath.getPath();
assertTrue(mockFileSystem.containsKey(v1FilePath));
assertTrue(mockFileSystem.get(v1FilePath).containsKey(changelogSSP));
assertEquals(1, mockFileSystem.get(v1FilePath).size());
assertEquals(newestOffset, mockFileSystem.get(v1FilePath).get(changelogSSP));
}
Aggregations