use of org.apache.samza.metrics.Timer in project samza by apache.
the class TestTransactionalStateTaskBackupManager method testFlushOrder.
@Test
public void testFlushOrder() {
ContainerStorageManager csm = mock(ContainerStorageManager.class);
StorageEngine mockStore = mock(StorageEngine.class);
java.util.Map<String, StorageEngine> taskStores = ImmutableMap.of("mockStore", mockStore);
when(csm.getAllStores(any())).thenReturn(taskStores);
when(mockStore.getStoreProperties()).thenReturn(new StoreProperties.StorePropertiesBuilder().setPersistedToDisk(true).setLoggedStore(true).build());
TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class);
Timer checkpointTimer = mock(Timer.class);
when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer);
KafkaTransactionalStateTaskBackupManager tsm = spy(buildTSM(csm, mock(Partition.class), new StorageManagerUtil()));
TaskStorageCommitManager commitManager = new TaskStorageCommitManager(new TaskName("task"), ImmutableMap.of("kafka", tsm), csm, null, null, null, null, ForkJoinPool.commonPool(), new StorageManagerUtil(), null, metrics);
// stub actual method call
doReturn(mock(java.util.Map.class)).when(tsm).getNewestChangelogSSPOffsets(any(), any(), any(), any());
// invoke Kafka flush
commitManager.init();
commitManager.snapshot(CheckpointId.create());
// ensure that stores are flushed before we get newest changelog offsets
InOrder inOrder = inOrder(mockStore, tsm);
inOrder.verify(mockStore).flush();
inOrder.verify(tsm).getNewestChangelogSSPOffsets(any(), any(), any(), any());
}
use of org.apache.samza.metrics.Timer in project samza by apache.
the class TestTaskStorageCommitManager method testPersistToFileSystemCheckpointV1AndV2Checkpoint.
@Test
public void testPersistToFileSystemCheckpointV1AndV2Checkpoint() throws IOException {
ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class);
StorageEngine mockLPStore = mock(StorageEngine.class);
StoreProperties lpStoreProps = mock(StoreProperties.class);
when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps);
when(lpStoreProps.isPersistedToDisk()).thenReturn(true);
when(lpStoreProps.isDurableStore()).thenReturn(true);
Path mockPath = mock(Path.class);
when(mockLPStore.checkpoint(any())).thenReturn(Optional.of(mockPath));
StorageEngine mockPStore = mock(StorageEngine.class);
StoreProperties pStoreProps = mock(StoreProperties.class);
when(mockPStore.getStoreProperties()).thenReturn(pStoreProps);
when(pStoreProps.isPersistedToDisk()).thenReturn(true);
when(pStoreProps.isDurableStore()).thenReturn(false);
StorageEngine mockLIStore = mock(StorageEngine.class);
StoreProperties liStoreProps = mock(StoreProperties.class);
when(mockLIStore.getStoreProperties()).thenReturn(liStoreProps);
when(liStoreProps.isPersistedToDisk()).thenReturn(false);
when(liStoreProps.isDurableStore()).thenReturn(true);
StorageEngine mockIStore = mock(StorageEngine.class);
StoreProperties iStoreProps = mock(StoreProperties.class);
when(mockIStore.getStoreProperties()).thenReturn(iStoreProps);
when(iStoreProps.isPersistedToDisk()).thenReturn(false);
when(iStoreProps.isDurableStore()).thenReturn(false);
Map<String, StorageEngine> taskStores = ImmutableMap.of("loggedPersistentStore", mockLPStore, "persistentStore", mockPStore, "loggedInMemStore", mockLIStore, "inMemStore", mockIStore);
Partition changelogPartition = new Partition(0);
SystemStream changelogSystemStream = new SystemStream("changelogSystem", "changelogStream");
SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition);
Map<String, SystemStream> storeChangelogsStreams = ImmutableMap.of("loggedPersistentStore", changelogSystemStream, "loggedInMemStore", new SystemStream("system", "stream"));
StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class);
File durableStoreDir = new File("durableStorePath");
when(storageManagerUtil.getTaskStoreDir(eq(durableStoreDir), any(), any(), any())).thenReturn(durableStoreDir);
TaskName taskName = new TaskName("task");
TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class);
Timer checkpointTimer = mock(Timer.class);
when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer);
when(containerStorageManager.getAllStores(taskName)).thenReturn(taskStores);
TaskStorageCommitManager commitManager = spy(new TaskStorageCommitManager(taskName, Collections.emptyMap(), containerStorageManager, storeChangelogsStreams, changelogPartition, null, null, ForkJoinPool.commonPool(), storageManagerUtil, durableStoreDir, metrics));
doNothing().when(commitManager).writeChangelogOffsetFile(any(), any(), any(), any());
when(storageManagerUtil.getStoreCheckpointDir(any(File.class), any(CheckpointId.class))).thenAnswer((Answer<String>) invocation -> {
File file = invocation.getArgumentAt(0, File.class);
CheckpointId checkpointId = invocation.getArgumentAt(1, CheckpointId.class);
return file + "-" + checkpointId;
});
CheckpointId newCheckpointId = CheckpointId.create();
String newestOffset = "1";
KafkaChangelogSSPOffset kafkaChangelogSSPOffset = new KafkaChangelogSSPOffset(newCheckpointId, newestOffset);
Map<SystemStreamPartition, String> offsetsJava = ImmutableMap.of(changelogSSP, kafkaChangelogSSPOffset.toString());
commitManager.init();
// invoke persist to file system for v2 checkpoint
commitManager.writeCheckpointToStoreDirectories(new CheckpointV1(offsetsJava));
verify(commitManager).writeChangelogOffsetFiles(offsetsJava);
// evoked twice, for OFFSET-V1 and OFFSET-V2
verify(commitManager).writeChangelogOffsetFile(eq("loggedPersistentStore"), eq(changelogSSP), eq(newestOffset), eq(durableStoreDir));
File checkpointFile = Paths.get(storageManagerUtil.getStoreCheckpointDir(durableStoreDir, kafkaChangelogSSPOffset.getCheckpointId())).toFile();
verify(commitManager).writeChangelogOffsetFile(eq("loggedPersistentStore"), eq(changelogSSP), eq(newestOffset), eq(checkpointFile));
Map<String, String> storeSCM = ImmutableMap.of("loggedPersistentStore", "system;loggedPersistentStoreStream;1", "persistentStore", "system;persistentStoreStream;1", "loggedInMemStore", "system;loggedInMemStoreStream;1", "inMemStore", "system;inMemStoreStream;1");
CheckpointV2 checkpoint = new CheckpointV2(newCheckpointId, Collections.emptyMap(), Collections.singletonMap("factory", storeSCM));
// invoke persist to file system for v2 checkpoint
commitManager.writeCheckpointToStoreDirectories(checkpoint);
// Validate only durable and persisted stores are persisted
// This should be evoked twice, for checkpointV1 and checkpointV2
verify(storageManagerUtil, times(2)).getTaskStoreDir(eq(durableStoreDir), eq("loggedPersistentStore"), eq(taskName), any());
File checkpointPath = Paths.get(storageManagerUtil.getStoreCheckpointDir(durableStoreDir, newCheckpointId)).toFile();
verify(storageManagerUtil).writeCheckpointV2File(eq(checkpointPath), eq(checkpoint));
}
use of org.apache.samza.metrics.Timer in project samza by apache.
the class TestTaskStorageCommitManager method testWriteChangelogOffsetFilesV2andV1.
@Test
public void testWriteChangelogOffsetFilesV2andV1() throws IOException {
Map<String, Map<SystemStreamPartition, String>> mockFileSystem = new HashMap<>();
ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class);
Map<String, CheckpointV2> mockCheckpointFileSystem = new HashMap<>();
StorageEngine mockLPStore = mock(StorageEngine.class);
StoreProperties lpStoreProps = mock(StoreProperties.class);
when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps);
when(lpStoreProps.isPersistedToDisk()).thenReturn(true);
when(lpStoreProps.isDurableStore()).thenReturn(true);
Path mockPath = mock(Path.class);
when(mockLPStore.checkpoint(any())).thenReturn(Optional.of(mockPath));
TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class);
Timer checkpointTimer = mock(Timer.class);
when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer);
java.util.Map<String, StorageEngine> taskStores = ImmutableMap.of("loggedPersistentStore", mockLPStore);
Partition changelogPartition = new Partition(0);
SystemStream changelogSystemStream = new SystemStream("changelogSystem", "changelogStream");
SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition);
java.util.Map<String, SystemStream> storeChangelogsStreams = ImmutableMap.of("loggedPersistentStore", changelogSystemStream);
StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class);
File tmpTestPath = new File("store-checkpoint-test");
when(storageManagerUtil.getTaskStoreDir(eq(tmpTestPath), eq("loggedPersistentStore"), any(), any())).thenReturn(tmpTestPath);
TaskName taskName = new TaskName("task");
when(containerStorageManager.getAllStores(taskName)).thenReturn(taskStores);
TaskStorageCommitManager commitManager = spy(new TaskStorageCommitManager(taskName, Collections.emptyMap(), containerStorageManager, storeChangelogsStreams, changelogPartition, null, null, ForkJoinPool.commonPool(), storageManagerUtil, tmpTestPath, metrics));
doAnswer(invocation -> {
String fileDir = invocation.getArgumentAt(3, File.class).getName();
SystemStreamPartition ssp = invocation.getArgumentAt(1, SystemStreamPartition.class);
String offset = invocation.getArgumentAt(2, String.class);
if (mockFileSystem.containsKey(fileDir)) {
mockFileSystem.get(fileDir).put(ssp, offset);
} else {
Map<SystemStreamPartition, String> sspOffsets = new HashMap<>();
sspOffsets.put(ssp, offset);
mockFileSystem.put(fileDir, sspOffsets);
}
return null;
}).when(commitManager).writeChangelogOffsetFile(any(), any(), any(), any());
doAnswer(invocation -> {
String storeDir = invocation.getArgumentAt(0, File.class).getName();
CheckpointV2 checkpointV2 = invocation.getArgumentAt(1, CheckpointV2.class);
mockCheckpointFileSystem.put(storeDir, checkpointV2);
return null;
}).when(storageManagerUtil).writeCheckpointV2File(any(), any());
when(storageManagerUtil.getStoreCheckpointDir(any(File.class), any(CheckpointId.class))).thenAnswer((Answer<String>) invocation -> {
File file = invocation.getArgumentAt(0, File.class);
CheckpointId checkpointId = invocation.getArgumentAt(1, CheckpointId.class);
return file + "-" + checkpointId;
});
CheckpointId newCheckpointId = CheckpointId.create();
String newestOffset = "1";
KafkaChangelogSSPOffset kafkaChangelogSSPOffset = new KafkaChangelogSSPOffset(newCheckpointId, newestOffset);
java.util.Map<SystemStreamPartition, String> offsetsJava = ImmutableMap.of(changelogSSP, kafkaChangelogSSPOffset.toString());
commitManager.init();
// invoke persist to file system for v1 checkpoint
commitManager.writeCheckpointToStoreDirectories(new CheckpointV1(offsetsJava));
assertEquals(2, mockFileSystem.size());
// check if v2 offsets are written correctly
String v2FilePath = storageManagerUtil.getStoreCheckpointDir(tmpTestPath, newCheckpointId);
assertTrue(mockFileSystem.containsKey(v2FilePath));
assertTrue(mockFileSystem.get(v2FilePath).containsKey(changelogSSP));
assertEquals(1, mockFileSystem.get(v2FilePath).size());
assertEquals(newestOffset, mockFileSystem.get(v2FilePath).get(changelogSSP));
// check if v1 offsets are written correctly
String v1FilePath = tmpTestPath.getPath();
assertTrue(mockFileSystem.containsKey(v1FilePath));
assertTrue(mockFileSystem.get(v1FilePath).containsKey(changelogSSP));
assertEquals(1, mockFileSystem.get(v1FilePath).size());
assertEquals(newestOffset, mockFileSystem.get(v1FilePath).get(changelogSSP));
java.util.Map<String, String> storeSCM = ImmutableMap.of("loggedPersistentStore", "system;loggedPersistentStoreStream;1", "persistentStore", "system;persistentStoreStream;1", "loggedInMemStore", "system;loggedInMemStoreStream;1", "inMemStore", "system;inMemStoreStream;1");
CheckpointV2 checkpoint = new CheckpointV2(newCheckpointId, Collections.emptyMap(), Collections.singletonMap("factory", storeSCM));
// invoke persist to file system with checkpoint v2
commitManager.writeCheckpointToStoreDirectories(checkpoint);
assertTrue(mockCheckpointFileSystem.containsKey(v2FilePath));
assertEquals(checkpoint, mockCheckpointFileSystem.get(v2FilePath));
assertTrue(mockCheckpointFileSystem.containsKey(v1FilePath));
assertEquals(checkpoint, mockCheckpointFileSystem.get(v1FilePath));
assertEquals(2, mockCheckpointFileSystem.size());
CheckpointV2 updatedCheckpoint = new CheckpointV2(newCheckpointId, ImmutableMap.of(new SystemStreamPartition("inputSystem", "inputStream", changelogPartition), "5"), Collections.singletonMap("factory", storeSCM));
commitManager.writeCheckpointToStoreDirectories(updatedCheckpoint);
assertEquals(updatedCheckpoint, mockCheckpointFileSystem.get(v2FilePath));
assertEquals(updatedCheckpoint, mockCheckpointFileSystem.get(v1FilePath));
assertEquals(2, mockCheckpointFileSystem.size());
}
use of org.apache.samza.metrics.Timer in project samza by apache.
the class TestBlobStoreBackupManager method setup.
@Before
public void setup() throws Exception {
when(clock.currentTimeMillis()).thenReturn(1234567L);
// setup test local and remote snapshots
indexBlobIdAndLocalRemoteSnapshotsPair = setupRemoteAndLocalSnapshots(true);
// setup test store name and SCMs map
testStoreNameAndSCMMap = setupTestStoreSCMMapAndStoreBackedFactoryConfig(indexBlobIdAndLocalRemoteSnapshotsPair);
// setup: setup task backup manager with expected storeName->storageEngine map
testStoreNameAndSCMMap.forEach((storeName, scm) -> storeStorageEngineMap.put(storeName, null));
mapConfig.putAll(new MapConfig(ImmutableMap.of("job.name", jobName, "job.id", jobId)));
Config config = new MapConfig(mapConfig);
// Mock - return snapshot index for blob id from test blob store map
ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);
when(blobStoreUtil.getSnapshotIndex(captor.capture(), any(Metadata.class))).then((Answer<CompletableFuture<SnapshotIndex>>) invocation -> {
String blobId = invocation.getArgumentAt(0, String.class);
return CompletableFuture.completedFuture(testBlobStore.get(blobId));
});
// doNothing().when(blobStoreManager).init();
when(taskModel.getTaskName().getTaskName()).thenReturn(taskName);
when(taskModel.getTaskMode()).thenReturn(TaskMode.Active);
when(metricsRegistry.newCounter(anyString(), anyString())).thenReturn(counter);
when(metricsRegistry.newGauge(anyString(), anyString(), anyLong())).thenReturn(longGauge);
when(metricsRegistry.newGauge(anyString(), anyString(), any(AtomicLong.class))).thenReturn(atomicLongGauge);
when(atomicLongGauge.getValue()).thenReturn(new AtomicLong());
when(metricsRegistry.newTimer(anyString(), anyString())).thenReturn(timer);
blobStoreTaskBackupMetrics = new BlobStoreBackupManagerMetrics(metricsRegistry);
blobStoreBackupManager = new MockBlobStoreBackupManager(jobModel, containerModel, taskModel, mockExecutor, blobStoreTaskBackupMetrics, config, Files.createTempDirectory("logged-store-").toFile(), storageManagerUtil, blobStoreManager);
}
use of org.apache.samza.metrics.Timer in project samza by apache.
the class TestTableRateLimiter method testThrottle.
@Test
public void testThrottle() {
TableRateLimiter<String, String> rateLimitHelper = getThrottler();
Timer timer = mock(Timer.class);
rateLimitHelper.setTimerMetric(timer);
int times = 0;
rateLimitHelper.throttle("foo");
verify(rateLimitHelper.rateLimiter, times(++times)).acquire(anyMapOf(String.class, Integer.class));
verify(timer, times(times)).update(anyLong());
rateLimitHelper.throttle("foo", "bar");
verify(rateLimitHelper.rateLimiter, times(++times)).acquire(anyMapOf(String.class, Integer.class));
verify(timer, times(times)).update(anyLong());
rateLimitHelper.throttle(Arrays.asList("foo", "bar"));
verify(rateLimitHelper.rateLimiter, times(++times)).acquire(anyMapOf(String.class, Integer.class));
verify(timer, times(times)).update(anyLong());
rateLimitHelper.throttle(1, 2);
verify(rateLimitHelper.rateLimiter, times(++times)).acquire(anyMapOf(String.class, Integer.class));
verify(timer, times(times)).update(anyLong());
}
Aggregations