use of org.apache.samza.system.SystemAdmins in project samza by apache.
the class TestTransactionalStateTaskRestoreManager method testGetStoreActionsForLoggedPersistentStore_RetainOneCheckpointIfMultipleCheckpointsWithSameOffset.
/**
* This can happen if no new messages were written to the store between commits. There may be more than one store
* checkpoint if container fails during commit after creating a checkpoint but before deleting the old one.
*/
@Test
public void testGetStoreActionsForLoggedPersistentStore_RetainOneCheckpointIfMultipleCheckpointsWithSameOffset() {
TaskModel mockTaskModel = mock(TaskModel.class);
TaskName taskName = new TaskName("Partition 0");
when(mockTaskModel.getTaskName()).thenReturn(taskName);
Partition taskChangelogPartition = new Partition(0);
when(mockTaskModel.getChangelogPartition()).thenReturn(taskChangelogPartition);
String store1Name = "store1";
StorageEngine store1Engine = mock(StorageEngine.class);
StoreProperties mockStore1Properties = mock(StoreProperties.class);
when(store1Engine.getStoreProperties()).thenReturn(mockStore1Properties);
when(mockStore1Properties.isLoggedStore()).thenReturn(true);
when(mockStore1Properties.isPersistedToDisk()).thenReturn(true);
Map<String, StorageEngine> mockStoreEngines = ImmutableMap.of(store1Name, store1Engine);
String changelog1SystemName = "system1";
String changelog1StreamName = "store1Changelog";
SystemStream changelog1SystemStream = new SystemStream(changelog1SystemName, changelog1StreamName);
SystemStreamPartition changelog1SSP = new SystemStreamPartition(changelog1SystemStream, taskChangelogPartition);
SystemStreamPartitionMetadata changelog1SSPMetadata = new SystemStreamPartitionMetadata("0", "10", "11");
Map<String, SystemStream> mockStoreChangelogs = ImmutableMap.of(store1Name, changelog1SystemStream);
String changelog1CheckpointedOffset = "5";
CheckpointId checkpointId = CheckpointId.create();
KafkaStateCheckpointMarker kafkaStateCheckpointMarker = new KafkaStateCheckpointMarker(changelog1SSP, changelog1CheckpointedOffset);
ImmutableMap<String, KafkaStateCheckpointMarker> mockCheckpointedChangelogOffset = ImmutableMap.of(store1Name, kafkaStateCheckpointMarker);
Map<SystemStreamPartition, SystemStreamPartitionMetadata> mockCurrentChangelogOffsets = ImmutableMap.of(changelog1SSP, changelog1SSPMetadata);
SystemAdmins mockSystemAdmins = mock(SystemAdmins.class);
SystemAdmin mockSystemAdmin = mock(SystemAdmin.class);
when(mockSystemAdmins.getSystemAdmin(changelog1SSP.getSystem())).thenReturn(mockSystemAdmin);
StorageManagerUtil mockStorageManagerUtil = mock(StorageManagerUtil.class);
File mockLoggedStoreBaseDir = mock(File.class);
File mockNonLoggedStoreBaseDir = mock(File.class);
Config mockConfig = mock(Config.class);
Clock mockClock = mock(Clock.class);
File mockCurrentStoreDir = mock(File.class);
File mockStoreNewerCheckpointDir = mock(File.class);
File mockStoreOlderCheckpointDir = mock(File.class);
when(mockStorageManagerUtil.getTaskStoreDir(eq(mockLoggedStoreBaseDir), eq(store1Name), eq(taskName), any())).thenReturn(mockCurrentStoreDir);
when(mockStorageManagerUtil.getTaskStoreCheckpointDirs(eq(mockLoggedStoreBaseDir), eq(store1Name), eq(taskName), any())).thenReturn(ImmutableList.of(mockStoreNewerCheckpointDir, mockStoreOlderCheckpointDir));
when(mockStorageManagerUtil.isLoggedStoreValid(eq(store1Name), eq(mockStoreNewerCheckpointDir), any(), eq(mockStoreChangelogs), eq(mockTaskModel), any(), eq(mockStoreEngines))).thenReturn(true);
when(mockStorageManagerUtil.isLoggedStoreValid(eq(store1Name), eq(mockStoreOlderCheckpointDir), any(), eq(mockStoreChangelogs), eq(mockTaskModel), any(), eq(mockStoreEngines))).thenReturn(true);
Set<SystemStreamPartition> mockChangelogSSPs = ImmutableSet.of(changelog1SSP);
when(mockStorageManagerUtil.readOffsetFile(eq(mockStoreNewerCheckpointDir), eq(mockChangelogSSPs), eq(false))).thenReturn(// equal to checkpointed offset (5)
ImmutableMap.of(changelog1SSP, changelog1CheckpointedOffset));
when(mockStorageManagerUtil.readOffsetFile(eq(mockStoreOlderCheckpointDir), eq(mockChangelogSSPs), eq(false))).thenReturn(// also equal to checkpointed offset (5)
ImmutableMap.of(changelog1SSP, changelog1CheckpointedOffset));
Mockito.when(mockSystemAdmin.offsetComparator(anyString(), anyString())).thenAnswer((Answer<Integer>) invocation -> {
String offset1 = (String) invocation.getArguments()[0];
String offset2 = (String) invocation.getArguments()[1];
return Long.valueOf(offset1).compareTo(Long.valueOf(offset2));
});
StoreActions storeActions = TransactionalStateTaskRestoreManager.getStoreActions(mockTaskModel, mockStoreEngines, mockStoreChangelogs, mockCheckpointedChangelogOffset, checkpointId, mockCurrentChangelogOffsets, mockSystemAdmins, mockStorageManagerUtil, mockLoggedStoreBaseDir, mockNonLoggedStoreBaseDir, mockConfig, mockClock);
// ensure that both the current dir and one of the checkpoint dirs are marked for deletion
assertEquals(2, storeActions.storeDirsToDelete.get(store1Name).size());
assertTrue(storeActions.storeDirsToDelete.get(store1Name).contains(mockCurrentStoreDir));
// ensure that the one of the store checkpoint is marked for retention
assertNotNull(storeActions.storeDirsToRetain.get(store1Name));
// ensure that we mark the store for restore even if local offset == checkpointed offset
// this is required since there may be message we need to trim
assertEquals(changelog1CheckpointedOffset, storeActions.storesToRestore.get(store1Name).startingOffset);
assertEquals(changelog1CheckpointedOffset, storeActions.storesToRestore.get(store1Name).endingOffset);
}
use of org.apache.samza.system.SystemAdmins in project samza by apache.
the class TestLocalApplicationRunner method testCreateCoordinatorStreamWithCoordinatorFactory.
/**
* Underlying coordinator stream should be created if using CoordinatorStreamMetadataStoreFactory
*/
@Test
public void testCreateCoordinatorStreamWithCoordinatorFactory() throws Exception {
CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
CoordinatorStreamMetadataStoreFactory coordinatorStreamMetadataStoreFactory = mock(CoordinatorStreamMetadataStoreFactory.class);
doReturn(coordinatorStreamStore).when(coordinatorStreamMetadataStoreFactory).getMetadataStore(anyString(), any(Config.class), any(MetricsRegistry.class));
SystemAdmins systemAdmins = mock(SystemAdmins.class);
PowerMockito.whenNew(SystemAdmins.class).withAnyArguments().thenReturn(systemAdmins);
LocalApplicationRunner localApplicationRunner = spy(new LocalApplicationRunner(mockApp, config, coordinatorStreamMetadataStoreFactory));
// create store only if successful in creating the underlying coordinator stream
doReturn(true).when(localApplicationRunner).createUnderlyingCoordinatorStream(eq(config));
assertEquals(coordinatorStreamStore, localApplicationRunner.createCoordinatorStreamStore(config));
verify(localApplicationRunner).createUnderlyingCoordinatorStream(eq(config));
// do not create store if creating the underlying coordinator stream fails
doReturn(false).when(localApplicationRunner).createUnderlyingCoordinatorStream(eq(config));
assertNull(localApplicationRunner.createCoordinatorStreamStore(config));
}
use of org.apache.samza.system.SystemAdmins in project samza by apache.
the class StaticResourceJobCoordinatorFactory method getJobCoordinator.
@Override
public JobCoordinator getJobCoordinator(String processorId, Config config, MetricsRegistry metricsRegistry, MetadataStore metadataStore) {
JobInfoServingContext jobModelServingContext = new JobInfoServingContext();
JobConfig jobConfig = new JobConfig(config);
CoordinatorCommunicationContext context = new CoordinatorCommunicationContext(jobModelServingContext, config, metricsRegistry);
CoordinatorCommunication coordinatorCommunication = new HttpCoordinatorToWorkerCommunicationFactory().coordinatorCommunication(context);
JobCoordinatorMetadataManager jobCoordinatorMetadataManager = new JobCoordinatorMetadataManager(new NamespaceAwareCoordinatorStreamStore(metadataStore, SetJobCoordinatorMetadataMessage.TYPE), JobCoordinatorMetadataManager.ClusterType.NON_YARN, metricsRegistry);
ChangelogStreamManager changelogStreamManager = new ChangelogStreamManager(new NamespaceAwareCoordinatorStreamStore(metadataStore, SetChangelogMapping.TYPE));
JobRestartSignal jobRestartSignal = ReflectionUtil.getObj(new JobCoordinatorConfig(config).getJobRestartSignalFactory(), JobRestartSignalFactory.class).build(new JobRestartSignalFactoryContext(config));
Optional<StartpointManager> startpointManager = jobConfig.getStartpointEnabled() ? Optional.of(new StartpointManager(metadataStore)) : Optional.empty();
SystemAdmins systemAdmins = new SystemAdmins(config, StaticResourceJobCoordinator.class.getSimpleName());
StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance());
JobModelHelper jobModelHelper = buildJobModelHelper(metadataStore, streamMetadataCache);
StreamPartitionCountMonitorFactory streamPartitionCountMonitorFactory = new StreamPartitionCountMonitorFactory(streamMetadataCache, metricsRegistry);
StreamRegexMonitorFactory streamRegexMonitorFactory = new StreamRegexMonitorFactory(streamMetadataCache, metricsRegistry);
Optional<String> executionEnvContainerId = Optional.ofNullable(System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID));
Optional<String> samzaEpochId = Optional.ofNullable(System.getenv(EnvironmentVariables.SAMZA_EPOCH_ID));
return new StaticResourceJobCoordinator(processorId, jobModelHelper, jobModelServingContext, coordinatorCommunication, jobCoordinatorMetadataManager, streamPartitionCountMonitorFactory, streamRegexMonitorFactory, startpointManager, changelogStreamManager, jobRestartSignal, metricsRegistry, systemAdmins, executionEnvContainerId, samzaEpochId, config);
}
use of org.apache.samza.system.SystemAdmins in project samza by apache.
the class TestTransactionalStateTaskBackupManager method testGetNewestOffsetsThrowsIfErrorGettingMetadata.
@Test(expected = SamzaException.class)
public void testGetNewestOffsetsThrowsIfErrorGettingMetadata() {
// empty topic == null newest offset
ContainerStorageManager csm = mock(ContainerStorageManager.class);
KafkaTransactionalStateTaskBackupManager tsm = buildTSM(csm, mock(Partition.class), new StorageManagerUtil());
TaskName taskName = mock(TaskName.class);
String changelogSystemName = "systemName";
String storeName = "storeName";
String changelogStreamName = "changelogName";
String newestChangelogSSPOffset = null;
SystemStream changelogSystemStream = new SystemStream(changelogSystemName, changelogStreamName);
Partition changelogPartition = new Partition(0);
SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition);
java.util.Map<String, SystemStream> storeChangelogs = new HashMap<>();
storeChangelogs.put(storeName, changelogSystemStream);
SystemAdmins systemAdmins = mock(SystemAdmins.class);
SystemAdmin systemAdmin = mock(SystemAdmin.class);
SystemStreamPartitionMetadata metadata = mock(SystemStreamPartitionMetadata.class);
when(metadata.getNewestOffset()).thenReturn(newestChangelogSSPOffset);
when(systemAdmins.getSystemAdmin(changelogSystemName)).thenThrow(new SamzaException("Error getting metadata"));
when(systemAdmin.getSSPMetadata(eq(ImmutableSet.of(changelogSSP)))).thenReturn(null);
// invoke the method
java.util.Map<String, String> offsets = tsm.getNewestChangelogSSPOffsets(taskName, storeChangelogs, changelogPartition, systemAdmins);
// verify results
fail("Should have thrown an exception if admin had an error getting metadata");
}
use of org.apache.samza.system.SystemAdmins in project samza by apache.
the class TestTransactionalStateTaskBackupManager method testGetNewestOffsetsReturnsCorrectOffset.
@Test
public void testGetNewestOffsetsReturnsCorrectOffset() {
ContainerStorageManager csm = mock(ContainerStorageManager.class);
KafkaTransactionalStateTaskBackupManager tsm = buildTSM(csm, mock(Partition.class), new StorageManagerUtil());
TaskName taskName = mock(TaskName.class);
String changelogSystemName = "systemName";
String storeName = "storeName";
String changelogStreamName = "changelogName";
String newestChangelogSSPOffset = "1";
SystemStream changelogSystemStream = new SystemStream(changelogSystemName, changelogStreamName);
Partition changelogPartition = new Partition(0);
SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition);
java.util.Map<String, SystemStream> storeChangelogs = new HashMap<>();
storeChangelogs.put(storeName, changelogSystemStream);
SystemAdmins systemAdmins = mock(SystemAdmins.class);
SystemAdmin systemAdmin = mock(SystemAdmin.class);
SystemStreamPartitionMetadata metadata = mock(SystemStreamPartitionMetadata.class);
when(metadata.getNewestOffset()).thenReturn(newestChangelogSSPOffset);
when(systemAdmins.getSystemAdmin(changelogSystemName)).thenReturn(systemAdmin);
when(systemAdmin.getSSPMetadata(eq(ImmutableSet.of(changelogSSP)))).thenReturn(ImmutableMap.of(changelogSSP, metadata));
// invoke the method
java.util.Map<String, String> stateCheckpointMarkerMap = tsm.getNewestChangelogSSPOffsets(taskName, storeChangelogs, changelogPartition, systemAdmins);
// verify results
assertEquals(1, stateCheckpointMarkerMap.size());
KafkaStateCheckpointMarker kscm = KafkaStateCheckpointMarker.deserialize(stateCheckpointMarkerMap.get(storeName));
assertEquals(newestChangelogSSPOffset, kscm.getChangelogOffset());
assertEquals(changelogSSP, kscm.getChangelogSSP());
}
Aggregations