Search in sources :

Example 1 with SnapshotIndex

use of org.apache.samza.storage.blobstore.index.SnapshotIndex in project samza by apache.

the class TestBlobStoreRestoreManager method testRestoreDeletesCheckpointDirsIfRestoring.

@Test
public void testRestoreDeletesCheckpointDirsIfRestoring() throws IOException {
    String jobName = "testJobName";
    String jobId = "testJobId";
    TaskName taskName = mock(TaskName.class);
    BlobStoreRestoreManagerMetrics metrics = new BlobStoreRestoreManagerMetrics(new MetricsRegistryMap());
    metrics.initStoreMetrics(ImmutableList.of("storeName"));
    Set<String> storesToRestore = ImmutableSet.of("storeName");
    SnapshotIndex snapshotIndex = mock(SnapshotIndex.class);
    Map<String, Pair<String, SnapshotIndex>> prevStoreSnapshotIndexes = ImmutableMap.of("storeName", Pair.of("blobId", snapshotIndex));
    DirIndex dirIndex = BlobStoreTestUtil.createDirIndex("[a]");
    when(snapshotIndex.getDirIndex()).thenReturn(dirIndex);
    CheckpointId checkpointId = CheckpointId.create();
    when(snapshotIndex.getSnapshotMetadata()).thenReturn(new SnapshotMetadata(checkpointId, "jobName", "jobId", "taskName", "storeName"));
    Path loggedBaseDir = Files.createTempDirectory(BlobStoreTestUtil.TEMP_DIR_PREFIX);
    // create store dir to be deleted during restore
    Path storeDir = Files.createTempDirectory(loggedBaseDir, "storeDir");
    Path storeCheckpointDir1 = Files.createTempDirectory(loggedBaseDir, "storeDir-" + checkpointId);
    CheckpointId olderCheckpoint = CheckpointId.create();
    Path storeCheckpointDir2 = Files.createTempDirectory(loggedBaseDir, "storeDir-" + olderCheckpoint);
    StorageConfig storageConfig = mock(StorageConfig.class);
    StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class);
    when(storageManagerUtil.getTaskStoreDir(eq(loggedBaseDir.toFile()), eq("storeName"), eq(taskName), eq(TaskMode.Active))).thenReturn(storeDir.toFile());
    when(storageManagerUtil.getStoreCheckpointDir(eq(storeDir.toFile()), eq(checkpointId))).thenReturn(Paths.get(storeDir.toString(), checkpointId.toString()).toString());
    when(storageManagerUtil.getTaskStoreCheckpointDirs(any(File.class), anyString(), any(TaskName.class), any(TaskMode.class))).thenReturn(ImmutableList.of(storeCheckpointDir1.toFile(), storeCheckpointDir2.toFile()));
    BlobStoreUtil blobStoreUtil = mock(BlobStoreUtil.class);
    DirDiffUtil dirDiffUtil = mock(DirDiffUtil.class);
    when(dirDiffUtil.areSameDir(anySet(), anyBoolean())).thenReturn((arg1, arg2) -> true);
    // return immediately without restoring.
    when(blobStoreUtil.restoreDir(eq(storeDir.toFile()), eq(dirIndex), any(Metadata.class))).thenReturn(CompletableFuture.completedFuture(null));
    BlobStoreRestoreManager.restoreStores(jobName, jobId, taskName, storesToRestore, prevStoreSnapshotIndexes, loggedBaseDir.toFile(), storageConfig, metrics, storageManagerUtil, blobStoreUtil, dirDiffUtil, EXECUTOR);
    // verify that the store directory restore was called and skipped (i.e. shouldRestore == true)
    verify(blobStoreUtil, times(1)).restoreDir(eq(storeDir.toFile()), eq(dirIndex), any(Metadata.class));
    // verify that the checkpoint directories were deleted prior to restore (should not exist at the end)
    assertFalse(storeCheckpointDir1.toFile().exists());
    assertFalse(storeCheckpointDir2.toFile().exists());
}
Also used : Path(java.nio.file.Path) SnapshotIndex(org.apache.samza.storage.blobstore.index.SnapshotIndex) StorageConfig(org.apache.samza.config.StorageConfig) StorageManagerUtil(org.apache.samza.storage.StorageManagerUtil) BlobStoreRestoreManagerMetrics(org.apache.samza.storage.blobstore.metrics.BlobStoreRestoreManagerMetrics) SnapshotMetadata(org.apache.samza.storage.blobstore.index.SnapshotMetadata) BlobStoreUtil(org.apache.samza.storage.blobstore.util.BlobStoreUtil) Mockito.anyString(org.mockito.Mockito.anyString) TaskMode(org.apache.samza.job.model.TaskMode) TaskName(org.apache.samza.container.TaskName) SnapshotMetadata(org.apache.samza.storage.blobstore.index.SnapshotMetadata) CheckpointId(org.apache.samza.checkpoint.CheckpointId) DirIndex(org.apache.samza.storage.blobstore.index.DirIndex) MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) File(java.io.File) DirDiffUtil(org.apache.samza.storage.blobstore.util.DirDiffUtil) Pair(org.apache.commons.lang3.tuple.Pair) Test(org.junit.Test)

Example 2 with SnapshotIndex

use of org.apache.samza.storage.blobstore.index.SnapshotIndex in project samza by apache.

the class TestBlobStoreRestoreManager method testRestoreDeletesStoreDir.

@Test
public void testRestoreDeletesStoreDir() throws IOException {
    String jobName = "testJobName";
    String jobId = "testJobId";
    TaskName taskName = mock(TaskName.class);
    BlobStoreRestoreManagerMetrics metrics = new BlobStoreRestoreManagerMetrics(new MetricsRegistryMap());
    metrics.initStoreMetrics(ImmutableList.of("storeName"));
    Set<String> storesToRestore = ImmutableSet.of("storeName");
    SnapshotIndex snapshotIndex = mock(SnapshotIndex.class);
    Map<String, Pair<String, SnapshotIndex>> prevStoreSnapshotIndexes = ImmutableMap.of("storeName", Pair.of("blobId", snapshotIndex));
    DirIndex dirIndex = BlobStoreTestUtil.createDirIndex("[a]");
    when(snapshotIndex.getDirIndex()).thenReturn(dirIndex);
    when(snapshotIndex.getSnapshotMetadata()).thenReturn(new SnapshotMetadata(CheckpointId.create(), "jobName", "jobId", "taskName", "storeName"));
    Path loggedBaseDir = Files.createTempDirectory(BlobStoreTestUtil.TEMP_DIR_PREFIX);
    // create store dir to be deleted during restore
    Path storeDir = Files.createTempDirectory(loggedBaseDir, "storeDir");
    StorageConfig storageConfig = mock(StorageConfig.class);
    StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class);
    when(storageManagerUtil.getStoreCheckpointDir(any(File.class), any(CheckpointId.class))).thenReturn(Paths.get(storeDir.toString(), "checkpointId").toString());
    when(storageManagerUtil.getTaskStoreDir(eq(loggedBaseDir.toFile()), eq("storeName"), eq(taskName), eq(TaskMode.Active))).thenReturn(storeDir.toFile());
    BlobStoreUtil blobStoreUtil = mock(BlobStoreUtil.class);
    DirDiffUtil dirDiffUtil = mock(DirDiffUtil.class);
    // return immediately without restoring.
    when(blobStoreUtil.restoreDir(eq(storeDir.toFile()), eq(dirIndex), any(Metadata.class))).thenReturn(CompletableFuture.completedFuture(null));
    when(dirDiffUtil.areSameDir(anySet(), anyBoolean())).thenReturn((arg1, arg2) -> true);
    BlobStoreRestoreManager.restoreStores(jobName, jobId, taskName, storesToRestore, prevStoreSnapshotIndexes, loggedBaseDir.toFile(), storageConfig, metrics, storageManagerUtil, blobStoreUtil, dirDiffUtil, EXECUTOR);
    // verify that the store directory restore was called and skipped (i.e. shouldRestore == true)
    verify(blobStoreUtil, times(1)).restoreDir(eq(storeDir.toFile()), eq(dirIndex), any(Metadata.class));
    // verify that the store directory was deleted prior to restore
    // (should still not exist at the end since restore is no-op)
    assertFalse(storeDir.toFile().exists());
}
Also used : Path(java.nio.file.Path) SnapshotIndex(org.apache.samza.storage.blobstore.index.SnapshotIndex) StorageConfig(org.apache.samza.config.StorageConfig) StorageManagerUtil(org.apache.samza.storage.StorageManagerUtil) BlobStoreRestoreManagerMetrics(org.apache.samza.storage.blobstore.metrics.BlobStoreRestoreManagerMetrics) SnapshotMetadata(org.apache.samza.storage.blobstore.index.SnapshotMetadata) BlobStoreUtil(org.apache.samza.storage.blobstore.util.BlobStoreUtil) Mockito.anyString(org.mockito.Mockito.anyString) TaskName(org.apache.samza.container.TaskName) SnapshotMetadata(org.apache.samza.storage.blobstore.index.SnapshotMetadata) CheckpointId(org.apache.samza.checkpoint.CheckpointId) DirIndex(org.apache.samza.storage.blobstore.index.DirIndex) MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) File(java.io.File) DirDiffUtil(org.apache.samza.storage.blobstore.util.DirDiffUtil) Pair(org.apache.commons.lang3.tuple.Pair) Test(org.junit.Test)

Example 3 with SnapshotIndex

use of org.apache.samza.storage.blobstore.index.SnapshotIndex in project samza by apache.

the class TestBlobStoreRestoreManager method testRestoreSkipsStoresWithMissingCheckpointSCM.

@Test
public void testRestoreSkipsStoresWithMissingCheckpointSCM() {
    // store renamed from oldStoreName to newStoreName. No SCM for newStoreName in previous checkpoint.
    String jobName = "testJobName";
    String jobId = "testJobId";
    TaskName taskName = mock(TaskName.class);
    BlobStoreRestoreManagerMetrics metrics = new BlobStoreRestoreManagerMetrics(new MetricsRegistryMap());
    metrics.initStoreMetrics(ImmutableList.of("newStoreName"));
    // new store in config
    Set<String> storesToRestore = ImmutableSet.of("newStoreName");
    SnapshotIndex snapshotIndex = mock(SnapshotIndex.class);
    Map<String, Pair<String, SnapshotIndex>> prevStoreSnapshotIndexes = mock(Map.class);
    when(prevStoreSnapshotIndexes.containsKey("newStoreName")).thenReturn(false);
    DirIndex dirIndex = mock(DirIndex.class);
    when(snapshotIndex.getDirIndex()).thenReturn(dirIndex);
    CheckpointId checkpointId = CheckpointId.create();
    when(snapshotIndex.getSnapshotMetadata()).thenReturn(new SnapshotMetadata(checkpointId, "jobName", "jobId", "taskName", "storeName"));
    Path loggedBaseDir = mock(Path.class);
    // create store dir to be deleted during restore
    StorageConfig storageConfig = mock(StorageConfig.class);
    StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class);
    BlobStoreUtil blobStoreUtil = mock(BlobStoreUtil.class);
    DirDiffUtil dirDiffUtil = mock(DirDiffUtil.class);
    BlobStoreRestoreManager.restoreStores(jobName, jobId, taskName, storesToRestore, prevStoreSnapshotIndexes, loggedBaseDir.toFile(), storageConfig, metrics, storageManagerUtil, blobStoreUtil, dirDiffUtil, EXECUTOR);
    // verify that we checked the previously checkpointed SCMs.
    verify(prevStoreSnapshotIndexes, times(1)).containsKey(eq("newStoreName"));
    // verify that the store directory restore was never called
    verify(blobStoreUtil, times(0)).restoreDir(any(File.class), any(DirIndex.class), any(Metadata.class));
}
Also used : Path(java.nio.file.Path) SnapshotIndex(org.apache.samza.storage.blobstore.index.SnapshotIndex) StorageConfig(org.apache.samza.config.StorageConfig) StorageManagerUtil(org.apache.samza.storage.StorageManagerUtil) BlobStoreRestoreManagerMetrics(org.apache.samza.storage.blobstore.metrics.BlobStoreRestoreManagerMetrics) SnapshotMetadata(org.apache.samza.storage.blobstore.index.SnapshotMetadata) BlobStoreUtil(org.apache.samza.storage.blobstore.util.BlobStoreUtil) Mockito.anyString(org.mockito.Mockito.anyString) TaskName(org.apache.samza.container.TaskName) SnapshotMetadata(org.apache.samza.storage.blobstore.index.SnapshotMetadata) CheckpointId(org.apache.samza.checkpoint.CheckpointId) DirIndex(org.apache.samza.storage.blobstore.index.DirIndex) MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) File(java.io.File) DirDiffUtil(org.apache.samza.storage.blobstore.util.DirDiffUtil) Pair(org.apache.commons.lang3.tuple.Pair) Test(org.junit.Test)

Example 4 with SnapshotIndex

use of org.apache.samza.storage.blobstore.index.SnapshotIndex in project samza by apache.

the class TestBlobStoreRestoreManager method testDeleteUnusedStoresRemovesStoresDeletedFromConfig.

@Test
public void testDeleteUnusedStoresRemovesStoresDeletedFromConfig() {
    String jobName = "testJobName";
    String jobId = "testJobId";
    String taskName = "taskName";
    StorageConfig storageConfig = mock(StorageConfig.class);
    BlobStoreConfig blobStoreConfig = mock(BlobStoreConfig.class);
    SnapshotIndex mockSnapshotIndex = mock(SnapshotIndex.class);
    String blobId = "blobId";
    Map<String, Pair<String, SnapshotIndex>> initialStoreSnapshotIndexes = ImmutableMap.of("oldStoreName", Pair.of(blobId, mockSnapshotIndex));
    when(storageConfig.getStoresWithBackupFactory(eq(BlobStoreStateBackendFactory.class.getName()))).thenReturn(ImmutableList.of("newStoreName"));
    when(storageConfig.getStoresWithRestoreFactory(eq(BlobStoreStateBackendFactory.class.getName()))).thenReturn(ImmutableList.of("newStoreName"));
    DirIndex dirIndex = mock(DirIndex.class);
    when(mockSnapshotIndex.getDirIndex()).thenReturn(dirIndex);
    BlobStoreUtil blobStoreUtil = mock(BlobStoreUtil.class);
    when(blobStoreUtil.cleanUpDir(any(DirIndex.class), any(Metadata.class))).thenReturn(CompletableFuture.completedFuture(null));
    when(blobStoreUtil.deleteDir(any(DirIndex.class), any(Metadata.class))).thenReturn(CompletableFuture.completedFuture(null));
    when(blobStoreUtil.deleteSnapshotIndexBlob(anyString(), any(Metadata.class))).thenReturn(CompletableFuture.completedFuture(null));
    BlobStoreRestoreManager.deleteUnusedStoresFromBlobStore(jobName, jobId, taskName, storageConfig, blobStoreConfig, initialStoreSnapshotIndexes, blobStoreUtil, EXECUTOR);
    verify(blobStoreUtil, times(1)).cleanUpDir(eq(dirIndex), any(Metadata.class));
    verify(blobStoreUtil, times(1)).deleteDir(eq(dirIndex), any(Metadata.class));
    verify(blobStoreUtil, times(1)).deleteSnapshotIndexBlob(eq(blobId), any(Metadata.class));
}
Also used : SnapshotIndex(org.apache.samza.storage.blobstore.index.SnapshotIndex) StorageConfig(org.apache.samza.config.StorageConfig) SnapshotMetadata(org.apache.samza.storage.blobstore.index.SnapshotMetadata) BlobStoreConfig(org.apache.samza.config.BlobStoreConfig) BlobStoreUtil(org.apache.samza.storage.blobstore.util.BlobStoreUtil) Mockito.anyString(org.mockito.Mockito.anyString) DirIndex(org.apache.samza.storage.blobstore.index.DirIndex) Pair(org.apache.commons.lang3.tuple.Pair) Test(org.junit.Test)

Example 5 with SnapshotIndex

use of org.apache.samza.storage.blobstore.index.SnapshotIndex in project samza by apache.

the class TestBlobStoreBackupManager method setup.

@Before
public void setup() throws Exception {
    when(clock.currentTimeMillis()).thenReturn(1234567L);
    // setup test local and remote snapshots
    indexBlobIdAndLocalRemoteSnapshotsPair = setupRemoteAndLocalSnapshots(true);
    // setup test store name and SCMs map
    testStoreNameAndSCMMap = setupTestStoreSCMMapAndStoreBackedFactoryConfig(indexBlobIdAndLocalRemoteSnapshotsPair);
    // setup: setup task backup manager with expected storeName->storageEngine map
    testStoreNameAndSCMMap.forEach((storeName, scm) -> storeStorageEngineMap.put(storeName, null));
    mapConfig.putAll(new MapConfig(ImmutableMap.of("job.name", jobName, "job.id", jobId)));
    Config config = new MapConfig(mapConfig);
    // Mock - return snapshot index for blob id from test blob store map
    ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);
    when(blobStoreUtil.getSnapshotIndex(captor.capture(), any(Metadata.class))).then((Answer<CompletableFuture<SnapshotIndex>>) invocation -> {
        String blobId = invocation.getArgumentAt(0, String.class);
        return CompletableFuture.completedFuture(testBlobStore.get(blobId));
    });
    // doNothing().when(blobStoreManager).init();
    when(taskModel.getTaskName().getTaskName()).thenReturn(taskName);
    when(taskModel.getTaskMode()).thenReturn(TaskMode.Active);
    when(metricsRegistry.newCounter(anyString(), anyString())).thenReturn(counter);
    when(metricsRegistry.newGauge(anyString(), anyString(), anyLong())).thenReturn(longGauge);
    when(metricsRegistry.newGauge(anyString(), anyString(), any(AtomicLong.class))).thenReturn(atomicLongGauge);
    when(atomicLongGauge.getValue()).thenReturn(new AtomicLong());
    when(metricsRegistry.newTimer(anyString(), anyString())).thenReturn(timer);
    blobStoreTaskBackupMetrics = new BlobStoreBackupManagerMetrics(metricsRegistry);
    blobStoreBackupManager = new MockBlobStoreBackupManager(jobModel, containerModel, taskModel, mockExecutor, blobStoreTaskBackupMetrics, config, Files.createTempDirectory("logged-store-").toFile(), storageManagerUtil, blobStoreManager);
}
Also used : SortedSet(java.util.SortedSet) TaskModel(org.apache.samza.job.model.TaskModel) Counter(org.apache.samza.metrics.Counter) Pair(org.apache.commons.lang3.tuple.Pair) Map(java.util.Map) StorageEngine(org.apache.samza.storage.StorageEngine) Path(java.nio.file.Path) MapConfig(org.apache.samza.config.MapConfig) TaskName(org.apache.samza.container.TaskName) ImmutableMap(com.google.common.collect.ImmutableMap) StorageManagerUtil(org.apache.samza.storage.StorageManagerUtil) Checkpoint(org.apache.samza.checkpoint.Checkpoint) MetricsRegistry(org.apache.samza.metrics.MetricsRegistry) Collectors(java.util.stream.Collectors) DirDiff(org.apache.samza.storage.blobstore.diff.DirDiff) CheckpointId(org.apache.samza.checkpoint.CheckpointId) List(java.util.List) CompletionStage(java.util.concurrent.CompletionStage) SnapshotIndex(org.apache.samza.storage.blobstore.index.SnapshotIndex) BlobStoreBackupManagerMetrics(org.apache.samza.storage.blobstore.metrics.BlobStoreBackupManagerMetrics) Optional(java.util.Optional) Config(org.apache.samza.config.Config) SnapshotMetadata(org.apache.samza.storage.blobstore.index.SnapshotMetadata) Mockito.eq(org.mockito.Mockito.eq) MoreExecutors(com.google.common.util.concurrent.MoreExecutors) DirIndex(org.apache.samza.storage.blobstore.index.DirIndex) Matchers(org.mockito.Matchers) CheckpointV2(org.apache.samza.checkpoint.CheckpointV2) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) TreeSet(java.util.TreeSet) ArrayList(java.util.ArrayList) CheckpointV1(org.apache.samza.checkpoint.CheckpointV1) Gauge(org.apache.samza.metrics.Gauge) Answer(org.mockito.stubbing.Answer) InvocationOnMock(org.mockito.invocation.InvocationOnMock) ArgumentCaptor(org.mockito.ArgumentCaptor) BlobStoreTestUtil(org.apache.samza.storage.blobstore.util.BlobStoreTestUtil) Mockito.anyLong(org.mockito.Mockito.anyLong) ExecutorService(java.util.concurrent.ExecutorService) JobModel(org.apache.samza.job.model.JobModel) Before(org.junit.Before) BlobStoreUtil(org.apache.samza.storage.blobstore.util.BlobStoreUtil) Files(java.nio.file.Files) Timer(org.apache.samza.metrics.Timer) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) Clock(org.apache.samza.util.Clock) Test(org.junit.Test) File(java.io.File) SamzaException(org.apache.samza.SamzaException) AtomicLong(java.util.concurrent.atomic.AtomicLong) Mockito(org.mockito.Mockito) TaskMode(org.apache.samza.job.model.TaskMode) ContainerModel(org.apache.samza.job.model.ContainerModel) DirDiffUtil(org.apache.samza.storage.blobstore.util.DirDiffUtil) Comparator(java.util.Comparator) Assert(org.junit.Assert) Collections(java.util.Collections) BlobStoreBackupManagerMetrics(org.apache.samza.storage.blobstore.metrics.BlobStoreBackupManagerMetrics) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicLong(java.util.concurrent.atomic.AtomicLong) MapConfig(org.apache.samza.config.MapConfig) Config(org.apache.samza.config.Config) SnapshotMetadata(org.apache.samza.storage.blobstore.index.SnapshotMetadata) MapConfig(org.apache.samza.config.MapConfig) Before(org.junit.Before)

Aggregations

SnapshotIndex (org.apache.samza.storage.blobstore.index.SnapshotIndex)20 SnapshotMetadata (org.apache.samza.storage.blobstore.index.SnapshotMetadata)18 Pair (org.apache.commons.lang3.tuple.Pair)16 DirIndex (org.apache.samza.storage.blobstore.index.DirIndex)15 Test (org.junit.Test)14 File (java.io.File)12 Path (java.nio.file.Path)12 Checkpoint (org.apache.samza.checkpoint.Checkpoint)11 CheckpointId (org.apache.samza.checkpoint.CheckpointId)10 ArrayList (java.util.ArrayList)9 SamzaException (org.apache.samza.SamzaException)9 IOException (java.io.IOException)8 HashMap (java.util.HashMap)8 CompletionStage (java.util.concurrent.CompletionStage)8 TaskName (org.apache.samza.container.TaskName)8 StorageManagerUtil (org.apache.samza.storage.StorageManagerUtil)8 CompletableFuture (java.util.concurrent.CompletableFuture)7 CheckpointV2 (org.apache.samza.checkpoint.CheckpointV2)7 Files (java.nio.file.Files)6 List (java.util.List)6