Examples with DirIndex org.apache.samza.storage.blobstore.index.DirIndex used on opensource projects

Example 36 with DirIndex

use of org.apache.samza.storage.blobstore.index.DirIndex in project samza by apache.

the class TestSnapshotIndexSerde method testSnapshotIndexSerde.

@Test
public void testSnapshotIndexSerde() throws IOException {
    // create local and remote snapshots
    String local = "[a, b, c/1, d/1/2]";
    String remote = "[a, b, z, c/1/2, e/1]";
    Path localSnapshot = BlobStoreTestUtil.createLocalDir(local);
    DirIndex dirIndex = BlobStoreTestUtil.createDirIndex(remote);
    SnapshotMetadata snapshotMetadata = new SnapshotMetadata(CheckpointId.create(), "job", "123", "task", "store");
    SnapshotIndex testRemoteSnapshot = new SnapshotIndex(System.currentTimeMillis(), snapshotMetadata, dirIndex, Optional.empty());
    SnapshotIndexSerde snapshotIndexSerde = new SnapshotIndexSerde();
    byte[] serialized = snapshotIndexSerde.toBytes(testRemoteSnapshot);
    SnapshotIndex deserialized = snapshotIndexSerde.fromBytes(serialized);
    Assert.assertNotNull(deserialized);
    Assert.assertEquals(deserialized, testRemoteSnapshot);
}

Example 37 with DirIndex

use of org.apache.samza.storage.blobstore.index.DirIndex in project samza by apache.

the class TestDirDiffUtil method testGetDirDiff.

@Test
public void testGetDirDiff() throws IOException {
    // Setup
    Path localSnapshotDir = BlobStoreTestUtil.createLocalDir(this.local);
    String basePath = localSnapshotDir.toAbsolutePath().toString();
    DirIndex remoteSnapshotDir = BlobStoreTestUtil.createDirIndex(this.remote);
    // Execute
    DirDiff dirDiff = DirDiffUtil.getDirDiff(localSnapshotDir.toFile(), remoteSnapshotDir, (localFile, remoteFile) -> localFile.getName().equals(remoteFile.getFileName()));
    SortedSet<String> allAdded = new TreeSet<>();
    SortedSet<String> allRemoved = new TreeSet<>();
    SortedSet<String> allRetained = new TreeSet<>();
    BlobStoreTestUtil.getAllAddedInDiff(basePath, dirDiff, allAdded);
    BlobStoreTestUtil.getAllRemovedInDiff("", dirDiff, allRemoved);
    BlobStoreTestUtil.getAllRetainedInDiff("", dirDiff, allRetained);
    // Assert
    SortedSet<String> expectedAddedFiles = BlobStoreTestUtil.getExpected(this.expectedAdded);
    SortedSet<String> expectedRetainedFiles = BlobStoreTestUtil.getExpected(this.expectedRetained);
    SortedSet<String> expectedRemovedFiles = BlobStoreTestUtil.getExpected(this.expectedRemoved);
    assertEquals(expectedAddedFiles, allAdded);
    assertEquals(expectedRetainedFiles, allRetained);
    assertEquals(expectedRemovedFiles, allRemoved);
}

Example 38 with DirIndex

use of org.apache.samza.storage.blobstore.index.DirIndex in project samza by apache.

the class TestBlobStoreRestoreManager method testShouldRestoreIfCleanStateOnRestartEnabled.

@Test
public void testShouldRestoreIfCleanStateOnRestartEnabled() throws IOException {
    String taskName = "taskName";
    String storeName = "storeName";
    DirIndex dirIndex = mock(DirIndex.class);
    // must exist
    Path storeCheckpointDir = Files.createTempDirectory(BlobStoreTestUtil.TEMP_DIR_PREFIX);
    StorageConfig storageConfig = mock(StorageConfig.class);
    // clean on restart
    when(storageConfig.cleanLoggedStoreDirsOnStart(anyString())).thenReturn(true);
    DirDiffUtil dirDiffUtil = mock(DirDiffUtil.class);
    boolean shouldRestore = BlobStoreRestoreManager.shouldRestore(taskName, storeName, dirIndex, storeCheckpointDir, storageConfig, dirDiffUtil);
    verifyZeroInteractions(dirDiffUtil);
    // should not restore, should retain checkpoint dir instead
    assertTrue(shouldRestore);
}

Example 39 with DirIndex

use of org.apache.samza.storage.blobstore.index.DirIndex in project samza by apache.

the class TestBlobStoreRestoreManager method testRestoreRetainsCheckpointDirsIfValid.

@Test
public void testRestoreRetainsCheckpointDirsIfValid() throws IOException {
    String jobName = "testJobName";
    String jobId = "testJobId";
    TaskName taskName = mock(TaskName.class);
    BlobStoreRestoreManagerMetrics metrics = new BlobStoreRestoreManagerMetrics(new MetricsRegistryMap());
    metrics.initStoreMetrics(ImmutableList.of("storeName"));
    Set<String> storesToRestore = ImmutableSet.of("storeName");
    SnapshotIndex snapshotIndex = mock(SnapshotIndex.class);
    Map<String, Pair<String, SnapshotIndex>> prevStoreSnapshotIndexes = ImmutableMap.of("storeName", Pair.of("blobId", snapshotIndex));
    DirIndex dirIndex = BlobStoreTestUtil.createDirIndex("[a]");
    when(snapshotIndex.getDirIndex()).thenReturn(dirIndex);
    CheckpointId checkpointId = CheckpointId.create();
    when(snapshotIndex.getSnapshotMetadata()).thenReturn(new SnapshotMetadata(checkpointId, "jobName", "jobId", "taskName", "storeName"));
    Path loggedBaseDir = Files.createTempDirectory(BlobStoreTestUtil.TEMP_DIR_PREFIX);
    // create store dir to be deleted during restore
    Path storeDir = Files.createTempDirectory(loggedBaseDir, "storeDir-");
    // create checkpoint dir so that shouldRestore = false (areSameDir == true later)
    Path storeCheckpointDir = Files.createTempDirectory(loggedBaseDir, "storeDir-" + checkpointId + "-");
    // create a dummy file to verify after dir rename.
    Path tempFile = Files.createTempFile(storeCheckpointDir, "tempFile-", null);
    StorageConfig storageConfig = mock(StorageConfig.class);
    StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class);
    when(storageManagerUtil.getTaskStoreDir(eq(loggedBaseDir.toFile()), eq("storeName"), eq(taskName), eq(TaskMode.Active))).thenReturn(storeDir.toFile());
    when(storageManagerUtil.getStoreCheckpointDir(any(File.class), eq(checkpointId))).thenReturn(storeCheckpointDir.toString());
    when(storageManagerUtil.getTaskStoreCheckpointDirs(any(File.class), anyString(), any(TaskName.class), any(TaskMode.class))).thenReturn(ImmutableList.of(storeCheckpointDir.toFile()));
    BlobStoreUtil blobStoreUtil = mock(BlobStoreUtil.class);
    DirDiffUtil dirDiffUtil = mock(DirDiffUtil.class);
    // ensures shouldRestore is not called
    when(dirDiffUtil.areSameDir(anySet(), anyBoolean())).thenReturn((arg1, arg2) -> true);
    // return immediately without restoring.
    when(blobStoreUtil.restoreDir(eq(storeDir.toFile()), eq(dirIndex), any(Metadata.class))).thenReturn(CompletableFuture.completedFuture(null));
    BlobStoreRestoreManager.restoreStores(jobName, jobId, taskName, storesToRestore, prevStoreSnapshotIndexes, loggedBaseDir.toFile(), storageConfig, metrics, storageManagerUtil, blobStoreUtil, dirDiffUtil, EXECUTOR);
    // verify that the store directory restore was not called (should have restored from checkpoint dir)
    verify(blobStoreUtil, times(0)).restoreDir(eq(storeDir.toFile()), eq(dirIndex), any(Metadata.class));
    // verify that the checkpoint dir was renamed to store dir
    assertFalse(storeCheckpointDir.toFile().exists());
    assertTrue(storeDir.toFile().exists());
    assertTrue(Files.exists(Paths.get(storeDir.toString(), tempFile.getFileName().toString())));
}