Examples with CheckpointV2 org.apache.samza.checkpoint.CheckpointV2 used on opensource projects

Example 6 with CheckpointV2

use of org.apache.samza.checkpoint.CheckpointV2 in project samza by apache.

the class TestContainerStorageManager method testCheckpointV2BasedRestoreFactoryCreation.

@Test
public void testCheckpointV2BasedRestoreFactoryCreation() {
    Set<String> storeNames = ImmutableSet.of("storeName0", "storeName1", "storeName2");
    StorageConfig mockConfig = mock(StorageConfig.class);
    when(mockConfig.getStoreRestoreFactories("storeName0")).thenReturn(ImmutableList.of("factory0", "factory1", "factory2"));
    when(mockConfig.getStoreRestoreFactories("storeName1")).thenReturn(ImmutableList.of("factory2", "factory1"));
    when(mockConfig.getStoreRestoreFactories("storeName2")).thenReturn(Collections.emptyList());
    when(mockConfig.getChangelogStream("storeName0")).thenReturn(Optional.empty());
    when(mockConfig.getChangelogStream("storeName1")).thenReturn(Optional.of("changelog"));
    when(mockConfig.getChangelogStream("storeName2")).thenReturn(Optional.of("changelog"));
    CheckpointV2 checkpointV2 = mock(CheckpointV2.class);
    when(checkpointV2.getVersion()).thenReturn((short) 2);
    when(checkpointV2.getStateCheckpointMarkers()).thenReturn(ImmutableMap.of("factory0", ImmutableMap.of("storeName0", "", "storeName1", "", "storeName2", ""), "factory1", ImmutableMap.of("storeName0", "", "storeName1", "", "storeName2", ""), "factory2", ImmutableMap.of("storeName0", "", "storeName1", "", "storeName2", "")));
    Map<String, Set<String>> factoriesToStores = this.containerStorageManager.getBackendFactoryStoreNames(checkpointV2, storeNames, mockConfig);
    Assert.assertEquals(2, factoriesToStores.size());
    Assert.assertEquals(ImmutableSet.of("storeName0"), factoriesToStores.get("factory0"));
    Assert.assertEquals(ImmutableSet.of("storeName1"), factoriesToStores.get("factory2"));
    when(checkpointV2.getStateCheckpointMarkers()).thenReturn(ImmutableMap.of("factory2", ImmutableMap.of("storeName0", "", "storeName1", "", "storeName2", "")));
    factoriesToStores = this.containerStorageManager.getBackendFactoryStoreNames(checkpointV2, storeNames, mockConfig);
    Assert.assertEquals(1, factoriesToStores.size());
    Assert.assertEquals(ImmutableSet.of("storeName1", "storeName0"), factoriesToStores.get("factory2"));
    when(checkpointV2.getStateCheckpointMarkers()).thenReturn(ImmutableMap.of("factory1", ImmutableMap.of("storeName0", "", "storeName1", "", "storeName2", ""), "factory2", ImmutableMap.of("storeName0", "", "storeName1", "", "storeName2", "")));
    factoriesToStores = this.containerStorageManager.getBackendFactoryStoreNames(checkpointV2, storeNames, mockConfig);
    Assert.assertEquals(2, factoriesToStores.size());
    Assert.assertEquals(ImmutableSet.of("storeName0"), factoriesToStores.get("factory1"));
    Assert.assertEquals(ImmutableSet.of("storeName1"), factoriesToStores.get("factory2"));
    when(checkpointV2.getStateCheckpointMarkers()).thenReturn(ImmutableMap.of("factory1", ImmutableMap.of("storeName0", "", "storeName1", "", "storeName2", ""), "factory2", ImmutableMap.of("storeName0", "", "storeName2", "")));
    factoriesToStores = this.containerStorageManager.getBackendFactoryStoreNames(checkpointV2, storeNames, mockConfig);
    Assert.assertEquals(1, factoriesToStores.size());
    Assert.assertEquals(ImmutableSet.of("storeName0", "storeName1"), factoriesToStores.get("factory1"));
    when(checkpointV2.getStateCheckpointMarkers()).thenReturn(ImmutableMap.of("factory1", ImmutableMap.of("storeName0", "", "storeName1", "", "storeName2", "")));
    factoriesToStores = this.containerStorageManager.getBackendFactoryStoreNames(checkpointV2, storeNames, mockConfig);
    Assert.assertEquals(1, factoriesToStores.size());
    Assert.assertEquals(ImmutableSet.of("storeName0", "storeName1"), factoriesToStores.get("factory1"));
    when(checkpointV2.getStateCheckpointMarkers()).thenReturn(Collections.emptyMap());
    factoriesToStores = this.containerStorageManager.getBackendFactoryStoreNames(checkpointV2, storeNames, mockConfig);
    Assert.assertEquals(2, factoriesToStores.size());
    Assert.assertEquals(ImmutableSet.of("storeName0"), factoriesToStores.get("factory0"));
    Assert.assertEquals(ImmutableSet.of("storeName1"), factoriesToStores.get("factory2"));
    when(checkpointV2.getStateCheckpointMarkers()).thenReturn(ImmutableMap.of("factory0", ImmutableMap.of("storeName1", "", "storeName2", ""), "factory1", ImmutableMap.of("storeName1", "", "storeName2", ""), "factory2", ImmutableMap.of("storeName0", "", "storeName2", "")));
    factoriesToStores = this.containerStorageManager.getBackendFactoryStoreNames(checkpointV2, storeNames, mockConfig);
    Assert.assertEquals(2, factoriesToStores.size());
    Assert.assertEquals(ImmutableSet.of("storeName1"), factoriesToStores.get("factory1"));
    Assert.assertEquals(ImmutableSet.of("storeName0"), factoriesToStores.get("factory2"));
}

Example 7 with CheckpointV2

use of org.apache.samza.checkpoint.CheckpointV2 in project samza by apache.

the class TransactionalStateTaskRestoreManager method getCheckpointId.

private CheckpointId getCheckpointId(Checkpoint checkpoint) {
    if (checkpoint == null)
        return null;
    if (checkpoint instanceof CheckpointV1) {
        for (Map.Entry<String, SystemStream> storeNameSystemStream : storeChangelogs.entrySet()) {
            SystemStreamPartition storeChangelogSSP = new SystemStreamPartition(storeNameSystemStream.getValue(), taskModel.getChangelogPartition());
            String checkpointMessage = checkpoint.getOffsets().get(storeChangelogSSP);
            if (StringUtils.isNotBlank(checkpointMessage)) {
                KafkaChangelogSSPOffset kafkaStateChanglogOffset = KafkaChangelogSSPOffset.fromString(checkpointMessage);
                return kafkaStateChanglogOffset.getCheckpointId();
            }
        }
    } else if (checkpoint instanceof CheckpointV2) {
        return ((CheckpointV2) checkpoint).getCheckpointId();
    } else {
        throw new SamzaException("Unsupported checkpoint version: " + checkpoint.getVersion());
    }
    return null;
}

Example 8 with CheckpointV2

use of org.apache.samza.checkpoint.CheckpointV2 in project samza by apache.

the class TestBlobStoreUtil method testGetSSIReturnsEmptyMapIfNoStoreForBlobStoreBackendFactory.

@Test
public void testGetSSIReturnsEmptyMapIfNoStoreForBlobStoreBackendFactory() {
    CheckpointV2 mockCheckpoint = mock(CheckpointV2.class);
    when(mockCheckpoint.getVersion()).thenReturn((short) 2);
    when(mockCheckpoint.getStateCheckpointMarkers()).thenReturn(ImmutableMap.of(BlobStoreStateBackendFactory.class.getName(), ImmutableMap.of()));
    BlobStoreUtil blobStoreUtil = new BlobStoreUtil(mock(BlobStoreManager.class), MoreExecutors.newDirectExecutorService(), null, null);
    Map<String, Pair<String, SnapshotIndex>> snapshotIndexes = blobStoreUtil.getStoreSnapshotIndexes("testJobName", "testJobId", "taskName", mockCheckpoint, new HashSet<>());
    assertTrue(snapshotIndexes.isEmpty());
}

Example 9 with CheckpointV2

use of org.apache.samza.checkpoint.CheckpointV2 in project samza by apache.

the class TestBlobStoreUtil method createCheckpointV2.

private CheckpointV2 createCheckpointV2(String stateBackendFactory, Map<String, String> storeSnapshotIndexBlobIds) {
    CheckpointId checkpointId = CheckpointId.create();
    Map<String, Map<String, String>> factoryStoreSCMs = new HashMap<>();
    Map<String, String> storeSCMs = new HashMap<>();
    for (Map.Entry<String, String> entry : storeSnapshotIndexBlobIds.entrySet()) {
        storeSCMs.put(entry.getKey(), entry.getValue());
    }
    factoryStoreSCMs.put(stateBackendFactory, storeSCMs);
    return new CheckpointV2(checkpointId, ImmutableMap.of(), factoryStoreSCMs);
}

Example 10 with CheckpointV2

use of org.apache.samza.checkpoint.CheckpointV2 in project samza by apache.

the class TestBlobStoreUtil method testGetCheckpointIndexIgnoresStoresNotInStoresToBackupRestoreSet.

@Test
public void testGetCheckpointIndexIgnoresStoresNotInStoresToBackupRestoreSet() {
    String store = "storeName1";
    String anotherStore = "storeName2";
    String oneMoreStore = "storeName3";
    SnapshotIndex mockStoreSnapshotIndex = mock(SnapshotIndex.class);
    Set<String> storesToBackupOrRestore = ImmutableSet.of(store, anotherStore);
    CheckpointV2 checkpoint = createCheckpointV2(BlobStoreStateBackendFactory.class.getName(), ImmutableMap.of(store, "1", anotherStore, "2", oneMoreStore, "3"));
    BlobStoreUtil mockBlobStoreUtil = mock(BlobStoreUtil.class);
    when(mockBlobStoreUtil.getSnapshotIndex(any(String.class), any(Metadata.class))).thenReturn(CompletableFuture.completedFuture(mockStoreSnapshotIndex));
    when(mockBlobStoreUtil.getStoreSnapshotIndexes(anyString(), anyString(), anyString(), any(Checkpoint.class), anySetOf(String.class))).thenCallRealMethod();
    Map<String, Pair<String, SnapshotIndex>> snapshotIndexes = mockBlobStoreUtil.getStoreSnapshotIndexes("testJobName", "testJobId", "taskName", checkpoint, storesToBackupOrRestore);
    verify(mockBlobStoreUtil, times(storesToBackupOrRestore.size())).getSnapshotIndex(anyString(), any(Metadata.class));
}