use of org.apache.samza.storage.blobstore.exceptions.DeletedException in project samza by apache.
the class TestBlobStoreUtil method testGetSSISkipsStoresWithSnapshotIndexAlreadyDeleted.
@Test
public void testGetSSISkipsStoresWithSnapshotIndexAlreadyDeleted() {
String store = "storeName1";
String otherStore = "storeName2";
Checkpoint checkpoint = createCheckpointV2(BlobStoreStateBackendFactory.class.getName(), ImmutableMap.of(store, "snapshotIndexBlobId1", otherStore, "snapshotIndexBlobId2"));
Set<String> storesToBackupOrRestore = new HashSet<>();
storesToBackupOrRestore.add(store);
storesToBackupOrRestore.add(otherStore);
SnapshotIndex store1SnapshotIndex = mock(SnapshotIndex.class);
BlobStoreUtil mockBlobStoreUtil = mock(BlobStoreUtil.class);
CompletableFuture<SnapshotIndex> failedFuture = FutureUtil.failedFuture(new DeletedException());
when(mockBlobStoreUtil.getSnapshotIndex(eq("snapshotIndexBlobId1"), any(Metadata.class))).thenReturn(CompletableFuture.completedFuture(store1SnapshotIndex));
when(mockBlobStoreUtil.getSnapshotIndex(eq("snapshotIndexBlobId2"), any(Metadata.class))).thenReturn(failedFuture);
when(mockBlobStoreUtil.getStoreSnapshotIndexes(anyString(), anyString(), anyString(), any(Checkpoint.class), anySetOf(String.class))).thenCallRealMethod();
Map<String, Pair<String, SnapshotIndex>> snapshotIndexes = mockBlobStoreUtil.getStoreSnapshotIndexes("testJobName", "testJobId", "taskName", checkpoint, storesToBackupOrRestore);
assertEquals(1, snapshotIndexes.size());
assertEquals("snapshotIndexBlobId1", snapshotIndexes.get("storeName1").getLeft());
assertEquals(store1SnapshotIndex, snapshotIndexes.get("storeName1").getRight());
}
use of org.apache.samza.storage.blobstore.exceptions.DeletedException in project samza by apache.
the class BlobStoreUtil method getStoreSnapshotIndexes.
/**
* Get the blob id of {@link SnapshotIndex} and {@link SnapshotIndex}es for the provided {@code task}
* in the provided {@code checkpoint}.
* @param jobName job name is used to build request metadata
* @param jobId job id is used to build request metadata
* @param taskName task name to get the store state checkpoint markers and snapshot indexes for
* @param checkpoint {@link Checkpoint} instance to get the store state checkpoint markers from. Only
* {@link CheckpointV2} and newer are supported for blob stores.
* @param storesToBackupOrRestore set of store names to be backed up or restored
* @return Map of store name to its blob id of snapshot indices and their corresponding snapshot indices for the task.
*/
public Map<String, Pair<String, SnapshotIndex>> getStoreSnapshotIndexes(String jobName, String jobId, String taskName, Checkpoint checkpoint, Set<String> storesToBackupOrRestore) {
// TODO MED shesharma document error handling (checkpoint ver, blob not found, getBlob)
if (checkpoint == null) {
LOG.debug("No previous checkpoint found for taskName: {}", taskName);
return ImmutableMap.of();
}
if (checkpoint.getVersion() == 1) {
LOG.warn("Checkpoint version 1 is not supported for blob store backup and restore.");
return ImmutableMap.of();
}
Map<String, CompletableFuture<Pair<String, SnapshotIndex>>> storeSnapshotIndexFutures = new HashMap<>();
CheckpointV2 checkpointV2 = (CheckpointV2) checkpoint;
Map<String, Map<String, String>> factoryToStoreSCMs = checkpointV2.getStateCheckpointMarkers();
Map<String, String> storeSnapshotIndexBlobIds = factoryToStoreSCMs.get(BlobStoreStateBackendFactory.class.getName());
if (storeSnapshotIndexBlobIds != null) {
storeSnapshotIndexBlobIds.forEach((storeName, snapshotIndexBlobId) -> {
if (storesToBackupOrRestore.contains(storeName)) {
try {
LOG.debug("Getting snapshot index for taskName: {} store: {} blobId: {}", taskName, storeName, snapshotIndexBlobId);
Metadata requestMetadata = new Metadata(Metadata.SNAPSHOT_INDEX_PAYLOAD_PATH, Optional.empty(), jobName, jobId, taskName, storeName);
CompletableFuture<SnapshotIndex> snapshotIndexFuture = getSnapshotIndex(snapshotIndexBlobId, requestMetadata).toCompletableFuture();
Pair<CompletableFuture<String>, CompletableFuture<SnapshotIndex>> pairOfFutures = Pair.of(CompletableFuture.completedFuture(snapshotIndexBlobId), snapshotIndexFuture);
// save the future and block once in the end instead of blocking for each request.
storeSnapshotIndexFutures.put(storeName, FutureUtil.toFutureOfPair(pairOfFutures));
} catch (Exception e) {
throw new SamzaException(String.format("Error getting SnapshotIndex for blobId: %s for taskName: %s store: %s", snapshotIndexBlobId, taskName, storeName), e);
}
} else {
LOG.debug("SnapshotIndex blob id {} for store {} is not present in the set of stores to be backed up/restores: {}", snapshotIndexBlobId, storeName, storesToBackupOrRestore);
}
});
} else {
LOG.debug("No store SCMs found for blob store state backend in for taskName: {} in checkpoint {}", taskName, checkpointV2.getCheckpointId());
}
try {
return FutureUtil.toFutureOfMap(t -> {
Throwable unwrappedException = FutureUtil.unwrapExceptions(CompletionException.class, t);
if (unwrappedException instanceof DeletedException) {
LOG.warn("Ignoring already deleted snapshot index for taskName: {}", taskName, t);
return true;
} else {
return false;
}
}, storeSnapshotIndexFutures).join();
} catch (Exception e) {
throw new SamzaException(String.format("Error while waiting to get store snapshot indexes for task %s", taskName), e);
}
}
use of org.apache.samza.storage.blobstore.exceptions.DeletedException in project samza by apache.
the class TestBlobStoreUtil method testGetSSIThrowsExceptionIfAnyNonIgnoredAsyncBlobStoreErrors.
@Test
public void testGetSSIThrowsExceptionIfAnyNonIgnoredAsyncBlobStoreErrors() {
String store = "storeName1";
String otherStore = "storeName2";
Set<String> storesToBackupOrRestore = new HashSet<>();
storesToBackupOrRestore.add(store);
storesToBackupOrRestore.add(otherStore);
Checkpoint checkpoint = createCheckpointV2(BlobStoreStateBackendFactory.class.getName(), ImmutableMap.of(store, "snapshotIndexBlobId1", otherStore, "snapshotIndexBlobId2"));
SnapshotIndex store1SnapshotIndex = mock(SnapshotIndex.class);
BlobStoreUtil mockBlobStoreUtil = mock(BlobStoreUtil.class);
when(mockBlobStoreUtil.getStoreSnapshotIndexes(anyString(), anyString(), anyString(), any(Checkpoint.class), anySetOf(String.class))).thenCallRealMethod();
RuntimeException nonIgnoredException = new RuntimeException();
CompletableFuture<SnapshotIndex> failedFuture = FutureUtil.failedFuture(nonIgnoredException);
when(mockBlobStoreUtil.getSnapshotIndex(eq("snapshotIndexBlobId1"), any(Metadata.class))).thenReturn(// should fail even if some errors are ignored
FutureUtil.failedFuture(new DeletedException()));
when(mockBlobStoreUtil.getSnapshotIndex(eq("snapshotIndexBlobId2"), any(Metadata.class))).thenReturn(failedFuture);
try {
mockBlobStoreUtil.getStoreSnapshotIndexes("testJobName", "testJobId", "taskName", checkpoint, storesToBackupOrRestore);
fail("Should have thrown an exception");
} catch (Exception e) {
Throwable cause = FutureUtil.unwrapExceptions(CompletionException.class, FutureUtil.unwrapExceptions(SamzaException.class, e));
assertEquals(nonIgnoredException, cause);
}
}
Aggregations