Search in sources :

Example 16 with Checkpoint

use of org.apache.samza.checkpoint.Checkpoint in project samza by apache.

the class TestBlobStoreUtil method testGetSSIThrowsExceptionOnSyncBlobStoreErrors.

@Test(expected = SamzaException.class)
public void testGetSSIThrowsExceptionOnSyncBlobStoreErrors() {
    Checkpoint checkpoint = createCheckpointV2(BlobStoreStateBackendFactory.class.getName(), ImmutableMap.of("storeName", "snapshotIndexBlobId"));
    Set<String> storesToBackupOrRestore = new HashSet<>();
    storesToBackupOrRestore.add("storeName");
    BlobStoreUtil mockBlobStoreUtil = mock(BlobStoreUtil.class);
    when(mockBlobStoreUtil.getSnapshotIndex(anyString(), any(Metadata.class))).thenThrow(new RuntimeException());
    when(mockBlobStoreUtil.getStoreSnapshotIndexes(anyString(), anyString(), anyString(), any(Checkpoint.class), anySetOf(String.class))).thenCallRealMethod();
    mockBlobStoreUtil.getStoreSnapshotIndexes("testJobName", "testJobId", "taskName", checkpoint, storesToBackupOrRestore);
}
Also used : BlobStoreStateBackendFactory(org.apache.samza.storage.blobstore.BlobStoreStateBackendFactory) Checkpoint(org.apache.samza.checkpoint.Checkpoint) FileMetadata(org.apache.samza.storage.blobstore.index.FileMetadata) SnapshotMetadata(org.apache.samza.storage.blobstore.index.SnapshotMetadata) Metadata(org.apache.samza.storage.blobstore.Metadata) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 17 with Checkpoint

use of org.apache.samza.checkpoint.Checkpoint in project samza by apache.

the class TestBlobStoreUtil method testGetSSIThrowsExceptionForCheckpointV1.

public void testGetSSIThrowsExceptionForCheckpointV1() {
    Checkpoint mockCheckpoint = mock(Checkpoint.class);
    when(mockCheckpoint.getVersion()).thenReturn((short) 1);
    BlobStoreUtil blobStoreUtil = new BlobStoreUtil(mock(BlobStoreManager.class), MoreExecutors.newDirectExecutorService(), null, null);
    Map<String, Pair<String, SnapshotIndex>> prevSnapshotIndexes = blobStoreUtil.getStoreSnapshotIndexes("testJobName", "testJobId", "taskName", mockCheckpoint, new HashSet<>());
    assertEquals(prevSnapshotIndexes.size(), 0);
}
Also used : Checkpoint(org.apache.samza.checkpoint.Checkpoint) BlobStoreManager(org.apache.samza.storage.blobstore.BlobStoreManager) Pair(org.apache.commons.lang3.tuple.Pair)

Example 18 with Checkpoint

use of org.apache.samza.checkpoint.Checkpoint in project samza by apache.

the class TestBlobStoreUtil method testGetCheckpointIndexIgnoresStoresNotInStoresToBackupRestoreSet.

@Test
public void testGetCheckpointIndexIgnoresStoresNotInStoresToBackupRestoreSet() {
    String store = "storeName1";
    String anotherStore = "storeName2";
    String oneMoreStore = "storeName3";
    SnapshotIndex mockStoreSnapshotIndex = mock(SnapshotIndex.class);
    Set<String> storesToBackupOrRestore = ImmutableSet.of(store, anotherStore);
    CheckpointV2 checkpoint = createCheckpointV2(BlobStoreStateBackendFactory.class.getName(), ImmutableMap.of(store, "1", anotherStore, "2", oneMoreStore, "3"));
    BlobStoreUtil mockBlobStoreUtil = mock(BlobStoreUtil.class);
    when(mockBlobStoreUtil.getSnapshotIndex(any(String.class), any(Metadata.class))).thenReturn(CompletableFuture.completedFuture(mockStoreSnapshotIndex));
    when(mockBlobStoreUtil.getStoreSnapshotIndexes(anyString(), anyString(), anyString(), any(Checkpoint.class), anySetOf(String.class))).thenCallRealMethod();
    Map<String, Pair<String, SnapshotIndex>> snapshotIndexes = mockBlobStoreUtil.getStoreSnapshotIndexes("testJobName", "testJobId", "taskName", checkpoint, storesToBackupOrRestore);
    verify(mockBlobStoreUtil, times(storesToBackupOrRestore.size())).getSnapshotIndex(anyString(), any(Metadata.class));
}
Also used : CheckpointV2(org.apache.samza.checkpoint.CheckpointV2) BlobStoreStateBackendFactory(org.apache.samza.storage.blobstore.BlobStoreStateBackendFactory) SnapshotIndex(org.apache.samza.storage.blobstore.index.SnapshotIndex) Checkpoint(org.apache.samza.checkpoint.Checkpoint) FileMetadata(org.apache.samza.storage.blobstore.index.FileMetadata) SnapshotMetadata(org.apache.samza.storage.blobstore.index.SnapshotMetadata) Metadata(org.apache.samza.storage.blobstore.Metadata) Pair(org.apache.commons.lang3.tuple.Pair) Test(org.junit.Test)

Example 19 with Checkpoint

use of org.apache.samza.checkpoint.Checkpoint in project samza by apache.

the class TransactionalStateTaskRestoreManager method getCheckpointedChangelogOffsets.

private Map<String, KafkaStateCheckpointMarker> getCheckpointedChangelogOffsets(Checkpoint checkpoint) {
    Map<String, KafkaStateCheckpointMarker> checkpointedChangelogOffsets = new HashMap<>();
    if (checkpoint == null)
        return checkpointedChangelogOffsets;
    if (checkpoint instanceof CheckpointV2) {
        Map<String, Map<String, String>> factoryStoreSCMs = ((CheckpointV2) checkpoint).getStateCheckpointMarkers();
        if (factoryStoreSCMs.containsKey(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME)) {
            factoryStoreSCMs.get(KafkaStateCheckpointMarker.KAFKA_STATE_BACKEND_FACTORY_NAME).forEach((storeName, scmString) -> {
                KafkaStateCheckpointMarker kafkaSCM = KafkaStateCheckpointMarker.deserialize(scmString);
                checkpointedChangelogOffsets.put(storeName, kafkaSCM);
            });
        }
    // skip the non-KafkaStateCheckpointMarkers
    } else if (checkpoint instanceof CheckpointV1) {
        // If the checkpoint v1 is used, we need to fetch the changelog SSPs in the inputOffsets in order to get the
        // store offset.
        Map<SystemStreamPartition, String> checkpointedOffsets = checkpoint.getOffsets();
        storeChangelogs.forEach((storeName, systemStream) -> {
            Partition changelogPartition = taskModel.getChangelogPartition();
            SystemStreamPartition storeChangelogSSP = new SystemStreamPartition(systemStream, changelogPartition);
            String checkpointedOffset = checkpointedOffsets.get(storeChangelogSSP);
            if (StringUtils.isNotBlank(checkpointedOffset)) {
                KafkaChangelogSSPOffset kafkaChangelogSSPOffset = KafkaChangelogSSPOffset.fromString(checkpointedOffset);
                KafkaStateCheckpointMarker marker = new KafkaStateCheckpointMarker(storeChangelogSSP, kafkaChangelogSSPOffset.getChangelogOffset());
                checkpointedChangelogOffsets.put(storeName, marker);
            }
        });
    } else {
        throw new SamzaException("Unsupported checkpoint version: " + checkpoint.getVersion());
    }
    return checkpointedChangelogOffsets;
}
Also used : ArrayListMultimap(com.google.common.collect.ArrayListMultimap) ListMultimap(com.google.common.collect.ListMultimap) SSPMetadataCache(org.apache.samza.system.SSPMetadataCache) CheckpointV2(org.apache.samza.checkpoint.CheckpointV2) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) TaskModel(org.apache.samza.job.model.TaskModel) Serde(org.apache.samza.serializers.Serde) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) StringUtils(org.apache.commons.lang3.StringUtils) CheckpointV1(org.apache.samza.checkpoint.CheckpointV1) FileUtil(org.apache.samza.util.FileUtil) SystemConsumer(org.apache.samza.system.SystemConsumer) MessageCollector(org.apache.samza.task.MessageCollector) SystemStream(org.apache.samza.system.SystemStream) Map(java.util.Map) ExecutorService(java.util.concurrent.ExecutorService) StorageConfig(org.apache.samza.config.StorageConfig) KafkaChangelogSSPOffset(org.apache.samza.checkpoint.kafka.KafkaChangelogSSPOffset) TaskName(org.apache.samza.container.TaskName) Logger(org.slf4j.Logger) ImmutableMap(com.google.common.collect.ImmutableMap) TaskConfig(org.apache.samza.config.TaskConfig) JobContext(org.apache.samza.context.JobContext) Partition(org.apache.samza.Partition) ContainerContext(org.apache.samza.context.ContainerContext) Set(java.util.Set) Checkpoint(org.apache.samza.checkpoint.Checkpoint) Clock(org.apache.samza.util.Clock) MetricsRegistry(org.apache.samza.metrics.MetricsRegistry) File(java.io.File) SamzaException(org.apache.samza.SamzaException) CheckpointId(org.apache.samza.checkpoint.CheckpointId) List(java.util.List) TaskMode(org.apache.samza.job.model.TaskMode) ChangelogSSPIterator(org.apache.samza.system.ChangelogSSPIterator) SystemAdmin(org.apache.samza.system.SystemAdmin) SystemStreamPartitionMetadata(org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata) Optional(java.util.Optional) Preconditions(com.google.common.base.Preconditions) KafkaStateCheckpointMarker(org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Config(org.apache.samza.config.Config) Collections(java.util.Collections) SystemAdmins(org.apache.samza.system.SystemAdmins) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Partition(org.apache.samza.Partition) HashMap(java.util.HashMap) SamzaException(org.apache.samza.SamzaException) CheckpointV2(org.apache.samza.checkpoint.CheckpointV2) CheckpointV1(org.apache.samza.checkpoint.CheckpointV1) KafkaChangelogSSPOffset(org.apache.samza.checkpoint.kafka.KafkaChangelogSSPOffset) HashMap(java.util.HashMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) KafkaStateCheckpointMarker(org.apache.samza.checkpoint.kafka.KafkaStateCheckpointMarker) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition)

Example 20 with Checkpoint

use of org.apache.samza.checkpoint.Checkpoint in project samza by apache.

the class TaskStorageCommitManager method writeCheckpointToStoreDirectories.

/**
 * Writes the {@link Checkpoint} information returned by {@link #upload(CheckpointId, Map)}
 * in each store directory and store checkpoint directory. Written content depends on the type of {@code checkpoint}.
 * For {@link CheckpointV2}, writes the entire task {@link CheckpointV2}.
 * For {@link CheckpointV1}, only writes the changelog ssp offsets in the OFFSET* files.
 *
 * Note: The assumption is that this method will be invoked once for each {@link Checkpoint} version that the
 * task needs to write as determined by {@link org.apache.samza.config.TaskConfig#getCheckpointWriteVersions()}.
 * This is required for upgrade and rollback compatibility.
 *
 * @param checkpoint the latest checkpoint to be persisted to local file system
 */
public void writeCheckpointToStoreDirectories(Checkpoint checkpoint) {
    if (checkpoint instanceof CheckpointV1) {
        LOG.debug("Writing CheckpointV1 to store and checkpoint directories for taskName: {} with checkpoint: {}", taskName, checkpoint);
        // Write CheckpointV1 changelog offsets to store and checkpoint directories
        writeChangelogOffsetFiles(checkpoint.getOffsets());
    } else if (checkpoint instanceof CheckpointV2) {
        LOG.debug("Writing CheckpointV2 to store and checkpoint directories for taskName: {} with checkpoint: {}", taskName, checkpoint);
        storageEngines.forEach((storeName, storageEngine) -> {
            // Only write the checkpoint file if the store is durable and persisted to disk
            if (storageEngine.getStoreProperties().isDurableStore() && storageEngine.getStoreProperties().isPersistedToDisk()) {
                CheckpointV2 checkpointV2 = (CheckpointV2) checkpoint;
                try {
                    File storeDir = storageManagerUtil.getTaskStoreDir(durableStoreBaseDir, storeName, taskName, TaskMode.Active);
                    storageManagerUtil.writeCheckpointV2File(storeDir, checkpointV2);
                    CheckpointId checkpointId = checkpointV2.getCheckpointId();
                    File checkpointDir = Paths.get(storageManagerUtil.getStoreCheckpointDir(storeDir, checkpointId)).toFile();
                    storageManagerUtil.writeCheckpointV2File(checkpointDir, checkpointV2);
                } catch (Exception e) {
                    throw new SamzaException(String.format("Write checkpoint file failed for task: %s, storeName: %s, checkpointId: %s", taskName, storeName, ((CheckpointV2) checkpoint).getCheckpointId()), e);
                }
            }
        });
    } else {
        throw new SamzaException("Unsupported checkpoint version: " + checkpoint.getVersion());
    }
}
Also used : CheckpointV2(org.apache.samza.checkpoint.CheckpointV2) CheckpointV2(org.apache.samza.checkpoint.CheckpointV2) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) ArrayList(java.util.ArrayList) CheckpointV1(org.apache.samza.checkpoint.CheckpointV1) CheckpointManager(org.apache.samza.checkpoint.CheckpointManager) SystemStream(org.apache.samza.system.SystemStream) Map(java.util.Map) ExecutorService(java.util.concurrent.ExecutorService) FutureUtil(org.apache.samza.util.FutureUtil) KafkaChangelogSSPOffset(org.apache.samza.checkpoint.kafka.KafkaChangelogSSPOffset) TaskInstanceMetrics(org.apache.samza.container.TaskInstanceMetrics) TaskName(org.apache.samza.container.TaskName) Logger(org.slf4j.Logger) Partition(org.apache.samza.Partition) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) Checkpoint(org.apache.samza.checkpoint.Checkpoint) File(java.io.File) SamzaException(org.apache.samza.SamzaException) CheckpointId(org.apache.samza.checkpoint.CheckpointId) List(java.util.List) TaskMode(org.apache.samza.job.model.TaskMode) FileFilter(java.io.FileFilter) Paths(java.nio.file.Paths) WildcardFileFilter(org.apache.commons.io.filefilter.WildcardFileFilter) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Config(org.apache.samza.config.Config) Collections(java.util.Collections) CheckpointV1(org.apache.samza.checkpoint.CheckpointV1) CheckpointId(org.apache.samza.checkpoint.CheckpointId) File(java.io.File) SamzaException(org.apache.samza.SamzaException) IOException(java.io.IOException) SamzaException(org.apache.samza.SamzaException)

Aggregations

Checkpoint (org.apache.samza.checkpoint.Checkpoint)35 Test (org.junit.Test)22 TaskName (org.apache.samza.container.TaskName)21 HashMap (java.util.HashMap)20 SystemStreamPartition (org.apache.samza.system.SystemStreamPartition)18 Map (java.util.Map)16 Partition (org.apache.samza.Partition)16 CheckpointV1 (org.apache.samza.checkpoint.CheckpointV1)16 CheckpointV2 (org.apache.samza.checkpoint.CheckpointV2)16 File (java.io.File)15 SamzaException (org.apache.samza.SamzaException)15 CheckpointId (org.apache.samza.checkpoint.CheckpointId)14 CompletableFuture (java.util.concurrent.CompletableFuture)13 MapConfig (org.apache.samza.config.MapConfig)13 ImmutableMap (com.google.common.collect.ImmutableMap)12 Collections (java.util.Collections)12 Optional (java.util.Optional)12 CheckpointManager (org.apache.samza.checkpoint.CheckpointManager)12 TaskMode (org.apache.samza.job.model.TaskMode)12 TaskInstanceMetrics (org.apache.samza.container.TaskInstanceMetrics)11