Search in sources :

Example 26 with Checkpoint

use of org.apache.samza.checkpoint.Checkpoint in project samza by apache.

the class TestTaskStorageCommitManager method testPersistToFileSystemCheckpointV2Only.

@Test
public void testPersistToFileSystemCheckpointV2Only() throws IOException {
    ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class);
    StorageEngine mockLPStore = mock(StorageEngine.class);
    StoreProperties lpStoreProps = mock(StoreProperties.class);
    when(mockLPStore.getStoreProperties()).thenReturn(lpStoreProps);
    when(lpStoreProps.isPersistedToDisk()).thenReturn(true);
    when(lpStoreProps.isDurableStore()).thenReturn(true);
    Path mockPath = mock(Path.class);
    when(mockLPStore.checkpoint(any())).thenReturn(Optional.of(mockPath));
    StorageEngine mockPStore = mock(StorageEngine.class);
    StoreProperties pStoreProps = mock(StoreProperties.class);
    when(mockPStore.getStoreProperties()).thenReturn(pStoreProps);
    when(pStoreProps.isPersistedToDisk()).thenReturn(true);
    when(pStoreProps.isDurableStore()).thenReturn(false);
    StorageEngine mockLIStore = mock(StorageEngine.class);
    StoreProperties liStoreProps = mock(StoreProperties.class);
    when(mockLIStore.getStoreProperties()).thenReturn(liStoreProps);
    when(liStoreProps.isPersistedToDisk()).thenReturn(false);
    when(liStoreProps.isDurableStore()).thenReturn(true);
    StorageEngine mockIStore = mock(StorageEngine.class);
    StoreProperties iStoreProps = mock(StoreProperties.class);
    when(mockIStore.getStoreProperties()).thenReturn(iStoreProps);
    when(iStoreProps.isPersistedToDisk()).thenReturn(false);
    when(iStoreProps.isDurableStore()).thenReturn(false);
    java.util.Map<String, StorageEngine> taskStores = ImmutableMap.of("loggedPersistentStore", mockLPStore, "persistentStore", mockPStore, "loggedInMemStore", mockLIStore, "inMemStore", mockIStore);
    Partition changelogPartition = new Partition(0);
    SystemStream changelogSystemStream = new SystemStream("changelogSystem", "changelogStream");
    SystemStreamPartition changelogSSP = new SystemStreamPartition(changelogSystemStream, changelogPartition);
    java.util.Map<String, SystemStream> storeChangelogsStreams = ImmutableMap.of("loggedPersistentStore", changelogSystemStream, "loggedInMemStore", new SystemStream("system", "stream"));
    StorageManagerUtil storageManagerUtil = mock(StorageManagerUtil.class);
    File durableStoreDir = new File("durableStorePath");
    when(storageManagerUtil.getTaskStoreDir(eq(durableStoreDir), eq("loggedPersistentStore"), any(), any())).thenReturn(durableStoreDir);
    TaskName taskName = new TaskName("task");
    TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class);
    Timer checkpointTimer = mock(Timer.class);
    when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer);
    when(containerStorageManager.getAllStores(taskName)).thenReturn(taskStores);
    TaskStorageCommitManager commitManager = spy(new TaskStorageCommitManager(taskName, Collections.emptyMap(), containerStorageManager, storeChangelogsStreams, changelogPartition, null, null, ForkJoinPool.commonPool(), storageManagerUtil, durableStoreDir, metrics));
    doNothing().when(commitManager).writeChangelogOffsetFile(any(), any(), any(), any());
    when(storageManagerUtil.getStoreCheckpointDir(any(File.class), any(CheckpointId.class))).thenAnswer((Answer<String>) invocation -> {
        File file = invocation.getArgumentAt(0, File.class);
        CheckpointId checkpointId = invocation.getArgumentAt(1, CheckpointId.class);
        return file + "-" + checkpointId;
    });
    CheckpointId newCheckpointId = CheckpointId.create();
    java.util.Map<String, String> storeSCM = ImmutableMap.of("loggedPersistentStore", "system;loggedPersistentStoreStream;1", "persistentStore", "system;persistentStoreStream;1", "loggedInMemStore", "system;loggedInMemStoreStream;1", "inMemStore", "system;inMemStoreStream;1");
    CheckpointV2 checkpoint = new CheckpointV2(newCheckpointId, Collections.emptyMap(), Collections.singletonMap("factory", storeSCM));
    commitManager.init();
    // invoke persist to file system
    commitManager.writeCheckpointToStoreDirectories(checkpoint);
    // Validate only durable and persisted stores are persisted
    verify(storageManagerUtil).getTaskStoreDir(eq(durableStoreDir), eq("loggedPersistentStore"), eq(taskName), any());
    File checkpointPath = Paths.get(storageManagerUtil.getStoreCheckpointDir(durableStoreDir, newCheckpointId)).toFile();
    verify(storageManagerUtil).writeCheckpointV2File(eq(checkpointPath), eq(checkpoint));
}
Also used : Path(java.nio.file.Path) CheckpointV2(org.apache.samza.checkpoint.CheckpointV2) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Mockito.spy(org.mockito.Mockito.spy) CheckpointV1(org.apache.samza.checkpoint.CheckpointV1) Answer(org.mockito.stubbing.Answer) Mockito.doThrow(org.mockito.Mockito.doThrow) CheckpointManager(org.apache.samza.checkpoint.CheckpointManager) SystemStream(org.apache.samza.system.SystemStream) Map(java.util.Map) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Assert.fail(org.junit.Assert.fail) Mockito.anyLong(org.mockito.Mockito.anyLong) Path(java.nio.file.Path) MapConfig(org.apache.samza.config.MapConfig) KafkaChangelogSSPOffset(org.apache.samza.checkpoint.kafka.KafkaChangelogSSPOffset) TaskInstanceMetrics(org.apache.samza.container.TaskInstanceMetrics) TaskName(org.apache.samza.container.TaskName) ImmutableMap(com.google.common.collect.ImmutableMap) Timer(org.apache.samza.metrics.Timer) Partition(org.apache.samza.Partition) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) Checkpoint(org.apache.samza.checkpoint.Checkpoint) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) Mockito.doNothing(org.mockito.Mockito.doNothing) Mockito.when(org.mockito.Mockito.when) File(java.io.File) SamzaException(org.apache.samza.SamzaException) CheckpointId(org.apache.samza.checkpoint.CheckpointId) Mockito.verify(org.mockito.Mockito.verify) Matchers.any(org.mockito.Matchers.any) TaskMode(org.apache.samza.job.model.TaskMode) Mockito.never(org.mockito.Mockito.never) FileFilter(java.io.FileFilter) Paths(java.nio.file.Paths) ForkJoinPool(java.util.concurrent.ForkJoinPool) Optional(java.util.Optional) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) Mockito.eq(org.mockito.Mockito.eq) Mockito.mock(org.mockito.Mockito.mock) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Partition(org.apache.samza.Partition) SystemStream(org.apache.samza.system.SystemStream) TaskInstanceMetrics(org.apache.samza.container.TaskInstanceMetrics) CheckpointV2(org.apache.samza.checkpoint.CheckpointV2) Timer(org.apache.samza.metrics.Timer) TaskName(org.apache.samza.container.TaskName) CheckpointId(org.apache.samza.checkpoint.CheckpointId) File(java.io.File) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Test(org.junit.Test)

Example 27 with Checkpoint

use of org.apache.samza.checkpoint.Checkpoint in project samza by apache.

the class TestBlobStoreBackupManager method testUploadWithPreviousCheckpoints.

@Test
public void testUploadWithPreviousCheckpoints() throws IOException {
    // Track directory for post cleanup
    List<String> checkpointDirsToClean = new ArrayList<>();
    // Setup: init back manager with previous checkpoints
    // indexBlobIdAndLocalRemoteSnapshotsPair = setupRemoteAndLocalSnapshots(true);
    Map<String, String> previousCheckpoints = // map store name, previous snapshot index blob id
    indexBlobIdAndLocalRemoteSnapshotsPair.entrySet().stream().collect(Collectors.toMap(e -> e.getValue().getLeft(), e -> e.getValue().getRight().getPrevSnapshotIndexBlobId().get()));
    Checkpoint checkpoint = new CheckpointV2(checkpointId, new HashMap<>(), ImmutableMap.of(BlobStoreStateBackendFactory.class.getName(), previousCheckpoints));
    when(blobStoreUtil.getStoreSnapshotIndexes(anyString(), anyString(), anyString(), any(Checkpoint.class), anySetOf(String.class))).thenCallRealMethod();
    blobStoreBackupManager.init(checkpoint);
    // mock: set task store dir to return corresponding test local store and create checkpoint dir
    ArgumentCaptor<String> stringCaptor = ArgumentCaptor.forClass(String.class);
    when(storageManagerUtil.getTaskStoreDir(any(File.class), stringCaptor.capture(), any(TaskName.class), any(TaskMode.class))).then((Answer<File>) invocation -> {
        String storeName = invocation.getArgumentAt(1, String.class);
        String snapshotIndexBlobId = testStoreNameAndSCMMap.get(storeName);
        String storeDir = indexBlobIdAndLocalRemoteSnapshotsPair.get(snapshotIndexBlobId).getLeft();
        try {
            BlobStoreTestUtil.createTestCheckpointDirectory(storeDir, checkpointId.serialize());
            checkpointDirsToClean.add(storeDir + "-" + checkpointId.serialize());
        } catch (IOException e) {
            Assert.fail("Couldn't create checkpoint directory. Test failed.");
        }
        return new File(storeDir);
    });
    ArgumentCaptor<File> storeDirCaptor = ArgumentCaptor.forClass(File.class);
    when(storageManagerUtil.getStoreCheckpointDir(storeDirCaptor.capture(), eq(checkpointId))).thenAnswer(new Answer<String>() {

        @Override
        public String answer(InvocationOnMock invocation) throws Throwable {
            File storeDir = invocation.getArgumentAt(0, File.class);
            return storeDir.getAbsolutePath() + "-" + checkpointId.serialize();
        }
    });
    // mock: mock putDir and capture DirDiff
    SortedSet<DirDiff> actualDirDiffs = new TreeSet<>(Comparator.comparing(DirDiff::getDirName));
    ArgumentCaptor<DirDiff> dirDiffCaptor = ArgumentCaptor.forClass(DirDiff.class);
    ArgumentCaptor<SnapshotMetadata> snapshotMetadataCaptor = ArgumentCaptor.forClass(SnapshotMetadata.class);
    when(blobStoreUtil.putDir(dirDiffCaptor.capture(), snapshotMetadataCaptor.capture())).then((Answer<CompletableFuture<DirIndex>>) invocation -> {
        DirDiff dirDiff = invocation.getArgumentAt(0, DirDiff.class);
        SnapshotMetadata snapshotMetadata = invocation.getArgumentAt(1, SnapshotMetadata.class);
        actualDirDiffs.add(dirDiff);
        SnapshotIndex snapshotIndex = testBlobStore.get(testStoreNameAndSCMMap.get(snapshotMetadata.getStoreName()));
        return CompletableFuture.completedFuture(snapshotIndex.getDirIndex());
    });
    // mock: mock putSnapshotIndex and capture previous snapshot index
    SortedSet<SnapshotIndex> expectedSnapshotIndexesUploaded = indexBlobIdAndLocalRemoteSnapshotsPair.values().stream().map(Pair::getRight).collect(Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(SnapshotIndex::getCreationTimeMillis))));
    SortedSet<SnapshotIndex> actualSnapshotIndexesUploaded = new TreeSet<>(Comparator.comparing(SnapshotIndex::getCreationTimeMillis));
    SortedSet<String> actualPreviousSnapshotIndexBlobIds = new TreeSet<>();
    SortedSet<String> expectedPreviousSnapshotIndexBlobIds = new TreeSet<>(previousCheckpoints.values());
    ArgumentCaptor<SnapshotIndex> snapshotIndexCaptor = ArgumentCaptor.forClass(SnapshotIndex.class);
    when(blobStoreUtil.putSnapshotIndex(snapshotIndexCaptor.capture())).then((Answer<CompletableFuture<String>>) invocation -> {
        SnapshotIndex snapshotIndex = invocation.getArgumentAt(0, SnapshotIndex.class);
        actualSnapshotIndexesUploaded.add(snapshotIndex);
        if (snapshotIndex.getPrevSnapshotIndexBlobId().isPresent()) {
            actualPreviousSnapshotIndexBlobIds.add(snapshotIndex.getPrevSnapshotIndexBlobId().get());
        }
        return CompletableFuture.completedFuture("random-blob-id");
    });
    // execute
    blobStoreBackupManager.upload(checkpointId, ImmutableMap.of());
    TreeSet<DirDiff> expectedDirDiffs = indexBlobIdAndLocalRemoteSnapshotsPair.values().stream().map(localRemoteSnapshotPair -> DirDiffUtil.getDirDiff(new File(localRemoteSnapshotPair.getLeft() + "-" + checkpointId.serialize()), localRemoteSnapshotPair.getRight().getDirIndex(), DirDiffUtil.areSameFile(false))).collect(Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(DirDiff::getDirName))));
    // assert - asset all DirDiff are put to blob store
    Assert.assertEquals(actualDirDiffs, expectedDirDiffs);
    // assert - assert no previous snapshot indexes were found
    Assert.assertEquals(actualPreviousSnapshotIndexBlobIds, expectedPreviousSnapshotIndexBlobIds);
    // assert - assert all snapshot indexes are uploaded
    Assert.assertEquals(actualSnapshotIndexesUploaded, expectedSnapshotIndexesUploaded);
    // cleanup
    checkpointDirsToClean.forEach(path -> {
        try {
            FileUtils.deleteDirectory(new File(path));
        } catch (IOException exception) {
            Assert.fail("Failed to cleanup temporary checkpoint dirs.");
        }
    });
}
Also used : SortedSet(java.util.SortedSet) TaskModel(org.apache.samza.job.model.TaskModel) Counter(org.apache.samza.metrics.Counter) Pair(org.apache.commons.lang3.tuple.Pair) Map(java.util.Map) StorageEngine(org.apache.samza.storage.StorageEngine) Path(java.nio.file.Path) MapConfig(org.apache.samza.config.MapConfig) TaskName(org.apache.samza.container.TaskName) ImmutableMap(com.google.common.collect.ImmutableMap) StorageManagerUtil(org.apache.samza.storage.StorageManagerUtil) Checkpoint(org.apache.samza.checkpoint.Checkpoint) MetricsRegistry(org.apache.samza.metrics.MetricsRegistry) Collectors(java.util.stream.Collectors) DirDiff(org.apache.samza.storage.blobstore.diff.DirDiff) CheckpointId(org.apache.samza.checkpoint.CheckpointId) List(java.util.List) CompletionStage(java.util.concurrent.CompletionStage) SnapshotIndex(org.apache.samza.storage.blobstore.index.SnapshotIndex) BlobStoreBackupManagerMetrics(org.apache.samza.storage.blobstore.metrics.BlobStoreBackupManagerMetrics) Optional(java.util.Optional) Config(org.apache.samza.config.Config) SnapshotMetadata(org.apache.samza.storage.blobstore.index.SnapshotMetadata) Mockito.eq(org.mockito.Mockito.eq) MoreExecutors(com.google.common.util.concurrent.MoreExecutors) DirIndex(org.apache.samza.storage.blobstore.index.DirIndex) Matchers(org.mockito.Matchers) CheckpointV2(org.apache.samza.checkpoint.CheckpointV2) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) TreeSet(java.util.TreeSet) ArrayList(java.util.ArrayList) CheckpointV1(org.apache.samza.checkpoint.CheckpointV1) Gauge(org.apache.samza.metrics.Gauge) Answer(org.mockito.stubbing.Answer) InvocationOnMock(org.mockito.invocation.InvocationOnMock) ArgumentCaptor(org.mockito.ArgumentCaptor) BlobStoreTestUtil(org.apache.samza.storage.blobstore.util.BlobStoreTestUtil) Mockito.anyLong(org.mockito.Mockito.anyLong) ExecutorService(java.util.concurrent.ExecutorService) JobModel(org.apache.samza.job.model.JobModel) Before(org.junit.Before) BlobStoreUtil(org.apache.samza.storage.blobstore.util.BlobStoreUtil) Files(java.nio.file.Files) Timer(org.apache.samza.metrics.Timer) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) Clock(org.apache.samza.util.Clock) Test(org.junit.Test) File(java.io.File) SamzaException(org.apache.samza.SamzaException) AtomicLong(java.util.concurrent.atomic.AtomicLong) Mockito(org.mockito.Mockito) TaskMode(org.apache.samza.job.model.TaskMode) ContainerModel(org.apache.samza.job.model.ContainerModel) DirDiffUtil(org.apache.samza.storage.blobstore.util.DirDiffUtil) Comparator(java.util.Comparator) Assert(org.junit.Assert) Collections(java.util.Collections) SnapshotIndex(org.apache.samza.storage.blobstore.index.SnapshotIndex) ArrayList(java.util.ArrayList) TaskMode(org.apache.samza.job.model.TaskMode) CheckpointV2(org.apache.samza.checkpoint.CheckpointV2) CompletableFuture(java.util.concurrent.CompletableFuture) TreeSet(java.util.TreeSet) DirDiff(org.apache.samza.storage.blobstore.diff.DirDiff) IOException(java.io.IOException) Checkpoint(org.apache.samza.checkpoint.Checkpoint) TaskName(org.apache.samza.container.TaskName) SnapshotMetadata(org.apache.samza.storage.blobstore.index.SnapshotMetadata) InvocationOnMock(org.mockito.invocation.InvocationOnMock) File(java.io.File) Test(org.junit.Test)

Example 28 with Checkpoint

use of org.apache.samza.checkpoint.Checkpoint in project samza by apache.

the class TestBlobStoreBackupManager method testInitWithInvalidCheckpoint.

@Test
public void testInitWithInvalidCheckpoint() {
    // init called with null checkpoint storeStorageEngineMap
    blobStoreBackupManager.init(null);
    // verify delete snapshot index blob called from init 0 times because prevSnapshotMap returned from init is empty
    // in case of null checkpoint.
    verify(blobStoreUtil, times(0)).deleteSnapshotIndexBlob(anyString(), any(Metadata.class));
    when(blobStoreUtil.getStoreSnapshotIndexes(anyString(), anyString(), anyString(), any(Checkpoint.class), anySetOf(String.class))).thenCallRealMethod();
    // init called with Checkpoint V1 -> unsupported
    Checkpoint checkpoint = new CheckpointV1(new HashMap<>());
    try {
        blobStoreBackupManager.init(checkpoint);
    } catch (SamzaException exception) {
        Assert.fail("Checkpoint V1 is expected to only log warning.");
    }
}
Also used : Checkpoint(org.apache.samza.checkpoint.Checkpoint) CheckpointV1(org.apache.samza.checkpoint.CheckpointV1) SnapshotMetadata(org.apache.samza.storage.blobstore.index.SnapshotMetadata) SamzaException(org.apache.samza.SamzaException) Test(org.junit.Test)

Example 29 with Checkpoint

use of org.apache.samza.checkpoint.Checkpoint in project samza by apache.

the class TestBlobStoreBackupManager method testUploadWithNoPreviousCheckpoints.

@Test
public void testUploadWithNoPreviousCheckpoints() throws IOException {
    // Track directory for post cleanup
    List<String> checkpointDirsToClean = new ArrayList<>();
    // Setup: init local/remote snapshots and back manager with no previous checkpoints
    indexBlobIdAndLocalRemoteSnapshotsPair = setupRemoteAndLocalSnapshots(false);
    Checkpoint checkpoint = new CheckpointV2(checkpointId, new HashMap<>(), ImmutableMap.of(BlobStoreStateBackendFactory.class.getName(), new HashMap<>()));
    blobStoreBackupManager.init(checkpoint);
    // mock: set task store dir to return corresponding test local store and create checkpoint dir
    ArgumentCaptor<String> stringCaptor = ArgumentCaptor.forClass(String.class);
    when(storageManagerUtil.getTaskStoreDir(any(File.class), stringCaptor.capture(), any(TaskName.class), any(TaskMode.class))).then((Answer<File>) invocation -> {
        String storeName = invocation.getArgumentAt(1, String.class);
        String snapshotIndexBlobId = testStoreNameAndSCMMap.get(storeName);
        String storeDir = indexBlobIdAndLocalRemoteSnapshotsPair.get(snapshotIndexBlobId).getLeft();
        try {
            BlobStoreTestUtil.createTestCheckpointDirectory(storeDir, checkpointId.serialize());
            checkpointDirsToClean.add(storeDir + "-" + checkpointId.serialize());
        } catch (IOException e) {
            Assert.fail("Couldn't create checkpoint directory. Test failed.");
        }
        return new File(storeDir);
    });
    ArgumentCaptor<File> storeDirCaptor = ArgumentCaptor.forClass(File.class);
    when(storageManagerUtil.getStoreCheckpointDir(storeDirCaptor.capture(), eq(checkpointId))).thenAnswer(new Answer<String>() {

        @Override
        public String answer(InvocationOnMock invocation) throws Throwable {
            File storeDir = invocation.getArgumentAt(0, File.class);
            return storeDir.getAbsolutePath() + "-" + checkpointId.serialize();
        }
    });
    SortedSet<DirDiff> actualDirDiffs = new TreeSet<>(Comparator.comparing(DirDiff::getDirName));
    // mock: mock putDir and capture DirDiff
    ArgumentCaptor<DirDiff> dirDiffCaptor = ArgumentCaptor.forClass(DirDiff.class);
    ArgumentCaptor<SnapshotMetadata> snapshotMetadataCaptor = ArgumentCaptor.forClass(SnapshotMetadata.class);
    when(blobStoreUtil.putDir(dirDiffCaptor.capture(), snapshotMetadataCaptor.capture())).then((Answer<CompletableFuture<DirIndex>>) invocation -> {
        DirDiff dirDiff = invocation.getArgumentAt(0, DirDiff.class);
        SnapshotMetadata snapshotMetadata = invocation.getArgumentAt(1, SnapshotMetadata.class);
        actualDirDiffs.add(dirDiff);
        SnapshotIndex snapshotIndex = testBlobStore.get(testStoreNameAndSCMMap.get(snapshotMetadata.getStoreName()));
        return CompletableFuture.completedFuture(snapshotIndex.getDirIndex());
    });
    SortedSet<SnapshotIndex> expectedSnapshotIndexesUploaded = indexBlobIdAndLocalRemoteSnapshotsPair.values().stream().map(Pair::getRight).collect(Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(SnapshotIndex::getCreationTimeMillis))));
    String expectedPreviousSnapshotIndexBlobId = "empty";
    // mock: mock putSnapshotIndex and capture previous snapshot index
    SortedSet<SnapshotIndex> actualSnapshotIndexesUploaded = new TreeSet<>(Comparator.comparing(SnapshotIndex::getCreationTimeMillis));
    final String[] actualPreviousSnapshotIndexBlobId = { "empty" };
    ArgumentCaptor<SnapshotIndex> snapshotIndexCaptor = ArgumentCaptor.forClass(SnapshotIndex.class);
    when(blobStoreUtil.putSnapshotIndex(snapshotIndexCaptor.capture())).then((Answer<CompletableFuture<String>>) invocation -> {
        SnapshotIndex snapshotIndex = invocation.getArgumentAt(0, SnapshotIndex.class);
        actualSnapshotIndexesUploaded.add(snapshotIndex);
        if (!snapshotIndex.getPrevSnapshotIndexBlobId().equals(Optional.empty())) {
            actualPreviousSnapshotIndexBlobId[0] = "not-empty";
        }
        return CompletableFuture.completedFuture("random-blob-id");
    });
    // execute
    blobStoreBackupManager.upload(checkpointId, testStoreNameAndSCMMap);
    // setup expected dir diffs after execute: needs checkpoint dirs created in upload()
    TreeSet<DirDiff> expectedDirDiffs = indexBlobIdAndLocalRemoteSnapshotsPair.values().stream().map(localRemoteSnapshotPair -> {
        File localCheckpointDir = new File(localRemoteSnapshotPair.getLeft() + "-" + checkpointId.serialize());
        DirIndex dirIndex = new DirIndex(localCheckpointDir.getName(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList());
        return DirDiffUtil.getDirDiff(localCheckpointDir, dirIndex, DirDiffUtil.areSameFile(false));
    }).collect(Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(DirDiff::getDirName))));
    // assert - asset all DirDiff are put to blob store
    Assert.assertEquals(actualDirDiffs, expectedDirDiffs);
    // assert - assert no previous snapshot indexes were found
    Assert.assertEquals(actualPreviousSnapshotIndexBlobId[0], expectedPreviousSnapshotIndexBlobId);
    // assert - assert all snapshot indexes are uploaded
    Assert.assertEquals(actualSnapshotIndexesUploaded, expectedSnapshotIndexesUploaded);
    // cleanup
    checkpointDirsToClean.forEach(path -> {
        try {
            FileUtils.deleteDirectory(new File(path));
        } catch (IOException exception) {
            Assert.fail("Failed to cleanup temporary checkpoint dirs.");
        }
    });
}
Also used : SortedSet(java.util.SortedSet) TaskModel(org.apache.samza.job.model.TaskModel) Counter(org.apache.samza.metrics.Counter) Pair(org.apache.commons.lang3.tuple.Pair) Map(java.util.Map) StorageEngine(org.apache.samza.storage.StorageEngine) Path(java.nio.file.Path) MapConfig(org.apache.samza.config.MapConfig) TaskName(org.apache.samza.container.TaskName) ImmutableMap(com.google.common.collect.ImmutableMap) StorageManagerUtil(org.apache.samza.storage.StorageManagerUtil) Checkpoint(org.apache.samza.checkpoint.Checkpoint) MetricsRegistry(org.apache.samza.metrics.MetricsRegistry) Collectors(java.util.stream.Collectors) DirDiff(org.apache.samza.storage.blobstore.diff.DirDiff) CheckpointId(org.apache.samza.checkpoint.CheckpointId) List(java.util.List) CompletionStage(java.util.concurrent.CompletionStage) SnapshotIndex(org.apache.samza.storage.blobstore.index.SnapshotIndex) BlobStoreBackupManagerMetrics(org.apache.samza.storage.blobstore.metrics.BlobStoreBackupManagerMetrics) Optional(java.util.Optional) Config(org.apache.samza.config.Config) SnapshotMetadata(org.apache.samza.storage.blobstore.index.SnapshotMetadata) Mockito.eq(org.mockito.Mockito.eq) MoreExecutors(com.google.common.util.concurrent.MoreExecutors) DirIndex(org.apache.samza.storage.blobstore.index.DirIndex) Matchers(org.mockito.Matchers) CheckpointV2(org.apache.samza.checkpoint.CheckpointV2) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) TreeSet(java.util.TreeSet) ArrayList(java.util.ArrayList) CheckpointV1(org.apache.samza.checkpoint.CheckpointV1) Gauge(org.apache.samza.metrics.Gauge) Answer(org.mockito.stubbing.Answer) InvocationOnMock(org.mockito.invocation.InvocationOnMock) ArgumentCaptor(org.mockito.ArgumentCaptor) BlobStoreTestUtil(org.apache.samza.storage.blobstore.util.BlobStoreTestUtil) Mockito.anyLong(org.mockito.Mockito.anyLong) ExecutorService(java.util.concurrent.ExecutorService) JobModel(org.apache.samza.job.model.JobModel) Before(org.junit.Before) BlobStoreUtil(org.apache.samza.storage.blobstore.util.BlobStoreUtil) Files(java.nio.file.Files) Timer(org.apache.samza.metrics.Timer) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) Clock(org.apache.samza.util.Clock) Test(org.junit.Test) File(java.io.File) SamzaException(org.apache.samza.SamzaException) AtomicLong(java.util.concurrent.atomic.AtomicLong) Mockito(org.mockito.Mockito) TaskMode(org.apache.samza.job.model.TaskMode) ContainerModel(org.apache.samza.job.model.ContainerModel) DirDiffUtil(org.apache.samza.storage.blobstore.util.DirDiffUtil) Comparator(java.util.Comparator) Assert(org.junit.Assert) Collections(java.util.Collections) SnapshotIndex(org.apache.samza.storage.blobstore.index.SnapshotIndex) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) TaskMode(org.apache.samza.job.model.TaskMode) CheckpointV2(org.apache.samza.checkpoint.CheckpointV2) CompletableFuture(java.util.concurrent.CompletableFuture) TreeSet(java.util.TreeSet) DirIndex(org.apache.samza.storage.blobstore.index.DirIndex) DirDiff(org.apache.samza.storage.blobstore.diff.DirDiff) IOException(java.io.IOException) Checkpoint(org.apache.samza.checkpoint.Checkpoint) TaskName(org.apache.samza.container.TaskName) SnapshotMetadata(org.apache.samza.storage.blobstore.index.SnapshotMetadata) InvocationOnMock(org.mockito.invocation.InvocationOnMock) File(java.io.File) Test(org.junit.Test)

Example 30 with Checkpoint

use of org.apache.samza.checkpoint.Checkpoint in project samza by apache.

the class TestTaskStorageCommitManager method testSnapshotAndCommitAllFactories.

@Test
public void testSnapshotAndCommitAllFactories() {
    CheckpointManager checkpointManager = mock(CheckpointManager.class);
    TaskBackupManager taskBackupManager1 = mock(TaskBackupManager.class);
    TaskBackupManager taskBackupManager2 = mock(TaskBackupManager.class);
    ContainerStorageManager containerStorageManager = mock(ContainerStorageManager.class);
    Checkpoint checkpoint = mock(Checkpoint.class);
    TaskInstanceMetrics metrics = mock(TaskInstanceMetrics.class);
    Timer checkpointTimer = mock(Timer.class);
    when(metrics.storeCheckpointNs()).thenReturn(checkpointTimer);
    TaskName taskName = new TaskName("task1");
    Map<String, TaskBackupManager> backupManagers = ImmutableMap.of("factory1", taskBackupManager1, "factory2", taskBackupManager2);
    TaskStorageCommitManager cm = new TaskStorageCommitManager(taskName, backupManagers, containerStorageManager, Collections.emptyMap(), new Partition(1), checkpointManager, new MapConfig(), ForkJoinPool.commonPool(), new StorageManagerUtil(), null, metrics);
    when(checkpointManager.readLastCheckpoint(taskName)).thenReturn(checkpoint);
    cm.init();
    verify(taskBackupManager1).init(eq(checkpoint));
    verify(taskBackupManager2).init(eq(checkpoint));
    CheckpointId newCheckpointId = CheckpointId.create();
    Map<String, String> factory1Checkpoints = ImmutableMap.of("store1", "system;stream;1", "store2", "system;stream;2");
    Map<String, String> factory2Checkpoints = ImmutableMap.of("store1", "blobId1", "store2", "blobId2");
    when(containerStorageManager.getAllStores(taskName)).thenReturn(Collections.emptyMap());
    when(taskBackupManager1.snapshot(newCheckpointId)).thenReturn(factory1Checkpoints);
    when(taskBackupManager2.snapshot(newCheckpointId)).thenReturn(factory2Checkpoints);
    when(taskBackupManager1.upload(newCheckpointId, factory1Checkpoints)).thenReturn(CompletableFuture.completedFuture(factory1Checkpoints));
    when(taskBackupManager2.upload(newCheckpointId, factory2Checkpoints)).thenReturn(CompletableFuture.completedFuture(factory2Checkpoints));
    Map<String, Map<String, String>> snapshotSCMs = cm.snapshot(newCheckpointId);
    cm.upload(newCheckpointId, snapshotSCMs);
    // Test flow for snapshot
    verify(taskBackupManager1).snapshot(newCheckpointId);
    verify(taskBackupManager2).snapshot(newCheckpointId);
    // Test flow for upload
    verify(taskBackupManager1).upload(newCheckpointId, factory1Checkpoints);
    verify(taskBackupManager2).upload(newCheckpointId, factory2Checkpoints);
    verify(checkpointTimer).update(anyLong());
}
Also used : SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Partition(org.apache.samza.Partition) CheckpointManager(org.apache.samza.checkpoint.CheckpointManager) TaskInstanceMetrics(org.apache.samza.container.TaskInstanceMetrics) Checkpoint(org.apache.samza.checkpoint.Checkpoint) Timer(org.apache.samza.metrics.Timer) TaskName(org.apache.samza.container.TaskName) CheckpointId(org.apache.samza.checkpoint.CheckpointId) MapConfig(org.apache.samza.config.MapConfig) HashMap(java.util.HashMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) Test(org.junit.Test)

Aggregations

Checkpoint (org.apache.samza.checkpoint.Checkpoint)35 Test (org.junit.Test)22 TaskName (org.apache.samza.container.TaskName)21 HashMap (java.util.HashMap)20 SystemStreamPartition (org.apache.samza.system.SystemStreamPartition)18 Map (java.util.Map)16 Partition (org.apache.samza.Partition)16 CheckpointV1 (org.apache.samza.checkpoint.CheckpointV1)16 CheckpointV2 (org.apache.samza.checkpoint.CheckpointV2)16 File (java.io.File)15 SamzaException (org.apache.samza.SamzaException)15 CheckpointId (org.apache.samza.checkpoint.CheckpointId)14 CompletableFuture (java.util.concurrent.CompletableFuture)13 MapConfig (org.apache.samza.config.MapConfig)13 ImmutableMap (com.google.common.collect.ImmutableMap)12 Collections (java.util.Collections)12 Optional (java.util.Optional)12 CheckpointManager (org.apache.samza.checkpoint.CheckpointManager)12 TaskMode (org.apache.samza.job.model.TaskMode)12 TaskInstanceMetrics (org.apache.samza.container.TaskInstanceMetrics)11