Examples with StorageEngine org.apache.samza.storage.StorageEngine used on opensource projects

Example 1 with StorageEngine

use of org.apache.samza.storage.StorageEngine in project samza by apache.

the class TestBaseKeyValueStorageEngineFactory method testAccessLogStore.

@Test
public void testAccessLogStore() {
    Config config = new MapConfig(BASE_CONFIG, DISABLE_CACHE, ACCESS_LOG_ENABLED);
    // AccessLoggedStore requires a changelog SSP
    StorageEngine storageEngine = callGetStorageEngine(config, CHANGELOG_SSP);
    KeyValueStorageEngine<?, ?> keyValueStorageEngine = baseStorageEngineValidation(storageEngine);
    assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, true, false);
    NullSafeKeyValueStore<?, ?> nullSafeKeyValueStore = assertAndCast(keyValueStorageEngine.getWrapperStore(), NullSafeKeyValueStore.class);
    AccessLoggedStore<?, ?> accessLoggedStore = assertAndCast(nullSafeKeyValueStore.getStore(), AccessLoggedStore.class);
    SerializedKeyValueStore<?, ?> serializedKeyValueStore = assertAndCast(accessLoggedStore.getStore(), SerializedKeyValueStore.class);
    LoggedStore<?, ?> loggedStore = assertAndCast(serializedKeyValueStore.getStore(), LoggedStore.class);
    // type generics don't match due to wildcard type, but checking reference equality, so type generics don't matter
    // noinspection AssertEqualsBetweenInconvertibleTypes
    assertEquals(this.rawKeyValueStore, loggedStore.getStore());
}

Example 2 with StorageEngine

use of org.apache.samza.storage.StorageEngine in project samza by apache.

the class TestBaseKeyValueStorageEngineFactory method testDropLargeMessages.

@Test
public void testDropLargeMessages() {
    Config config = new MapConfig(BASE_CONFIG, DISABLE_CACHE, DROP_LARGE_MESSAGES);
    StorageEngine storageEngine = callGetStorageEngine(config, null);
    KeyValueStorageEngine<?, ?> keyValueStorageEngine = baseStorageEngineValidation(storageEngine);
    assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, false, false);
    NullSafeKeyValueStore<?, ?> nullSafeKeyValueStore = assertAndCast(keyValueStorageEngine.getWrapperStore(), NullSafeKeyValueStore.class);
    SerializedKeyValueStore<?, ?> serializedKeyValueStore = assertAndCast(nullSafeKeyValueStore.getStore(), SerializedKeyValueStore.class);
    LargeMessageSafeStore largeMessageSafeStore = assertAndCast(serializedKeyValueStore.getStore(), LargeMessageSafeStore.class);
    assertEquals(this.rawKeyValueStore, largeMessageSafeStore.getStore());
}

Example 3 with StorageEngine

use of org.apache.samza.storage.StorageEngine in project samza by apache.

the class TestBaseKeyValueStorageEngineFactory method testDisallowLargeMessagesWithCache.

@Test
public void testDisallowLargeMessagesWithCache() {
    Config config = new MapConfig(BASE_CONFIG, DISALLOW_LARGE_MESSAGES);
    StorageEngine storageEngine = callGetStorageEngine(config, null);
    KeyValueStorageEngine<?, ?> keyValueStorageEngine = baseStorageEngineValidation(storageEngine);
    assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, false, false);
    NullSafeKeyValueStore<?, ?> nullSafeKeyValueStore = assertAndCast(keyValueStorageEngine.getWrapperStore(), NullSafeKeyValueStore.class);
    SerializedKeyValueStore<?, ?> serializedKeyValueStore = assertAndCast(nullSafeKeyValueStore.getStore(), SerializedKeyValueStore.class);
    LargeMessageSafeStore largeMessageSafeStore = assertAndCast(serializedKeyValueStore.getStore(), LargeMessageSafeStore.class);
    CachedStore<?, ?> cachedStore = assertAndCast(largeMessageSafeStore.getStore(), CachedStore.class);
    // type generics don't match due to wildcard type, but checking reference equality, so type generics don't matter
    // noinspection AssertEqualsBetweenInconvertibleTypes
    assertEquals(this.rawKeyValueStore, cachedStore.getStore());
}

Example 4 with StorageEngine

use of org.apache.samza.storage.StorageEngine in project samza by apache.

the class TestBlobStoreBackupManager method setup.

@Before
public void setup() throws Exception {
    when(clock.currentTimeMillis()).thenReturn(1234567L);
    // setup test local and remote snapshots
    indexBlobIdAndLocalRemoteSnapshotsPair = setupRemoteAndLocalSnapshots(true);
    // setup test store name and SCMs map
    testStoreNameAndSCMMap = setupTestStoreSCMMapAndStoreBackedFactoryConfig(indexBlobIdAndLocalRemoteSnapshotsPair);
    // setup: setup task backup manager with expected storeName->storageEngine map
    testStoreNameAndSCMMap.forEach((storeName, scm) -> storeStorageEngineMap.put(storeName, null));
    mapConfig.putAll(new MapConfig(ImmutableMap.of("job.name", jobName, "job.id", jobId)));
    Config config = new MapConfig(mapConfig);
    // Mock - return snapshot index for blob id from test blob store map
    ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);
    when(blobStoreUtil.getSnapshotIndex(captor.capture(), any(Metadata.class))).then((Answer<CompletableFuture<SnapshotIndex>>) invocation -> {
        String blobId = invocation.getArgumentAt(0, String.class);
        return CompletableFuture.completedFuture(testBlobStore.get(blobId));
    });
    // doNothing().when(blobStoreManager).init();
    when(taskModel.getTaskName().getTaskName()).thenReturn(taskName);
    when(taskModel.getTaskMode()).thenReturn(TaskMode.Active);
    when(metricsRegistry.newCounter(anyString(), anyString())).thenReturn(counter);
    when(metricsRegistry.newGauge(anyString(), anyString(), anyLong())).thenReturn(longGauge);
    when(metricsRegistry.newGauge(anyString(), anyString(), any(AtomicLong.class))).thenReturn(atomicLongGauge);
    when(atomicLongGauge.getValue()).thenReturn(new AtomicLong());
    when(metricsRegistry.newTimer(anyString(), anyString())).thenReturn(timer);
    blobStoreTaskBackupMetrics = new BlobStoreBackupManagerMetrics(metricsRegistry);
    blobStoreBackupManager = new MockBlobStoreBackupManager(jobModel, containerModel, taskModel, mockExecutor, blobStoreTaskBackupMetrics, config, Files.createTempDirectory("logged-store-").toFile(), storageManagerUtil, blobStoreManager);
}

Example 5 with StorageEngine

use of org.apache.samza.storage.StorageEngine in project samza by apache.

the class TestBaseKeyValueStorageEngineFactory method testRawStoreOnly.

@Test
public void testRawStoreOnly() {
    Config config = new MapConfig(BASE_CONFIG, DISABLE_CACHE);
    StorageEngine storageEngine = callGetStorageEngine(config, null);
    KeyValueStorageEngine<?, ?> keyValueStorageEngine = baseStorageEngineValidation(storageEngine);
    assertStoreProperties(keyValueStorageEngine.getStoreProperties(), true, false, false);
    NullSafeKeyValueStore<?, ?> nullSafeKeyValueStore = assertAndCast(keyValueStorageEngine.getWrapperStore(), NullSafeKeyValueStore.class);
    SerializedKeyValueStore<?, ?> serializedKeyValueStore = assertAndCast(nullSafeKeyValueStore.getStore(), SerializedKeyValueStore.class);
    assertEquals(this.rawKeyValueStore, serializedKeyValueStore.getStore());
}