Examples with Entry org.apache.samza.storage.kv.Entry used on opensource projects

Example 6 with Entry

use of org.apache.samza.storage.kv.Entry in project samza by apache.

the class TestRemoteTable method testPutUpdateAndDeleteDelegation.

@Test
public void testPutUpdateAndDeleteDelegation() {
    TableReadFunction<String, String> readFn = mock(TableReadFunction.class);
    TableWriteFunction writeFn = mock(TableWriteFunction.class);
    doReturn(true).when(writeFn).isRetriable(any());
    doReturn(CompletableFuture.completedFuture(null)).when(writeFn).putAsync(any(), any());
    doReturn(CompletableFuture.completedFuture(null)).when(writeFn).putAsync(any(), any(), any());
    doReturn(CompletableFuture.completedFuture(null)).when(writeFn).putAllAsync(anyCollection());
    doReturn(CompletableFuture.completedFuture(null)).when(writeFn).putAllAsync(anyCollection(), any());
    doReturn(CompletableFuture.completedFuture(null)).when(writeFn).updateAsync(any(), any());
    doReturn(CompletableFuture.completedFuture(null)).when(writeFn).updateAllAsync(anyCollection());
    doReturn(CompletableFuture.completedFuture(null)).when(writeFn).deleteAsync(any());
    doReturn(CompletableFuture.completedFuture(null)).when(writeFn).deleteAsync(any(), any());
    doReturn(CompletableFuture.completedFuture(null)).when(writeFn).deleteAllAsync(anyCollection());
    doReturn(CompletableFuture.completedFuture(null)).when(writeFn).deleteAllAsync(anyCollection(), any());
    doReturn(CompletableFuture.completedFuture(null)).when(writeFn).writeAsync(anyInt(), any());
    RemoteTable<String, String, String> table = getTable("testGetDelegation", readFn, writeFn, Executors.newSingleThreadExecutor(), true);
    verify(readFn, times(1)).init(any(), any());
    // PutAsync
    verify(writeFn, times(0)).putAsync(any(), any());
    verify(writeFn, times(0)).putAsync(any(), any(), any());
    table.putAsync("roo", "bar").join();
    verify(writeFn, times(1)).putAsync(any(), any());
    verify(writeFn, times(0)).putAsync(any(), any(), any());
    table.putAsync("foo", "bar", 3).join();
    verify(writeFn, times(1)).putAsync(any(), any());
    verify(writeFn, times(1)).putAsync(any(), any(), any());
    // PutAllAsync
    verify(writeFn, times(0)).putAllAsync(anyCollection());
    verify(writeFn, times(0)).putAllAsync(anyCollection(), any());
    table.putAllAsync(Arrays.asList(new Entry("foo", "bar"))).join();
    verify(writeFn, times(1)).putAllAsync(anyCollection());
    verify(writeFn, times(0)).putAllAsync(anyCollection(), any());
    table.putAllAsync(Arrays.asList(new Entry("foo", "bar")), 2).join();
    verify(writeFn, times(1)).putAllAsync(anyCollection());
    verify(writeFn, times(1)).putAllAsync(anyCollection(), any());
    // UpdateAsync
    verify(writeFn, times(0)).updateAsync(any(), any());
    table.updateAsync("foo", "bar").join();
    verify(writeFn, times(1)).updateAsync(any(), any());
    // UpdateAllAsync
    verify(writeFn, times(0)).updateAllAsync(anyCollection());
    table.updateAllAsync(Arrays.asList(new Entry<>("foo", "bar"))).join();
    verify(writeFn, times(1)).updateAllAsync(anyCollection());
    // DeleteAsync
    verify(writeFn, times(0)).deleteAsync(any());
    verify(writeFn, times(0)).deleteAsync(any(), any());
    table.deleteAsync("foo").join();
    verify(writeFn, times(1)).deleteAsync(any());
    verify(writeFn, times(0)).deleteAsync(any(), any());
    table.deleteAsync("foo", 2).join();
    verify(writeFn, times(1)).deleteAsync(any());
    verify(writeFn, times(1)).deleteAsync(any(), any());
    // DeleteAllAsync
    verify(writeFn, times(0)).deleteAllAsync(anyCollection());
    verify(writeFn, times(0)).deleteAllAsync(anyCollection(), any());
    table.deleteAllAsync(Arrays.asList("foo")).join();
    verify(writeFn, times(1)).deleteAllAsync(anyCollection());
    verify(writeFn, times(0)).deleteAllAsync(anyCollection(), any());
    table.deleteAllAsync(Arrays.asList("foo"), Arrays.asList(2)).join();
    verify(writeFn, times(1)).deleteAllAsync(anyCollection());
    verify(writeFn, times(1)).deleteAllAsync(anyCollection(), any());
    // WriteAsync
    verify(writeFn, times(0)).writeAsync(anyInt(), any());
    table.writeAsync(1, 2).join();
    verify(writeFn, times(1)).writeAsync(anyInt(), any());
}

Example 7 with Entry

use of org.apache.samza.storage.kv.Entry in project samza by apache.

the class ContainerStorageManager method createSideInputProcessors.

// Create sideInput store processors, one per store per task
private Map<TaskName, Map<String, SideInputsProcessor>> createSideInputProcessors(StorageConfig config, ContainerModel containerModel, Map<TaskName, TaskInstanceMetrics> taskInstanceMetrics) {
    Map<TaskName, Map<String, SideInputsProcessor>> sideInputStoresToProcessors = new HashMap<>();
    containerModel.getTasks().forEach((taskName, taskModel) -> {
        sideInputStoresToProcessors.put(taskName, new HashMap<>());
        TaskMode taskMode = taskModel.getTaskMode();
        for (String storeName : this.taskSideInputStoreSSPs.get(taskName).keySet()) {
            SideInputsProcessor sideInputsProcessor;
            Optional<String> sideInputsProcessorSerializedInstance = config.getSideInputsProcessorSerializedInstance(storeName);
            if (sideInputsProcessorSerializedInstance.isPresent()) {
                sideInputsProcessor = SerdeUtils.deserialize("Side Inputs Processor", sideInputsProcessorSerializedInstance.get());
                LOG.info("Using serialized side-inputs-processor for store: {}, task: {}", storeName, taskName);
            } else if (config.getSideInputsProcessorFactory(storeName).isPresent()) {
                String sideInputsProcessorFactoryClassName = config.getSideInputsProcessorFactory(storeName).get();
                SideInputsProcessorFactory sideInputsProcessorFactory = ReflectionUtil.getObj(sideInputsProcessorFactoryClassName, SideInputsProcessorFactory.class);
                sideInputsProcessor = sideInputsProcessorFactory.getSideInputsProcessor(config, taskInstanceMetrics.get(taskName).registry());
                LOG.info("Using side-inputs-processor from factory: {} for store: {}, task: {}", config.getSideInputsProcessorFactory(storeName).get(), storeName, taskName);
            } else {
                // if this is a active-task with a side-input store but no sideinput-processor-factory defined in config, we rely on upstream validations to fail the deploy
                // if this is a standby-task and the store is a non-side-input changelog store
                // we creating identity sideInputProcessor for stores of standbyTasks
                // have to use the right serde because the sideInput stores are created
                Serde keySerde = serdes.get(config.getStorageKeySerde(storeName).orElseThrow(() -> new SamzaException("Could not find storage key serde for store: " + storeName)));
                Serde msgSerde = serdes.get(config.getStorageMsgSerde(storeName).orElseThrow(() -> new SamzaException("Could not find storage msg serde for store: " + storeName)));
                sideInputsProcessor = new SideInputsProcessor() {

                    @Override
                    public Collection<Entry<?, ?>> process(IncomingMessageEnvelope message, KeyValueStore store) {
                        // Ignore message if the key is null
                        if (message.getKey() == null) {
                            return ImmutableList.of();
                        } else {
                            // Skip serde if the message is null
                            return ImmutableList.of(new Entry<>(keySerde.fromBytes((byte[]) message.getKey()), message.getMessage() == null ? null : msgSerde.fromBytes((byte[]) message.getMessage())));
                        }
                    }
                };
                LOG.info("Using identity side-inputs-processor for store: {}, task: {}", storeName, taskName);
            }
            sideInputStoresToProcessors.get(taskName).put(storeName, sideInputsProcessor);
        }
    });
    return sideInputStoresToProcessors;
}

Example 8 with Entry

use of org.apache.samza.storage.kv.Entry in project samza by apache.

the class TestCachingTable method testGuavaCacheAndRemoteTable.

/**
 * Testing caching in a more realistic scenario with Guava cache + remote table
 */
@Test
public void testGuavaCacheAndRemoteTable() throws Exception {
    String tableId = "testGuavaCacheAndRemoteTable";
    Cache<String, String> guavaCache = CacheBuilder.newBuilder().initialCapacity(100).build();
    final ReadWriteUpdateTable<String, String, String> guavaTable = new GuavaCacheTable<>(tableId + "-cache", guavaCache);
    // It is okay to share rateLimitHelper and async helper for read/write in test
    TableRateLimiter<String, String> readRateLimitHelper = mock(TableRateLimiter.class);
    TableRateLimiter<String, String> writeRateLimitHelper = mock(TableRateLimiter.class);
    TableRateLimiter<String, String> updateRateLimitHelper = mock(TableRateLimiter.class);
    TableReadFunction<String, String> readFn = mock(TableReadFunction.class);
    TableWriteFunction<String, String, String> writeFn = mock(TableWriteFunction.class);
    final RemoteTable<String, String, String> remoteTable = new RemoteTable<>(tableId + "-remote", readFn, writeFn, readRateLimitHelper, writeRateLimitHelper, updateRateLimitHelper, Executors.newSingleThreadExecutor(), null, null, null, null, null, Executors.newSingleThreadExecutor());
    final CachingTable<String, String, String> cachingTable = new CachingTable<>(tableId, remoteTable, guavaTable, false);
    initTables(cachingTable, guavaTable, remoteTable);
    // 4 per readable table (12)
    // 8 per read/write table (24)
    verify(metricsRegistry, times(36)).newCounter(any(), anyString());
    // 3 per readable table (9)
    // 8 per read/write table (24)
    // 1 per remote readable table (1)
    // 2 per remote read/write table (2)
    verify(metricsRegistry, times(36)).newTimer(any(), anyString());
    // 1 per guava table (1)
    // 3 per caching table (2)
    verify(metricsRegistry, times(4)).newGauge(anyString(), any());
    // GET
    doReturn(CompletableFuture.completedFuture("bar")).when(readFn).getAsync(any());
    Assert.assertEquals(cachingTable.getAsync("foo").get(), "bar");
    // Ensure cache is updated
    Assert.assertEquals(guavaCache.getIfPresent("foo"), "bar");
    // PUT
    doReturn(CompletableFuture.completedFuture(null)).when(writeFn).putAsync(any(), any());
    cachingTable.putAsync("foo", "baz").get();
    // Ensure cache is updated
    Assert.assertEquals(guavaCache.getIfPresent("foo"), "baz");
    // DELETE
    doReturn(CompletableFuture.completedFuture(null)).when(writeFn).deleteAsync(any());
    cachingTable.deleteAsync("foo").get();
    // Ensure cache is updated
    Assert.assertNull(guavaCache.getIfPresent("foo"));
    // GET-ALL
    Map<String, String> records = new HashMap<>();
    records.put("foo1", "bar1");
    records.put("foo2", "bar2");
    doReturn(CompletableFuture.completedFuture(records)).when(readFn).getAllAsync(any());
    Assert.assertEquals(cachingTable.getAllAsync(Arrays.asList("foo1", "foo2")).get(), records);
    // Ensure cache is updated
    Assert.assertEquals(guavaCache.getIfPresent("foo1"), "bar1");
    Assert.assertEquals(guavaCache.getIfPresent("foo2"), "bar2");
    // GET-ALL with partial miss
    doReturn(CompletableFuture.completedFuture(Collections.singletonMap("foo3", "bar3"))).when(readFn).getAllAsync(any());
    records = cachingTable.getAllAsync(Arrays.asList("foo1", "foo2", "foo3")).get();
    Assert.assertEquals(records.get("foo3"), "bar3");
    // Ensure cache is updated
    Assert.assertEquals(guavaCache.getIfPresent("foo3"), "bar3");
    // Calling again for the same keys should not trigger IO, ie. no exception is thrown
    CompletableFuture<String> exFuture = new CompletableFuture<>();
    exFuture.completeExceptionally(new RuntimeException("Test exception"));
    doReturn(exFuture).when(readFn).getAllAsync(any());
    cachingTable.getAllAsync(Arrays.asList("foo1", "foo2", "foo3")).get();
    // Partial results should throw
    try {
        cachingTable.getAllAsync(Arrays.asList("foo1", "foo2", "foo5")).get();
        Assert.fail();
    } catch (Exception e) {
    }
    // PUT-ALL
    doReturn(CompletableFuture.completedFuture(null)).when(writeFn).putAllAsync(any());
    List<Entry<String, String>> entries = new ArrayList<>();
    entries.add(new Entry<>("foo1", "bar111"));
    entries.add(new Entry<>("foo2", "bar222"));
    cachingTable.putAllAsync(entries).get();
    // Ensure cache is updated
    Assert.assertEquals(guavaCache.getIfPresent("foo1"), "bar111");
    Assert.assertEquals(guavaCache.getIfPresent("foo2"), "bar222");
    // PUT-ALL with delete
    doReturn(CompletableFuture.completedFuture(null)).when(writeFn).putAllAsync(any());
    doReturn(CompletableFuture.completedFuture(null)).when(writeFn).deleteAllAsync(any());
    entries = new ArrayList<>();
    entries.add(new Entry<>("foo1", "bar111"));
    entries.add(new Entry<>("foo2", null));
    cachingTable.putAllAsync(entries).get();
    // Ensure cache is updated
    Assert.assertNull(guavaCache.getIfPresent("foo2"));
    // At this point, foo1 and foo3 should still exist
    Assert.assertNotNull(guavaCache.getIfPresent("foo1"));
    Assert.assertNotNull(guavaCache.getIfPresent("foo3"));
    // DELETE-ALL
    doReturn(CompletableFuture.completedFuture(null)).when(writeFn).deleteAllAsync(any());
    cachingTable.deleteAllAsync(Arrays.asList("foo1", "foo3")).get();
    // Ensure foo1 and foo3 are gone
    Assert.assertNull(guavaCache.getIfPresent("foo1"));
    Assert.assertNull(guavaCache.getIfPresent("foo3"));
}

Example 9 with Entry

use of org.apache.samza.storage.kv.Entry in project samza by apache.

the class TestAsyncRateLimitedTable method testPutAllAsyncWithArgs.

@Test
public void testPutAllAsyncWithArgs() {
    writeTable.putAllAsync(Arrays.asList(new Entry("1", "2")), Arrays.asList(1)).join();
    verify(writeFn, times(0)).putAllAsync(anyCollection());
    verify(writeFn, times(1)).putAllAsync(anyCollection(), any());
    verify(writeRateLimiter, times(0)).throttle(anyString());
    verify(writeRateLimiter, times(0)).throttle(anyString(), anyString());
    verify(writeRateLimiter, times(0)).throttle(anyCollection());
    verify(writeRateLimiter, times(1)).throttleRecords(anyCollection());
    verify(writeRateLimiter, times(0)).throttle(anyInt(), any());
    verifyReadPartNotCalled();
}

Example 10 with Entry

use of org.apache.samza.storage.kv.Entry in project samza by apache.

the class TestAsyncRateLimitedTable method testUpdateAllAsync.

@Test
public void testUpdateAllAsync() {
    writeTable.updateAllAsync(Arrays.asList(new Entry("1", "2"))).join();
    verify(writeFn, times(1)).updateAllAsync(anyCollection());
    verify(writeRateLimiter, times(0)).throttle(anyString());
    verify(writeRateLimiter, times(0)).throttle(anyString(), anyString());
    verify(writeRateLimiter, times(0)).throttle(anyCollection());
    verify(writeRateLimiter, times(1)).throttleRecords(anyCollection());
    verify(writeRateLimiter, times(0)).throttle(anyInt(), any());
    verifyReadPartNotCalled();
}