Examples with ReadWriteUpdateTable org.apache.samza.table.ReadWriteUpdateTable used on opensource projects

Example 6 with ReadWriteUpdateTable

use of org.apache.samza.table.ReadWriteUpdateTable in project samza by apache.

the class TestCachingTable method doTestCacheOps.

private void doTestCacheOps(boolean isWriteAround) {
    CachingTableDescriptor desc = new CachingTableDescriptor("1", createDummyTableDescriptor("realTable"), createDummyTableDescriptor("cacheTable"));
    if (isWriteAround) {
        desc.withWriteAround();
    }
    Context context = new MockContext();
    final ReadWriteUpdateTable cacheTable = getMockCache().getLeft();
    final ReadWriteUpdateTable realTable = mock(ReadWriteUpdateTable.class);
    doAnswer(invocation -> {
        String key = invocation.getArgumentAt(0, String.class);
        return CompletableFuture.completedFuture("test-data-" + key);
    }).when(realTable).getAsync(any());
    doReturn(CompletableFuture.completedFuture(null)).when(realTable).putAsync(any(), any());
    doAnswer(invocation -> {
        String tableId = invocation.getArgumentAt(0, String.class);
        if (tableId.equals("realTable")) {
            // cache
            return realTable;
        } else if (tableId.equals("cacheTable")) {
            return cacheTable;
        }
        Assert.fail();
        return null;
    }).when(context.getTaskContext()).getUpdatableTable(anyString());
    when(context.getContainerContext().getContainerMetricsRegistry()).thenReturn(new NoOpMetricsRegistry());
    Map<String, String> tableConfig = desc.toConfig(new MapConfig());
    when(context.getJobContext().getConfig()).thenReturn(new MapConfig(tableConfig));
    CachingTableProvider tableProvider = new CachingTableProvider(desc.getTableId());
    tableProvider.init(context);
    CachingTable cachingTable = (CachingTable) tableProvider.getTable();
    Assert.assertEquals("test-data-1", cachingTable.get("1"));
    verify(realTable, times(1)).getAsync(any());
    // cache miss
    verify(cacheTable, times(1)).get(any());
    verify(cacheTable, times(1)).put(any(), any());
    // 0 hit, 1 request
    Assert.assertEquals(cachingTable.hitRate(), 0.0, 0.0);
    Assert.assertEquals(cachingTable.missRate(), 1.0, 0.0);
    Assert.assertEquals("test-data-1", cachingTable.get("1"));
    // no change
    verify(realTable, times(1)).getAsync(any());
    verify(cacheTable, times(2)).get(any());
    // no change
    verify(cacheTable, times(1)).put(any(), any());
    // 1 hit, 2 requests
    Assert.assertEquals(0.5, cachingTable.hitRate(), 0.0);
    Assert.assertEquals(0.5, cachingTable.missRate(), 0.0);
    cachingTable.put("2", "test-data-XXXX");
    verify(cacheTable, times(isWriteAround ? 1 : 2)).put(any(), any());
    verify(realTable, times(1)).putAsync(any(), any());
    if (isWriteAround) {
        // expects value from table
        Assert.assertEquals("test-data-2", cachingTable.get("2"));
        // should have one more fetch
        verify(realTable, times(2)).getAsync(any());
        // 1 hit, 3 requests
        Assert.assertEquals(cachingTable.hitRate(), 0.33, 0.1);
    } else {
        // expect value from cache
        Assert.assertEquals("test-data-XXXX", cachingTable.get("2"));
        // no change
        verify(realTable, times(1)).getAsync(any());
        // 2 hits, 3 requests
        Assert.assertEquals(cachingTable.hitRate(), 0.66, 0.1);
    }
}

Example 7 with ReadWriteUpdateTable

use of org.apache.samza.table.ReadWriteUpdateTable in project samza by apache.

the class TestStreamTableJoinOperatorImpl method testHandleMessage.

@Test
public void testHandleMessage() {
    String tableId = "t1";
    StreamTableJoinOperatorSpec mockJoinOpSpec = mock(StreamTableJoinOperatorSpec.class);
    when(mockJoinOpSpec.getTableId()).thenReturn(tableId);
    when(mockJoinOpSpec.getArgs()).thenReturn(new Object[0]);
    when(mockJoinOpSpec.getJoinFn()).thenReturn(new StreamTableJoinFunction<String, KV<String, String>, KV<String, String>, String>() {

        @Override
        public String apply(KV<String, String> message, KV<String, String> record) {
            if ("1".equals(message.getKey())) {
                Assert.assertEquals("m1", message.getValue());
                Assert.assertEquals("r1", record.getValue());
                return "m1r1";
            } else if ("2".equals(message.getKey())) {
                Assert.assertEquals("m2", message.getValue());
                Assert.assertNull(record);
                return null;
            }
            throw new SamzaException("Should never reach here!");
        }

        @Override
        public String getMessageKey(KV<String, String> message) {
            return message.getKey();
        }

        @Override
        public String getRecordKey(KV<String, String> record) {
            return record.getKey();
        }
    });
    ReadWriteUpdateTable table = mock(ReadWriteUpdateTable.class);
    when(table.getAsync("1")).thenReturn(CompletableFuture.completedFuture("r1"));
    when(table.getAsync("2")).thenReturn(CompletableFuture.completedFuture(null));
    Context context = new MockContext();
    when(context.getTaskContext().getUpdatableTable(tableId)).thenReturn(table);
    MessageCollector mockMessageCollector = mock(MessageCollector.class);
    TaskCoordinator mockTaskCoordinator = mock(TaskCoordinator.class);
    StreamTableJoinOperatorImpl streamTableJoinOperator = new StreamTableJoinOperatorImpl(mockJoinOpSpec, context);
    // Table has the key
    Collection<TestMessageEnvelope> result;
    result = streamTableJoinOperator.handleMessage(KV.of("1", "m1"), mockMessageCollector, mockTaskCoordinator);
    Assert.assertEquals(1, result.size());
    Assert.assertEquals("m1r1", result.iterator().next());
    // Table doesn't have the key
    result = streamTableJoinOperator.handleMessage(KV.of("2", "m2"), mockMessageCollector, mockTaskCoordinator);
    Assert.assertEquals(0, result.size());
}

Example 8 with ReadWriteUpdateTable

use of org.apache.samza.table.ReadWriteUpdateTable in project samza by apache.

the class TestLocalTableRead method testTimerDisabled.

@Test
public void testTimerDisabled() throws Exception {
    ReadWriteUpdateTable table = createTable(true);
    table.get("");
    table.getAsync("").get();
    table.getAll(keys);
    table.getAllAsync(keys).get();
    Assert.assertEquals(2, numGets.getCount());
    Assert.assertEquals(4, numMissedLookups.getCount());
    Assert.assertEquals(2, numGetAlls.getCount());
    Assert.assertEquals(0, getNs.getSnapshot().getAverage(), 0.001);
    Assert.assertEquals(0, getAllNs.getSnapshot().getAverage(), 0.001);
    Assert.assertEquals(0, getCallbackNs.getSnapshot().getAverage(), 0.001);
}

Example 9 with ReadWriteUpdateTable

use of org.apache.samza.table.ReadWriteUpdateTable in project samza by apache.

the class RemoteTableProvider method getTable.

@Override
public ReadWriteUpdateTable getTable() {
    Preconditions.checkNotNull(context, String.format("Table %s not initialized", tableId));
    JavaTableConfig tableConfig = new JavaTableConfig(context.getJobContext().getConfig());
    // Read part
    TableReadFunction readFn = deserializeObject(tableConfig, RemoteTableDescriptor.READ_FN);
    RateLimiter rateLimiter = deserializeObject(tableConfig, RemoteTableDescriptor.RATE_LIMITER);
    if (rateLimiter != null) {
        rateLimiter.init(this.context);
    }
    TableRateLimiter.CreditFunction<?, ?> readCreditFn = deserializeObject(tableConfig, RemoteTableDescriptor.READ_CREDIT_FN);
    TableRateLimiter readRateLimiter = rateLimiter != null && rateLimiter.getSupportedTags().contains(RemoteTableDescriptor.RL_READ_TAG) ? new TableRateLimiter(tableId, rateLimiter, readCreditFn, RemoteTableDescriptor.RL_READ_TAG) : null;
    TableRetryPolicy readRetryPolicy = deserializeObject(tableConfig, RemoteTableDescriptor.READ_RETRY_POLICY);
    // Write part
    // Reuse write rate limiter for update
    TableRateLimiter writeRateLimiter = null;
    TableRetryPolicy writeRetryPolicy = null;
    TableWriteFunction writeFn = deserializeObject(tableConfig, RemoteTableDescriptor.WRITE_FN);
    if (writeFn != null) {
        TableRateLimiter.CreditFunction<?, ?> writeCreditFn = deserializeObject(tableConfig, RemoteTableDescriptor.WRITE_CREDIT_FN);
        writeRateLimiter = rateLimiter != null && rateLimiter.getSupportedTags().contains(RemoteTableDescriptor.RL_WRITE_TAG) ? new TableRateLimiter(tableId, rateLimiter, writeCreditFn, RemoteTableDescriptor.RL_WRITE_TAG) : null;
        writeRetryPolicy = deserializeObject(tableConfig, RemoteTableDescriptor.WRITE_RETRY_POLICY);
    }
    if (readRetryPolicy != null || writeRetryPolicy != null) {
        retryExecutor = createRetryExecutor();
    }
    // Optional executor for future callback/completion. Shared by both read and write operations.
    int callbackPoolSize = Integer.parseInt(tableConfig.getForTable(tableId, RemoteTableDescriptor.ASYNC_CALLBACK_POOL_SIZE, "-1"));
    if (callbackPoolSize > 0) {
        callbackExecutors.computeIfAbsent(tableId, (arg) -> Executors.newFixedThreadPool(callbackPoolSize, (runnable) -> {
            Thread thread = new Thread(runnable);
            thread.setName("table-" + tableId + "-async-callback-pool");
            thread.setDaemon(true);
            return thread;
        }));
    }
    boolean isRateLimited = readRateLimiter != null || writeRateLimiter != null;
    if (isRateLimited) {
        rateLimitingExecutors.computeIfAbsent(tableId, (arg) -> Executors.newSingleThreadExecutor(runnable -> {
            Thread thread = new Thread(runnable);
            thread.setName("table-" + tableId + "-async-executor");
            thread.setDaemon(true);
            return thread;
        }));
    }
    BatchProvider batchProvider = deserializeObject(tableConfig, RemoteTableDescriptor.BATCH_PROVIDER);
    if (batchProvider != null) {
        batchExecutors.computeIfAbsent(tableId, (arg) -> Executors.newSingleThreadScheduledExecutor(runnable -> {
            Thread thread = new Thread(runnable);
            thread.setName("table-" + tableId + "-batch-scheduled-executor");
            thread.setDaemon(true);
            return thread;
        }));
    }
    RemoteTable table = new RemoteTable(tableId, readFn, writeFn, readRateLimiter, writeRateLimiter, writeRateLimiter, rateLimitingExecutors.get(tableId), readRetryPolicy, writeRetryPolicy, retryExecutor, batchProvider, batchExecutors.get(tableId), callbackExecutors.get(tableId));
    table.init(this.context);
    tables.add(table);
    return table;
}

Example 10 with ReadWriteUpdateTable

use of org.apache.samza.table.ReadWriteUpdateTable in project samza by apache.

the class TestStreamTableJoinOperatorImpl method testJoinFunctionIsInvokedOnlyOnce.

/**
 * Ensure join function is not invoked more than once when join function returns null on the first invocation
 */
@Test
public void testJoinFunctionIsInvokedOnlyOnce() {
    final String tableId = "testTable";
    final CountDownLatch joinInvokedLatch = new CountDownLatch(1);
    StreamTableJoinOperatorSpec mockJoinOpSpec = mock(StreamTableJoinOperatorSpec.class);
    when(mockJoinOpSpec.getTableId()).thenReturn(tableId);
    when(mockJoinOpSpec.getArgs()).thenReturn(new Object[0]);
    when(mockJoinOpSpec.getJoinFn()).thenReturn(new StreamTableJoinFunction<String, KV<String, String>, KV<String, String>, String>() {

        @Override
        public String apply(KV<String, String> message, KV<String, String> record) {
            joinInvokedLatch.countDown();
            return null;
        }

        @Override
        public String getMessageKey(KV<String, String> message) {
            return message.getKey();
        }

        @Override
        public String getRecordKey(KV<String, String> record) {
            return record.getKey();
        }
    });
    ReadWriteUpdateTable table = mock(ReadWriteUpdateTable.class);
    when(table.getAsync("1")).thenReturn(CompletableFuture.completedFuture("r1"));
    Context context = new MockContext();
    when(context.getTaskContext().getUpdatableTable(tableId)).thenReturn(table);
    MessageCollector mockMessageCollector = mock(MessageCollector.class);
    TaskCoordinator mockTaskCoordinator = mock(TaskCoordinator.class);
    StreamTableJoinOperatorImpl streamTableJoinOperator = new StreamTableJoinOperatorImpl(mockJoinOpSpec, context);
    // Table has the key
    streamTableJoinOperator.handleMessage(KV.of("1", "m1"), mockMessageCollector, mockTaskCoordinator);
    assertEquals("Join function should only be invoked once", 0, joinInvokedLatch.getCount());
}