Examples with CachingTableDescriptor org.apache.samza.table.descriptors.CachingTableDescriptor used on opensource projects

Example 1 with CachingTableDescriptor

use of org.apache.samza.table.descriptors.CachingTableDescriptor in project samza by apache.

the class TestCachingTable method doTestCacheOps.

private void doTestCacheOps(boolean isWriteAround) {
    CachingTableDescriptor desc = new CachingTableDescriptor("1", createDummyTableDescriptor("realTable"), createDummyTableDescriptor("cacheTable"));
    if (isWriteAround) {
        desc.withWriteAround();
    }
    Context context = new MockContext();
    final ReadWriteUpdateTable cacheTable = getMockCache().getLeft();
    final ReadWriteUpdateTable realTable = mock(ReadWriteUpdateTable.class);
    doAnswer(invocation -> {
        String key = invocation.getArgumentAt(0, String.class);
        return CompletableFuture.completedFuture("test-data-" + key);
    }).when(realTable).getAsync(any());
    doReturn(CompletableFuture.completedFuture(null)).when(realTable).putAsync(any(), any());
    doAnswer(invocation -> {
        String tableId = invocation.getArgumentAt(0, String.class);
        if (tableId.equals("realTable")) {
            // cache
            return realTable;
        } else if (tableId.equals("cacheTable")) {
            return cacheTable;
        }
        Assert.fail();
        return null;
    }).when(context.getTaskContext()).getUpdatableTable(anyString());
    when(context.getContainerContext().getContainerMetricsRegistry()).thenReturn(new NoOpMetricsRegistry());
    Map<String, String> tableConfig = desc.toConfig(new MapConfig());
    when(context.getJobContext().getConfig()).thenReturn(new MapConfig(tableConfig));
    CachingTableProvider tableProvider = new CachingTableProvider(desc.getTableId());
    tableProvider.init(context);
    CachingTable cachingTable = (CachingTable) tableProvider.getTable();
    Assert.assertEquals("test-data-1", cachingTable.get("1"));
    verify(realTable, times(1)).getAsync(any());
    // cache miss
    verify(cacheTable, times(1)).get(any());
    verify(cacheTable, times(1)).put(any(), any());
    // 0 hit, 1 request
    Assert.assertEquals(cachingTable.hitRate(), 0.0, 0.0);
    Assert.assertEquals(cachingTable.missRate(), 1.0, 0.0);
    Assert.assertEquals("test-data-1", cachingTable.get("1"));
    // no change
    verify(realTable, times(1)).getAsync(any());
    verify(cacheTable, times(2)).get(any());
    // no change
    verify(cacheTable, times(1)).put(any(), any());
    // 1 hit, 2 requests
    Assert.assertEquals(0.5, cachingTable.hitRate(), 0.0);
    Assert.assertEquals(0.5, cachingTable.missRate(), 0.0);
    cachingTable.put("2", "test-data-XXXX");
    verify(cacheTable, times(isWriteAround ? 1 : 2)).put(any(), any());
    verify(realTable, times(1)).putAsync(any(), any());
    if (isWriteAround) {
        // expects value from table
        Assert.assertEquals("test-data-2", cachingTable.get("2"));
        // should have one more fetch
        verify(realTable, times(2)).getAsync(any());
        // 1 hit, 3 requests
        Assert.assertEquals(cachingTable.hitRate(), 0.33, 0.1);
    } else {
        // expect value from cache
        Assert.assertEquals("test-data-XXXX", cachingTable.get("2"));
        // no change
        verify(realTable, times(1)).getAsync(any());
        // 2 hits, 3 requests
        Assert.assertEquals(cachingTable.hitRate(), 0.66, 0.1);
    }
}

Example 2 with CachingTableDescriptor

use of org.apache.samza.table.descriptors.CachingTableDescriptor in project samza by apache.

the class TestCachingTable method doTestSerialize.

private void doTestSerialize(TableDescriptor cache) {
    CachingTableDescriptor desc;
    TableDescriptor table = createDummyTableDescriptor("2");
    if (cache == null) {
        desc = new CachingTableDescriptor("1", table).withReadTtl(Duration.ofMinutes(3)).withWriteTtl(Duration.ofMinutes(4)).withCacheSize(1000);
    } else {
        desc = new CachingTableDescriptor("1", table, cache);
    }
    desc.withWriteAround();
    Map<String, String> tableConfig = desc.toConfig(new MapConfig());
    assertEquals("2", CachingTableDescriptor.REAL_TABLE_ID, "1", tableConfig);
    if (cache == null) {
        assertEquals("180000", CachingTableDescriptor.READ_TTL_MS, "1", tableConfig);
        assertEquals("240000", CachingTableDescriptor.WRITE_TTL_MS, "1", tableConfig);
    } else {
        assertEquals(cache.getTableId(), CachingTableDescriptor.CACHE_TABLE_ID, "1", tableConfig);
    }
    assertEquals("true", CachingTableDescriptor.WRITE_AROUND, "1", tableConfig);
}

Example 3 with CachingTableDescriptor

use of org.apache.samza.table.descriptors.CachingTableDescriptor in project samza by apache.

the class ScanTranslator method translate.

// ScanMapFunction
void translate(final TableScan tableScan, final String queryLogicalId, final String logicalOpId, final TranslatorContext context, Map<String, DelegatingSystemDescriptor> systemDescriptors, Map<String, MessageStream<SamzaSqlInputMessage>> inputMsgStreams) {
    StreamApplicationDescriptor streamAppDesc = context.getStreamAppDescriptor();
    List<String> tableNameParts = tableScan.getTable().getQualifiedName();
    String sourceName = SqlIOConfig.getSourceFromSourceParts(tableNameParts);
    Validate.isTrue(relMsgConverters.containsKey(sourceName), String.format("Unknown source %s", sourceName));
    SqlIOConfig sqlIOConfig = systemStreamConfig.get(sourceName);
    final String systemName = sqlIOConfig.getSystemName();
    final String streamId = sqlIOConfig.getStreamId();
    final String source = sqlIOConfig.getSource();
    final boolean isRemoteTable = sqlIOConfig.getTableDescriptor().isPresent() && (sqlIOConfig.getTableDescriptor().get() instanceof RemoteTableDescriptor || sqlIOConfig.getTableDescriptor().get() instanceof CachingTableDescriptor);
    // descriptor to load the local table.
    if (isRemoteTable) {
        return;
    }
    // set the wrapper input transformer (SamzaSqlInputTransformer) in system descriptor
    DelegatingSystemDescriptor systemDescriptor = systemDescriptors.get(systemName);
    if (systemDescriptor == null) {
        systemDescriptor = new DelegatingSystemDescriptor(systemName, new SamzaSqlInputTransformer());
        systemDescriptors.put(systemName, systemDescriptor);
    } else {
        /* in SamzaSQL, there should be no systemDescriptor setup by user, so this branch happens only
       * in case of Fan-OUT (i.e., same input stream used in multiple sql statements), or when same input
       * used twice in same sql statement (e.g., select ... from input as i1, input as i2 ...), o.w., throw error */
        if (systemDescriptor.getTransformer().isPresent()) {
            InputTransformer existingTransformer = systemDescriptor.getTransformer().get();
            if (!(existingTransformer instanceof SamzaSqlInputTransformer)) {
                throw new SamzaException("SamzaSQL Exception: existing transformer for " + systemName + " is not SamzaSqlInputTransformer");
            }
        }
    }
    InputDescriptor inputDescriptor = systemDescriptor.getInputDescriptor(streamId, new NoOpSerde<>());
    if (!inputMsgStreams.containsKey(source)) {
        MessageStream<SamzaSqlInputMessage> inputMsgStream = streamAppDesc.getInputStream(inputDescriptor);
        inputMsgStreams.put(source, inputMsgStream.map(new SystemMessageMapperFunction(source, queryId)));
    }
    MessageStream<SamzaSqlRelMessage> samzaSqlRelMessageStream = inputMsgStreams.get(source).filter(new FilterSystemMessageFunction(sourceName, queryId)).map(new ScanMapFunction(sourceName, queryId, queryLogicalId, logicalOpId));
    context.registerMessageStream(tableScan.getId(), samzaSqlRelMessageStream);
}