Examples with JavaTableConfig org.apache.samza.config.JavaTableConfig used on opensource projects

Example 1 with JavaTableConfig

use of org.apache.samza.config.JavaTableConfig in project samza by apache.

the class TableManager method addTable.

private void addTable(String tableId, Config config) {
    if (tableContexts.containsKey(tableId)) {
        throw new SamzaException("Table " + tableId + " already exists");
    }
    JavaTableConfig tableConfig = new JavaTableConfig(config);
    String providerFactoryClassName = tableConfig.getTableProviderFactory(tableId);
    TableProviderFactory tableProviderFactory = ReflectionUtil.getObj(providerFactoryClassName, TableProviderFactory.class);
    TableCtx ctx = new TableCtx();
    ctx.tableProvider = tableProviderFactory.getTableProvider(tableId);
    tableContexts.put(tableId, ctx);
}

Example 2 with JavaTableConfig

use of org.apache.samza.config.JavaTableConfig in project samza by apache.

the class CachingTableProvider method getTable.

@Override
public ReadWriteUpdateTable getTable() {
    Preconditions.checkNotNull(context, String.format("Table %s not initialized", tableId));
    JavaTableConfig tableConfig = new JavaTableConfig(context.getJobContext().getConfig());
    String realTableId = tableConfig.getForTable(tableId, CachingTableDescriptor.REAL_TABLE_ID);
    ReadWriteUpdateTable table = this.context.getTaskContext().getUpdatableTable(realTableId);
    String cacheTableId = tableConfig.getForTable(tableId, CachingTableDescriptor.CACHE_TABLE_ID);
    ReadWriteUpdateTable cache;
    if (cacheTableId != null) {
        cache = this.context.getTaskContext().getUpdatableTable(cacheTableId);
    } else {
        cache = createDefaultCacheTable(realTableId, tableConfig);
        defaultCaches.add(cache);
    }
    boolean isWriteAround = Boolean.parseBoolean(tableConfig.getForTable(tableId, CachingTableDescriptor.WRITE_AROUND));
    CachingTable cachingTable = new CachingTable(tableId, table, cache, isWriteAround);
    cachingTable.init(this.context);
    return cachingTable;
}

Example 3 with JavaTableConfig

use of org.apache.samza.config.JavaTableConfig in project samza by apache.

the class TestTableConfigGenerator method testWithSerdes.

@Test
public void testWithSerdes() {
    List<TableDescriptor> descriptors = Arrays.asList(new MockLocalTableDescriptor("t1", KVSerde.of(new StringSerde(), new IntegerSerde())), new MockLocalTableDescriptor("t2", KVSerde.of(new StringSerde(), new IntegerSerde())));
    Config jobConfig = new MapConfig(TableConfigGenerator.generateSerdeConfig(descriptors));
    JavaTableConfig javaTableConfig = new JavaTableConfig(jobConfig);
    assertNotNull(javaTableConfig.getKeySerde("t1"));
    assertNotNull(javaTableConfig.getMsgSerde("t1"));
    assertNotNull(javaTableConfig.getKeySerde("t2"));
    assertNotNull(javaTableConfig.getMsgSerde("t2"));
    MapConfig tableConfig = new MapConfig(TableConfigGenerator.generate(jobConfig, descriptors));
    javaTableConfig = new JavaTableConfig(tableConfig);
    assertNotNull(javaTableConfig.getTableProviderFactory("t1"));
    assertNotNull(javaTableConfig.getTableProviderFactory("t2"));
}

Example 4 with JavaTableConfig

use of org.apache.samza.config.JavaTableConfig in project samza by apache.

the class RemoteTableProvider method getTable.

@Override
public ReadWriteUpdateTable getTable() {
    Preconditions.checkNotNull(context, String.format("Table %s not initialized", tableId));
    JavaTableConfig tableConfig = new JavaTableConfig(context.getJobContext().getConfig());
    // Read part
    TableReadFunction readFn = deserializeObject(tableConfig, RemoteTableDescriptor.READ_FN);
    RateLimiter rateLimiter = deserializeObject(tableConfig, RemoteTableDescriptor.RATE_LIMITER);
    if (rateLimiter != null) {
        rateLimiter.init(this.context);
    }
    TableRateLimiter.CreditFunction<?, ?> readCreditFn = deserializeObject(tableConfig, RemoteTableDescriptor.READ_CREDIT_FN);
    TableRateLimiter readRateLimiter = rateLimiter != null && rateLimiter.getSupportedTags().contains(RemoteTableDescriptor.RL_READ_TAG) ? new TableRateLimiter(tableId, rateLimiter, readCreditFn, RemoteTableDescriptor.RL_READ_TAG) : null;
    TableRetryPolicy readRetryPolicy = deserializeObject(tableConfig, RemoteTableDescriptor.READ_RETRY_POLICY);
    // Write part
    // Reuse write rate limiter for update
    TableRateLimiter writeRateLimiter = null;
    TableRetryPolicy writeRetryPolicy = null;
    TableWriteFunction writeFn = deserializeObject(tableConfig, RemoteTableDescriptor.WRITE_FN);
    if (writeFn != null) {
        TableRateLimiter.CreditFunction<?, ?> writeCreditFn = deserializeObject(tableConfig, RemoteTableDescriptor.WRITE_CREDIT_FN);
        writeRateLimiter = rateLimiter != null && rateLimiter.getSupportedTags().contains(RemoteTableDescriptor.RL_WRITE_TAG) ? new TableRateLimiter(tableId, rateLimiter, writeCreditFn, RemoteTableDescriptor.RL_WRITE_TAG) : null;
        writeRetryPolicy = deserializeObject(tableConfig, RemoteTableDescriptor.WRITE_RETRY_POLICY);
    }
    if (readRetryPolicy != null || writeRetryPolicy != null) {
        retryExecutor = createRetryExecutor();
    }
    // Optional executor for future callback/completion. Shared by both read and write operations.
    int callbackPoolSize = Integer.parseInt(tableConfig.getForTable(tableId, RemoteTableDescriptor.ASYNC_CALLBACK_POOL_SIZE, "-1"));
    if (callbackPoolSize > 0) {
        callbackExecutors.computeIfAbsent(tableId, (arg) -> Executors.newFixedThreadPool(callbackPoolSize, (runnable) -> {
            Thread thread = new Thread(runnable);
            thread.setName("table-" + tableId + "-async-callback-pool");
            thread.setDaemon(true);
            return thread;
        }));
    }
    boolean isRateLimited = readRateLimiter != null || writeRateLimiter != null;
    if (isRateLimited) {
        rateLimitingExecutors.computeIfAbsent(tableId, (arg) -> Executors.newSingleThreadExecutor(runnable -> {
            Thread thread = new Thread(runnable);
            thread.setName("table-" + tableId + "-async-executor");
            thread.setDaemon(true);
            return thread;
        }));
    }
    BatchProvider batchProvider = deserializeObject(tableConfig, RemoteTableDescriptor.BATCH_PROVIDER);
    if (batchProvider != null) {
        batchExecutors.computeIfAbsent(tableId, (arg) -> Executors.newSingleThreadScheduledExecutor(runnable -> {
            Thread thread = new Thread(runnable);
            thread.setName("table-" + tableId + "-batch-scheduled-executor");
            thread.setDaemon(true);
            return thread;
        }));
    }
    RemoteTable table = new RemoteTable(tableId, readFn, writeFn, readRateLimiter, writeRateLimiter, writeRateLimiter, rateLimitingExecutors.get(tableId), readRetryPolicy, writeRetryPolicy, retryExecutor, batchProvider, batchExecutors.get(tableId), callbackExecutors.get(tableId));
    table.init(this.context);
    tables.add(table);
    return table;
}

Example 5 with JavaTableConfig

use of org.apache.samza.config.JavaTableConfig in project samza by apache.

the class GuavaCacheTableProvider method getTable.

@Override
public ReadWriteUpdateTable getTable() {
    Preconditions.checkNotNull(context, String.format("Table %s not initialized", tableId));
    JavaTableConfig tableConfig = new JavaTableConfig(context.getJobContext().getConfig());
    Cache guavaCache = SerdeUtils.deserialize(GuavaCacheTableDescriptor.GUAVA_CACHE, tableConfig.getForTable(tableId, GuavaCacheTableDescriptor.GUAVA_CACHE));
    GuavaCacheTable table = new GuavaCacheTable(tableId, guavaCache);
    table.init(this.context);
    guavaTables.add(table);
    return table;
}