Examples with MapConfig org.apache.samza.config.MapConfig used on opensource projects

Example 86 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestRemoteTableDescriptor method testTablePartToConfig.

@Test
public void testTablePartToConfig() {
    int key = 0;
    TableReadFunction readFn = createMockTableReadFunction();
    when(readFn.toConfig(any(), any())).thenReturn(createConfigPair(key));
    TableWriteFunction writeFn = createMockTableWriteFunction();
    when(writeFn.toConfig(any(), any())).thenReturn(createConfigPair(key));
    RateLimiter rateLimiter = createMockRateLimiter();
    when(((TablePart) rateLimiter).toConfig(any(), any())).thenReturn(createConfigPair(key));
    CreditFunction readCredFn = createMockCreditFunction();
    when(readCredFn.toConfig(any(), any())).thenReturn(createConfigPair(key));
    CreditFunction writeCredFn = createMockCreditFunction();
    when(writeCredFn.toConfig(any(), any())).thenReturn(createConfigPair(key));
    TableRetryPolicy readRetryPolicy = createMockTableRetryPolicy();
    when(readRetryPolicy.toConfig(any(), any())).thenReturn(createConfigPair(key));
    TableRetryPolicy writeRetryPolicy = createMockTableRetryPolicy();
    when(writeRetryPolicy.toConfig(any(), any())).thenReturn(createConfigPair(key));
    Map<String, String> tableConfig = new RemoteTableDescriptor("1").withReadFunction(readFn).withWriteFunction(writeFn).withRateLimiter(rateLimiter, readCredFn, writeCredFn).withReadRetryPolicy(readRetryPolicy).withWriteRetryPolicy(writeRetryPolicy).toConfig(new MapConfig());
    verify(readFn, times(1)).toConfig(any(), any());
    verify(writeFn, times(1)).toConfig(any(), any());
    verify((TablePart) rateLimiter, times(1)).toConfig(any(), any());
    verify(readCredFn, times(1)).toConfig(any(), any());
    verify(writeCredFn, times(1)).toConfig(any(), any());
    verify(readRetryPolicy, times(1)).toConfig(any(), any());
    verify(writeRetryPolicy, times(1)).toConfig(any(), any());
    Assert.assertEquals(tableConfig.get("tables.1.io.read.func.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.write.func.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.ratelimiter.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.read.credit.func.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.write.credit.func.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.read.retry.policy.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.write.retry.policy.k0"), "v0");
}

Example 87 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestRemoteTableDescriptor method testSpecifyBothRateAndRateLimiter.

@Test(expected = IllegalArgumentException.class)
public void testSpecifyBothRateAndRateLimiter() {
    RemoteTableDescriptor desc = new RemoteTableDescriptor("1");
    desc.withReadFunction(createMockTableReadFunction());
    desc.withReadRateLimit(100);
    desc.withRateLimiter(createMockRateLimiter(), null, null);
    desc.toConfig(new MapConfig());
}

Example 88 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestRemoteTableDescriptor method createMockContext.

private Context createMockContext(TableDescriptor tableDescriptor) {
    Context context = mock(Context.class);
    ContainerContext containerContext = mock(ContainerContext.class);
    when(context.getContainerContext()).thenReturn(containerContext);
    MetricsRegistry metricsRegistry = mock(MetricsRegistry.class);
    when(metricsRegistry.newTimer(anyString(), anyString())).thenReturn(mock(Timer.class));
    when(metricsRegistry.newCounter(anyString(), anyString())).thenReturn(mock(Counter.class));
    when(containerContext.getContainerMetricsRegistry()).thenReturn(metricsRegistry);
    TaskContextImpl taskContext = mock(TaskContextImpl.class);
    when(context.getTaskContext()).thenReturn(taskContext);
    TaskName taskName = new TaskName("MyTask");
    TaskModel taskModel = mock(TaskModel.class);
    when(taskModel.getTaskName()).thenReturn(taskName);
    when(context.getTaskContext().getTaskModel()).thenReturn(taskModel);
    ContainerModel containerModel = mock(ContainerModel.class);
    when(containerModel.getTasks()).thenReturn(ImmutableMap.of(taskName, taskModel));
    when(containerContext.getContainerModel()).thenReturn(containerModel);
    String containerId = "container-1";
    JobModel jobModel = mock(JobModel.class);
    when(taskContext.getJobModel()).thenReturn(jobModel);
    when(jobModel.getContainers()).thenReturn(ImmutableMap.of(containerId, containerModel));
    JobContext jobContext = mock(JobContext.class);
    Config jobConfig = new MapConfig(tableDescriptor.toConfig(new MapConfig()));
    when(jobContext.getConfig()).thenReturn(jobConfig);
    when(context.getJobContext()).thenReturn(jobContext);
    return context;
}

Example 89 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestCachingTable method doTestCacheOps.

private void doTestCacheOps(boolean isWriteAround) {
    CachingTableDescriptor desc = new CachingTableDescriptor("1", createDummyTableDescriptor("realTable"), createDummyTableDescriptor("cacheTable"));
    if (isWriteAround) {
        desc.withWriteAround();
    }
    Context context = new MockContext();
    final ReadWriteUpdateTable cacheTable = getMockCache().getLeft();
    final ReadWriteUpdateTable realTable = mock(ReadWriteUpdateTable.class);
    doAnswer(invocation -> {
        String key = invocation.getArgumentAt(0, String.class);
        return CompletableFuture.completedFuture("test-data-" + key);
    }).when(realTable).getAsync(any());
    doReturn(CompletableFuture.completedFuture(null)).when(realTable).putAsync(any(), any());
    doAnswer(invocation -> {
        String tableId = invocation.getArgumentAt(0, String.class);
        if (tableId.equals("realTable")) {
            // cache
            return realTable;
        } else if (tableId.equals("cacheTable")) {
            return cacheTable;
        }
        Assert.fail();
        return null;
    }).when(context.getTaskContext()).getUpdatableTable(anyString());
    when(context.getContainerContext().getContainerMetricsRegistry()).thenReturn(new NoOpMetricsRegistry());
    Map<String, String> tableConfig = desc.toConfig(new MapConfig());
    when(context.getJobContext().getConfig()).thenReturn(new MapConfig(tableConfig));
    CachingTableProvider tableProvider = new CachingTableProvider(desc.getTableId());
    tableProvider.init(context);
    CachingTable cachingTable = (CachingTable) tableProvider.getTable();
    Assert.assertEquals("test-data-1", cachingTable.get("1"));
    verify(realTable, times(1)).getAsync(any());
    // cache miss
    verify(cacheTable, times(1)).get(any());
    verify(cacheTable, times(1)).put(any(), any());
    // 0 hit, 1 request
    Assert.assertEquals(cachingTable.hitRate(), 0.0, 0.0);
    Assert.assertEquals(cachingTable.missRate(), 1.0, 0.0);
    Assert.assertEquals("test-data-1", cachingTable.get("1"));
    // no change
    verify(realTable, times(1)).getAsync(any());
    verify(cacheTable, times(2)).get(any());
    // no change
    verify(cacheTable, times(1)).put(any(), any());
    // 1 hit, 2 requests
    Assert.assertEquals(0.5, cachingTable.hitRate(), 0.0);
    Assert.assertEquals(0.5, cachingTable.missRate(), 0.0);
    cachingTable.put("2", "test-data-XXXX");
    verify(cacheTable, times(isWriteAround ? 1 : 2)).put(any(), any());
    verify(realTable, times(1)).putAsync(any(), any());
    if (isWriteAround) {
        // expects value from table
        Assert.assertEquals("test-data-2", cachingTable.get("2"));
        // should have one more fetch
        verify(realTable, times(2)).getAsync(any());
        // 1 hit, 3 requests
        Assert.assertEquals(cachingTable.hitRate(), 0.33, 0.1);
    } else {
        // expect value from cache
        Assert.assertEquals("test-data-XXXX", cachingTable.get("2"));
        // no change
        verify(realTable, times(1)).getAsync(any());
        // 2 hits, 3 requests
        Assert.assertEquals(cachingTable.hitRate(), 0.66, 0.1);
    }
}

Example 90 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestCachingTable method doTestSerialize.

private void doTestSerialize(TableDescriptor cache) {
    CachingTableDescriptor desc;
    TableDescriptor table = createDummyTableDescriptor("2");
    if (cache == null) {
        desc = new CachingTableDescriptor("1", table).withReadTtl(Duration.ofMinutes(3)).withWriteTtl(Duration.ofMinutes(4)).withCacheSize(1000);
    } else {
        desc = new CachingTableDescriptor("1", table, cache);
    }
    desc.withWriteAround();
    Map<String, String> tableConfig = desc.toConfig(new MapConfig());
    assertEquals("2", CachingTableDescriptor.REAL_TABLE_ID, "1", tableConfig);
    if (cache == null) {
        assertEquals("180000", CachingTableDescriptor.READ_TTL_MS, "1", tableConfig);
        assertEquals("240000", CachingTableDescriptor.WRITE_TTL_MS, "1", tableConfig);
    } else {
        assertEquals(cache.getTableId(), CachingTableDescriptor.CACHE_TABLE_ID, "1", tableConfig);
    }
    assertEquals("true", CachingTableDescriptor.WRITE_AROUND, "1", tableConfig);
}