Search in sources :

Example 1 with RemoteTableDescriptor

use of org.apache.samza.table.descriptors.RemoteTableDescriptor in project samza by apache.

the class TestRemoteTableDescriptor method testReadWriteRateLimitToConfig.

@Test
public void testReadWriteRateLimitToConfig() {
    Map<String, String> tableConfig = new RemoteTableDescriptor("1").withReadFunction(createMockTableReadFunction()).withReadRetryPolicy(new TableRetryPolicy()).withWriteRateLimit(1000).withReadRateLimit(2000).toConfig(new MapConfig());
    Assert.assertEquals(String.valueOf(2000), tableConfig.get("tables.1.io.read.credits"));
    Assert.assertEquals(String.valueOf(1000), tableConfig.get("tables.1.io.write.credits"));
}
Also used : TableRetryPolicy(org.apache.samza.table.retry.TableRetryPolicy) RemoteTableDescriptor(org.apache.samza.table.descriptors.RemoteTableDescriptor) Mockito.anyString(org.mockito.Mockito.anyString) MapConfig(org.apache.samza.config.MapConfig) Test(org.junit.Test)

Example 2 with RemoteTableDescriptor

use of org.apache.samza.table.descriptors.RemoteTableDescriptor in project samza by apache.

the class TestRemoteTableDescriptor method testSerializeNullReadFunction.

@Test(expected = IllegalArgumentException.class)
public void testSerializeNullReadFunction() {
    RemoteTableDescriptor desc = new RemoteTableDescriptor("1");
    Map<String, String> tableConfig = desc.toConfig(new MapConfig());
    Assert.assertTrue(tableConfig.containsKey(RemoteTableDescriptor.READ_FN));
}
Also used : RemoteTableDescriptor(org.apache.samza.table.descriptors.RemoteTableDescriptor) Mockito.anyString(org.mockito.Mockito.anyString) MapConfig(org.apache.samza.config.MapConfig) Test(org.junit.Test)

Example 3 with RemoteTableDescriptor

use of org.apache.samza.table.descriptors.RemoteTableDescriptor in project samza by apache.

the class TestRemoteTableDescriptor method doTestDeserializeReadFunctionAndLimiter.

private void doTestDeserializeReadFunctionAndLimiter(boolean rateOnly, boolean rlGets, boolean rlPuts) {
    int numRateLimitOps = (rlGets ? 1 : 0) + (rlPuts ? 1 : 0);
    RemoteTableDescriptor<String, String, String> desc = new RemoteTableDescriptor("1").withReadFunction(createMockTableReadFunction()).withReadRetryPolicy(new TableRetryPolicy().withRetryPredicate((ex) -> false)).withWriteFunction(createMockTableWriteFunction()).withAsyncCallbackExecutorPoolSize(10);
    if (rateOnly) {
        if (rlGets) {
            desc.withReadRateLimit(1000);
        } else {
            desc.withReadRateLimiterDisabled();
        }
        if (rlPuts) {
            desc.withWriteRateLimit(2000);
        } else {
            desc.withWriteRateLimiterDisabled();
        }
    } else {
        if (numRateLimitOps > 0) {
            Map<String, Integer> tagCredits = new HashMap<>();
            if (rlGets) {
                tagCredits.put(RemoteTableDescriptor.RL_READ_TAG, 1000);
            } else {
                desc.withReadRateLimiterDisabled();
            }
            if (rlPuts) {
                tagCredits.put(RemoteTableDescriptor.RL_WRITE_TAG, 2000);
            } else {
                desc.withWriteRateLimiterDisabled();
            }
            // Spy the rate limiter to verify call count
            RateLimiter rateLimiter = spy(new EmbeddedTaggedRateLimiter(tagCredits));
            desc.withRateLimiter(rateLimiter, new CountingCreditFunction(), new CountingCreditFunction());
        } else {
            desc.withRateLimiterDisabled();
        }
    }
    RemoteTableProvider provider = new RemoteTableProvider(desc.getTableId());
    provider.init(createMockContext(desc));
    Table table = provider.getTable();
    Assert.assertTrue(table instanceof RemoteTable);
    RemoteTable rwTable = (RemoteTable) table;
    AsyncReadWriteUpdateTable delegate = TestUtils.getFieldValue(rwTable, "asyncTable");
    Assert.assertTrue(delegate instanceof AsyncRetriableTable);
    if (rlGets || rlPuts) {
        delegate = TestUtils.getFieldValue(delegate, "table");
        Assert.assertTrue(delegate instanceof AsyncRateLimitedTable);
    }
    delegate = TestUtils.getFieldValue(delegate, "table");
    Assert.assertTrue(delegate instanceof AsyncRemoteTable);
    if (numRateLimitOps > 0) {
        TableRateLimiter readRateLimiter = TestUtils.getFieldValue(rwTable, "readRateLimiter");
        TableRateLimiter writeRateLimiter = TestUtils.getFieldValue(rwTable, "writeRateLimiter");
        Assert.assertTrue(!rlGets || readRateLimiter != null);
        Assert.assertTrue(!rlPuts || writeRateLimiter != null);
    }
    ThreadPoolExecutor callbackExecutor = TestUtils.getFieldValue(rwTable, "callbackExecutor");
    Assert.assertEquals(10, callbackExecutor.getCorePoolSize());
}
Also used : ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) TableDescriptor(org.apache.samza.table.descriptors.TableDescriptor) TaskModel(org.apache.samza.job.model.TaskModel) HashMap(java.util.HashMap) Mockito.spy(org.mockito.Mockito.spy) TableReadFunction(org.apache.samza.table.remote.TableReadFunction) CreditFunction(org.apache.samza.table.remote.TableRateLimiter.CreditFunction) TestUtils(org.apache.samza.testUtils.TestUtils) RemoteTable(org.apache.samza.table.remote.RemoteTable) Counter(org.apache.samza.metrics.Counter) AsyncRetriableTable(org.apache.samza.table.retry.AsyncRetriableTable) Map(java.util.Map) TableWriteFunction(org.apache.samza.table.remote.TableWriteFunction) MapConfig(org.apache.samza.config.MapConfig) Mockito.anyString(org.mockito.Mockito.anyString) JobModel(org.apache.samza.job.model.JobModel) Table(org.apache.samza.table.Table) AsyncRateLimitedTable(org.apache.samza.table.ratelimit.AsyncRateLimitedTable) TaskName(org.apache.samza.container.TaskName) EmbeddedTaggedRateLimiter(org.apache.samza.util.EmbeddedTaggedRateLimiter) ImmutableMap(com.google.common.collect.ImmutableMap) JobContext(org.apache.samza.context.JobContext) Timer(org.apache.samza.metrics.Timer) ContainerContext(org.apache.samza.context.ContainerContext) TableRetryPolicy(org.apache.samza.table.retry.TableRetryPolicy) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) JavaTableConfig(org.apache.samza.config.JavaTableConfig) MetricsRegistry(org.apache.samza.metrics.MetricsRegistry) Mockito.when(org.mockito.Mockito.when) AsyncRemoteTable(org.apache.samza.table.remote.AsyncRemoteTable) Mockito.verify(org.mockito.Mockito.verify) Context(org.apache.samza.context.Context) ContainerModel(org.apache.samza.job.model.ContainerModel) RateLimiter(org.apache.samza.util.RateLimiter) RemoteTableProvider(org.apache.samza.table.remote.RemoteTableProvider) Config(org.apache.samza.config.Config) TablePart(org.apache.samza.table.remote.TablePart) Mockito.withSettings(org.mockito.Mockito.withSettings) TaskContextImpl(org.apache.samza.context.TaskContextImpl) AsyncReadWriteUpdateTable(org.apache.samza.table.AsyncReadWriteUpdateTable) Assert(org.junit.Assert) Mockito.any(org.mockito.Mockito.any) RemoteTableDescriptor(org.apache.samza.table.descriptors.RemoteTableDescriptor) TableRateLimiter(org.apache.samza.table.remote.TableRateLimiter) Mockito.mock(org.mockito.Mockito.mock) RemoteTable(org.apache.samza.table.remote.RemoteTable) AsyncRetriableTable(org.apache.samza.table.retry.AsyncRetriableTable) Table(org.apache.samza.table.Table) AsyncRateLimitedTable(org.apache.samza.table.ratelimit.AsyncRateLimitedTable) AsyncRemoteTable(org.apache.samza.table.remote.AsyncRemoteTable) AsyncReadWriteUpdateTable(org.apache.samza.table.AsyncReadWriteUpdateTable) AsyncRemoteTable(org.apache.samza.table.remote.AsyncRemoteTable) HashMap(java.util.HashMap) TableRetryPolicy(org.apache.samza.table.retry.TableRetryPolicy) RemoteTableDescriptor(org.apache.samza.table.descriptors.RemoteTableDescriptor) Mockito.anyString(org.mockito.Mockito.anyString) RemoteTableProvider(org.apache.samza.table.remote.RemoteTableProvider) EmbeddedTaggedRateLimiter(org.apache.samza.util.EmbeddedTaggedRateLimiter) RateLimiter(org.apache.samza.util.RateLimiter) TableRateLimiter(org.apache.samza.table.remote.TableRateLimiter) EmbeddedTaggedRateLimiter(org.apache.samza.util.EmbeddedTaggedRateLimiter) AsyncRetriableTable(org.apache.samza.table.retry.AsyncRetriableTable) TableRateLimiter(org.apache.samza.table.remote.TableRateLimiter) RemoteTable(org.apache.samza.table.remote.RemoteTable) AsyncRemoteTable(org.apache.samza.table.remote.AsyncRemoteTable) AsyncReadWriteUpdateTable(org.apache.samza.table.AsyncReadWriteUpdateTable) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) AsyncRateLimitedTable(org.apache.samza.table.ratelimit.AsyncRateLimitedTable)

Example 4 with RemoteTableDescriptor

use of org.apache.samza.table.descriptors.RemoteTableDescriptor in project samza by apache.

the class TestRemoteTableDescriptor method testTablePartToConfigDefault.

@Test
public void testTablePartToConfigDefault() {
    TableReadFunction readFn = createMockTableReadFunction();
    when(readFn.toConfig(any(), any())).thenReturn(createConfigPair(1));
    Map<String, String> tableConfig = new RemoteTableDescriptor("1").withReadFunction(readFn).withReadRateLimit(100).toConfig(new MapConfig());
    verify(readFn, times(1)).toConfig(any(), any());
    Assert.assertEquals("v1", tableConfig.get("tables.1.io.read.func.k1"));
}
Also used : TableReadFunction(org.apache.samza.table.remote.TableReadFunction) RemoteTableDescriptor(org.apache.samza.table.descriptors.RemoteTableDescriptor) Mockito.anyString(org.mockito.Mockito.anyString) MapConfig(org.apache.samza.config.MapConfig) Test(org.junit.Test)

Example 5 with RemoteTableDescriptor

use of org.apache.samza.table.descriptors.RemoteTableDescriptor in project samza by apache.

the class TestRemoteTableDescriptor method testTablePartToConfig.

@Test
public void testTablePartToConfig() {
    int key = 0;
    TableReadFunction readFn = createMockTableReadFunction();
    when(readFn.toConfig(any(), any())).thenReturn(createConfigPair(key));
    TableWriteFunction writeFn = createMockTableWriteFunction();
    when(writeFn.toConfig(any(), any())).thenReturn(createConfigPair(key));
    RateLimiter rateLimiter = createMockRateLimiter();
    when(((TablePart) rateLimiter).toConfig(any(), any())).thenReturn(createConfigPair(key));
    CreditFunction readCredFn = createMockCreditFunction();
    when(readCredFn.toConfig(any(), any())).thenReturn(createConfigPair(key));
    CreditFunction writeCredFn = createMockCreditFunction();
    when(writeCredFn.toConfig(any(), any())).thenReturn(createConfigPair(key));
    TableRetryPolicy readRetryPolicy = createMockTableRetryPolicy();
    when(readRetryPolicy.toConfig(any(), any())).thenReturn(createConfigPair(key));
    TableRetryPolicy writeRetryPolicy = createMockTableRetryPolicy();
    when(writeRetryPolicy.toConfig(any(), any())).thenReturn(createConfigPair(key));
    Map<String, String> tableConfig = new RemoteTableDescriptor("1").withReadFunction(readFn).withWriteFunction(writeFn).withRateLimiter(rateLimiter, readCredFn, writeCredFn).withReadRetryPolicy(readRetryPolicy).withWriteRetryPolicy(writeRetryPolicy).toConfig(new MapConfig());
    verify(readFn, times(1)).toConfig(any(), any());
    verify(writeFn, times(1)).toConfig(any(), any());
    verify((TablePart) rateLimiter, times(1)).toConfig(any(), any());
    verify(readCredFn, times(1)).toConfig(any(), any());
    verify(writeCredFn, times(1)).toConfig(any(), any());
    verify(readRetryPolicy, times(1)).toConfig(any(), any());
    verify(writeRetryPolicy, times(1)).toConfig(any(), any());
    Assert.assertEquals(tableConfig.get("tables.1.io.read.func.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.write.func.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.ratelimiter.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.read.credit.func.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.write.credit.func.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.read.retry.policy.k0"), "v0");
    Assert.assertEquals(tableConfig.get("tables.1.io.write.retry.policy.k0"), "v0");
}
Also used : TableReadFunction(org.apache.samza.table.remote.TableReadFunction) TableRetryPolicy(org.apache.samza.table.retry.TableRetryPolicy) TablePart(org.apache.samza.table.remote.TablePart) RemoteTableDescriptor(org.apache.samza.table.descriptors.RemoteTableDescriptor) Mockito.anyString(org.mockito.Mockito.anyString) MapConfig(org.apache.samza.config.MapConfig) TableWriteFunction(org.apache.samza.table.remote.TableWriteFunction) CreditFunction(org.apache.samza.table.remote.TableRateLimiter.CreditFunction) EmbeddedTaggedRateLimiter(org.apache.samza.util.EmbeddedTaggedRateLimiter) RateLimiter(org.apache.samza.util.RateLimiter) TableRateLimiter(org.apache.samza.table.remote.TableRateLimiter) Test(org.junit.Test)

Aggregations

RemoteTableDescriptor (org.apache.samza.table.descriptors.RemoteTableDescriptor)19 Test (org.junit.Test)17 MapConfig (org.apache.samza.config.MapConfig)15 TableReadFunction (org.apache.samza.table.remote.TableReadFunction)10 DelegatingSystemDescriptor (org.apache.samza.system.descriptors.DelegatingSystemDescriptor)9 Table (org.apache.samza.table.Table)9 TableRateLimiter (org.apache.samza.table.remote.TableRateLimiter)9 TableWriteFunction (org.apache.samza.table.remote.TableWriteFunction)9 RateLimiter (org.apache.samza.util.RateLimiter)9 Assert (org.junit.Assert)9 Mockito.anyString (org.mockito.Mockito.anyString)9 Duration (java.time.Duration)8 Arrays (java.util.Arrays)8 HashMap (java.util.HashMap)8 Map (java.util.Map)8 SamzaException (org.apache.samza.SamzaException)8 StreamApplication (org.apache.samza.application.StreamApplication)8 KV (org.apache.samza.operators.KV)8 NoOpSerde (org.apache.samza.serializers.NoOpSerde)8 GenericInputDescriptor (org.apache.samza.system.descriptors.GenericInputDescriptor)8