Search in sources :

Example 1 with InMemorySystemDescriptor

use of org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor in project samza by apache.

the class TestCouchbaseRemoteTableEndToEnd method testEndToEnd.

@Test
public void testEndToEnd() {
    Bucket inputBucket = cluster.openBucket(inputBucketName);
    inputBucket.upsert(ByteArrayDocument.create("Alice", "20".getBytes()));
    inputBucket.upsert(ByteArrayDocument.create("Bob", "30".getBytes()));
    inputBucket.upsert(ByteArrayDocument.create("Chris", "40".getBytes()));
    inputBucket.upsert(ByteArrayDocument.create("David", "50".getBytes()));
    inputBucket.close();
    List<String> users = Arrays.asList("Alice", "Bob", "Chris", "David");
    final StreamApplication app = appDesc -> {
        DelegatingSystemDescriptor inputSystemDescriptor = new DelegatingSystemDescriptor("test");
        GenericInputDescriptor<String> inputDescriptor = inputSystemDescriptor.getInputDescriptor("User", new NoOpSerde<>());
        CouchbaseTableReadFunction<String> readFunction = new CouchbaseTableReadFunction<>(inputBucketName, String.class, "couchbase://127.0.0.1").withBootstrapCarrierDirectPort(couchbaseMock.getCarrierPort(inputBucketName)).withBootstrapHttpDirectPort(couchbaseMock.getHttpPort()).withSerde(new StringSerde());
        CouchbaseTableWriteFunction<JsonObject> writeFunction = new CouchbaseTableWriteFunction<>(outputBucketName, JsonObject.class, "couchbase://127.0.0.1").withBootstrapCarrierDirectPort(couchbaseMock.getCarrierPort(outputBucketName)).withBootstrapHttpDirectPort(couchbaseMock.getHttpPort());
        RemoteTableDescriptor inputTableDesc = new RemoteTableDescriptor<String, String, Void>("input-table").withReadFunction(readFunction).withRateLimiterDisabled();
        Table<KV<String, String>> inputTable = appDesc.getTable(inputTableDesc);
        RemoteTableDescriptor outputTableDesc = new RemoteTableDescriptor<String, JsonObject, Object>("output-table").withReadFunction(new NoOpTableReadFunction<>()).withWriteFunction(writeFunction).withRateLimiterDisabled();
        Table<KV<String, JsonObject>> outputTable = appDesc.getTable(outputTableDesc);
        appDesc.getInputStream(inputDescriptor).map(k -> KV.of(k, k)).join(inputTable, new JoinFunction()).sendTo(outputTable);
    };
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<TestTableData.PageView> inputDescriptor = isd.getInputDescriptor("User", new NoOpSerde<>());
    TestRunner.of(app).addInputStream(inputDescriptor, users).run(Duration.ofSeconds(10));
    Bucket outputBucket = cluster.openBucket(outputBucketName);
    Assert.assertEquals("{\"name\":\"Alice\",\"age\":\"20\"}", outputBucket.get("Alice").content().toString());
    Assert.assertEquals("{\"name\":\"Bob\",\"age\":\"30\"}", outputBucket.get("Bob").content().toString());
    Assert.assertEquals("{\"name\":\"Chris\",\"age\":\"40\"}", outputBucket.get("Chris").content().toString());
    Assert.assertEquals("{\"name\":\"David\",\"age\":\"50\"}", outputBucket.get("David").content().toString());
    outputBucket.close();
}
Also used : CouchbaseTableWriteFunction(org.apache.samza.table.remote.couchbase.CouchbaseTableWriteFunction) Arrays(java.util.Arrays) InMemorySystemDescriptor(org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor) CouchbaseEnvironment(com.couchbase.client.java.env.CouchbaseEnvironment) CouchbaseTableReadFunction(org.apache.samza.table.remote.couchbase.CouchbaseTableReadFunction) GenericInputDescriptor(org.apache.samza.system.descriptors.GenericInputDescriptor) JsonObject(com.couchbase.client.java.document.json.JsonObject) BucketConfiguration(com.couchbase.mock.BucketConfiguration) ArrayList(java.util.ArrayList) StringSerde(org.apache.samza.serializers.StringSerde) DefaultCouchbaseEnvironment(com.couchbase.client.java.env.DefaultCouchbaseEnvironment) InMemoryInputDescriptor(org.apache.samza.test.framework.system.descriptors.InMemoryInputDescriptor) CouchbaseMock(com.couchbase.mock.CouchbaseMock) DelegatingSystemDescriptor(org.apache.samza.system.descriptors.DelegatingSystemDescriptor) Duration(java.time.Duration) After(org.junit.After) NoOpTableReadFunction(org.apache.samza.table.remote.NoOpTableReadFunction) KV(org.apache.samza.operators.KV) NoOpSerde(org.apache.samza.serializers.NoOpSerde) Before(org.junit.Before) Table(org.apache.samza.table.Table) ByteArrayDocument(com.couchbase.client.java.document.ByteArrayDocument) StreamTableJoinFunction(org.apache.samza.operators.functions.StreamTableJoinFunction) Test(org.junit.Test) TestRunner(org.apache.samza.test.framework.TestRunner) Bucket(com.couchbase.client.java.Bucket) List(java.util.List) CouchbaseCluster(com.couchbase.client.java.CouchbaseCluster) Cluster(com.couchbase.client.java.Cluster) StreamApplication(org.apache.samza.application.StreamApplication) Assert(org.junit.Assert) RemoteTableDescriptor(org.apache.samza.table.descriptors.RemoteTableDescriptor) StringSerde(org.apache.samza.serializers.StringSerde) Table(org.apache.samza.table.Table) StreamApplication(org.apache.samza.application.StreamApplication) JsonObject(com.couchbase.client.java.document.json.JsonObject) RemoteTableDescriptor(org.apache.samza.table.descriptors.RemoteTableDescriptor) InMemorySystemDescriptor(org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor) DelegatingSystemDescriptor(org.apache.samza.system.descriptors.DelegatingSystemDescriptor) NoOpTableReadFunction(org.apache.samza.table.remote.NoOpTableReadFunction) GenericInputDescriptor(org.apache.samza.system.descriptors.GenericInputDescriptor) Bucket(com.couchbase.client.java.Bucket) CouchbaseTableWriteFunction(org.apache.samza.table.remote.couchbase.CouchbaseTableWriteFunction) StreamTableJoinFunction(org.apache.samza.operators.functions.StreamTableJoinFunction) NoOpSerde(org.apache.samza.serializers.NoOpSerde) CouchbaseTableReadFunction(org.apache.samza.table.remote.couchbase.CouchbaseTableReadFunction) Test(org.junit.Test)

Example 2 with InMemorySystemDescriptor

use of org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor in project samza by apache.

the class TestLocalTableWithSideInputsEndToEnd method runTest.

private <T extends ApplicationDescriptor<?>> void runTest(SamzaApplication<T> app, Map<Integer, List<PageView>> pageViews, Map<Integer, List<Profile>> profiles) throws InterruptedException {
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor(SYSTEM_NAME);
    InMemoryInputDescriptor<PageView> pageViewStreamDesc = isd.getInputDescriptor(PAGEVIEW_STREAM, new NoOpSerde<>());
    InMemoryInputDescriptor<Profile> profileStreamDesc = isd.getInputDescriptor(PROFILE_STREAM, new NoOpSerde<>());
    InMemoryOutputDescriptor<EnrichedPageView> outputStreamDesc = isd.getOutputDescriptor(ENRICHED_PAGEVIEW_STREAM, new NoOpSerde<>());
    TestRunner.of(app).addInputStream(pageViewStreamDesc, pageViews).addInputStream(profileStreamDesc, profiles).addOutputStream(outputStreamDesc, 1).run(Duration.ofSeconds(10));
    List<EnrichedPageView> expectedEnrichedPageViews = buildExpectedEnrichedPageViews(pageViews, profiles);
    StreamAssert.containsInAnyOrder(expectedEnrichedPageViews, outputStreamDesc, Duration.ofSeconds(1));
}
Also used : EnrichedPageView(org.apache.samza.test.table.TestTableData.EnrichedPageView) PageView(org.apache.samza.test.table.TestTableData.PageView) EnrichedPageView(org.apache.samza.test.table.TestTableData.EnrichedPageView) InMemorySystemDescriptor(org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor) Profile(org.apache.samza.test.table.TestTableData.Profile)

Example 3 with InMemorySystemDescriptor

use of org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor in project samza by apache.

the class TestRemoteTableWithBatchEndToEnd method doTestStreamTableJoinRemoteTable.

private void doTestStreamTableJoinRemoteTable(String testName, boolean batchRead, boolean batchWrite) throws Exception {
    final InMemoryWriteFunction writer = new InMemoryWriteFunction(testName);
    BATCH_READS.put(testName, new AtomicInteger());
    BATCH_WRITES.put(testName, new AtomicInteger());
    WRITTEN_RECORDS.put(testName, new HashMap<>());
    int count = 16;
    int batchSize = 4;
    String profiles = Base64Serializer.serialize(generateProfiles(count));
    final RateLimiter readRateLimiter = mock(RateLimiter.class, withSettings().serializable());
    final RateLimiter writeRateLimiter = mock(RateLimiter.class, withSettings().serializable());
    final TableRateLimiter.CreditFunction creditFunction = (k, v, args) -> 1;
    final StreamApplication app = appDesc -> {
        RemoteTableDescriptor<Integer, Profile, Void> inputTableDesc = new RemoteTableDescriptor<>("profile-table-1");
        inputTableDesc.withReadFunction(InMemoryReadFunction.getInMemoryReadFunction(testName, profiles)).withRateLimiter(readRateLimiter, creditFunction, null);
        if (batchRead) {
            inputTableDesc.withBatchProvider(new CompactBatchProvider().withMaxBatchSize(batchSize).withMaxBatchDelay(Duration.ofHours(1)));
        }
        // dummy reader
        TableReadFunction readFn = new MyReadFunction();
        RemoteTableDescriptor<Integer, EnrichedPageView, EnrichedPageView> outputTableDesc = new RemoteTableDescriptor<>("enriched-page-view-table-1");
        outputTableDesc.withReadFunction(readFn).withWriteFunction(writer).withRateLimiter(writeRateLimiter, creditFunction, creditFunction);
        if (batchWrite) {
            outputTableDesc.withBatchProvider(new CompactBatchProvider().withMaxBatchSize(batchSize).withMaxBatchDelay(Duration.ofHours(1)));
        }
        Table outputTable = appDesc.getTable(outputTableDesc);
        Table<KV<Integer, Profile>> inputTable = appDesc.getTable(inputTableDesc);
        DelegatingSystemDescriptor ksd = new DelegatingSystemDescriptor("test");
        GenericInputDescriptor<PageView> isd = ksd.getInputDescriptor("PageView", new NoOpSerde<>());
        appDesc.getInputStream(isd).map(pv -> new KV<>(pv.getMemberId(), pv)).join(inputTable, new PageViewToProfileJoinFunction()).map(m -> new KV<>(m.getMemberId(), m)).sendTo(outputTable);
    };
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<PageView> inputDescriptor = isd.getInputDescriptor("PageView", new NoOpSerde<>());
    TestRunner.of(app).addInputStream(inputDescriptor, Arrays.asList(generatePageViewsWithDistinctKeys(count))).addConfig("task.max.concurrency", String.valueOf(count)).addConfig("task.async.commit", String.valueOf(true)).run(Duration.ofSeconds(10));
    Assert.assertEquals(count, WRITTEN_RECORDS.get(testName).size());
    Assert.assertNotNull(WRITTEN_RECORDS.get(testName).get(0));
    if (batchWrite) {
        Assert.assertEquals(count / batchSize, BATCH_WRITES.get(testName).get());
    }
}
Also used : Arrays(java.util.Arrays) UpdateMessage(org.apache.samza.operators.UpdateMessage) InMemorySystemDescriptor(org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor) ObjectInputStream(java.io.ObjectInputStream) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) GenericInputDescriptor(org.apache.samza.system.descriptors.GenericInputDescriptor) Function(java.util.function.Function) TableReadFunction(org.apache.samza.table.remote.TableReadFunction) Base64Serializer(org.apache.samza.test.util.Base64Serializer) InMemoryInputDescriptor(org.apache.samza.test.framework.system.descriptors.InMemoryInputDescriptor) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) TestTableData(org.apache.samza.test.table.TestTableData) DelegatingSystemDescriptor(org.apache.samza.system.descriptors.DelegatingSystemDescriptor) Duration(java.time.Duration) Map(java.util.Map) TableWriteFunction(org.apache.samza.table.remote.TableWriteFunction) KV(org.apache.samza.operators.KV) NoOpSerde(org.apache.samza.serializers.NoOpSerde) Table(org.apache.samza.table.Table) Collection(java.util.Collection) BaseTableFunction(org.apache.samza.table.remote.BaseTableFunction) IOException(java.io.IOException) Test(org.junit.Test) Collectors(java.util.stream.Collectors) SamzaException(org.apache.samza.SamzaException) TestRunner(org.apache.samza.test.framework.TestRunner) Mockito(org.mockito.Mockito) Entry(org.apache.samza.storage.kv.Entry) RateLimiter(org.apache.samza.util.RateLimiter) UpdateOptions(org.apache.samza.operators.UpdateOptions) CompactBatchProvider(org.apache.samza.table.batching.CompactBatchProvider) StreamApplication(org.apache.samza.application.StreamApplication) Assert(org.junit.Assert) CompleteBatchProvider(org.apache.samza.table.batching.CompleteBatchProvider) RemoteTableDescriptor(org.apache.samza.table.descriptors.RemoteTableDescriptor) TableRateLimiter(org.apache.samza.table.remote.TableRateLimiter) TableReadFunction(org.apache.samza.table.remote.TableReadFunction) InMemorySystemDescriptor(org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor) GenericInputDescriptor(org.apache.samza.system.descriptors.GenericInputDescriptor) Table(org.apache.samza.table.Table) StreamApplication(org.apache.samza.application.StreamApplication) RemoteTableDescriptor(org.apache.samza.table.descriptors.RemoteTableDescriptor) RateLimiter(org.apache.samza.util.RateLimiter) TableRateLimiter(org.apache.samza.table.remote.TableRateLimiter) TableRateLimiter(org.apache.samza.table.remote.TableRateLimiter) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DelegatingSystemDescriptor(org.apache.samza.system.descriptors.DelegatingSystemDescriptor) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) NoOpSerde(org.apache.samza.serializers.NoOpSerde) CompactBatchProvider(org.apache.samza.table.batching.CompactBatchProvider)

Example 4 with InMemorySystemDescriptor

use of org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor in project samza by apache.

the class StreamApplicationIntegrationTest method testSamzaJobFailureForStreamApplication.

/**
 * Null page key is passed in input data which should fail filter logic
 */
@Test(expected = SamzaException.class)
public void testSamzaJobFailureForStreamApplication() {
    int count = 10;
    List<TestData.PageView> pageviews = new ArrayList<>();
    for (int memberId = 0; memberId < count; memberId++) {
        pageviews.add(new TestData.PageView(null, memberId));
    }
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<PageView> imid = isd.getInputDescriptor("PageView", new NoOpSerde<PageView>());
    InMemoryOutputDescriptor<PageView> imod = isd.getOutputDescriptor("Output", new NoOpSerde<PageView>());
    TestRunner.of(new PageViewFilterApplication()).addInputStream(imid, pageviews).addOutputStream(imod, 10).run(Duration.ofMillis(1000));
}
Also used : PageView(org.apache.samza.test.controlmessages.TestData.PageView) TestData(org.apache.samza.test.controlmessages.TestData) ArrayList(java.util.ArrayList) PageView(org.apache.samza.test.controlmessages.TestData.PageView) InMemorySystemDescriptor(org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor) Test(org.junit.Test)

Example 5 with InMemorySystemDescriptor

use of org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor in project samza by apache.

the class StreamApplicationIntegrationTest method testHighLevelApi.

@Test
public void testHighLevelApi() throws Exception {
    Random random = new Random();
    int count = 10;
    List<PageView> pageViews = new ArrayList<>();
    for (int memberId = 0; memberId < count; memberId++) {
        String pagekey = PAGEKEYS[random.nextInt(PAGEKEYS.length - 1)];
        PageView pv = new PageView(pagekey, memberId);
        pageViews.add(pv);
    }
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<PageView> imid = isd.getInputDescriptor("PageView", new NoOpSerde<PageView>());
    InMemoryOutputDescriptor<PageView> imod = isd.getOutputDescriptor("Output", new NoOpSerde<PageView>());
    TestRunner.of(new PageViewRepartitionApplication()).addInputStream(imid, pageViews).addOutputStream(imod, 10).run(Duration.ofMillis(1500));
    Assert.assertEquals(TestRunner.consumeStream(imod, Duration.ofMillis(1000)).get(random.nextInt(count)).size(), 1);
}
Also used : PageView(org.apache.samza.test.controlmessages.TestData.PageView) Random(java.util.Random) ArrayList(java.util.ArrayList) InMemorySystemDescriptor(org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor) Test(org.junit.Test)

Aggregations

InMemorySystemDescriptor (org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor)34 Test (org.junit.Test)29 KV (org.apache.samza.operators.KV)17 ArrayList (java.util.ArrayList)16 List (java.util.List)16 NoOpSerde (org.apache.samza.serializers.NoOpSerde)13 HashMap (java.util.HashMap)12 StreamApplication (org.apache.samza.application.StreamApplication)12 Duration (java.time.Duration)11 InMemoryInputDescriptor (org.apache.samza.test.framework.system.descriptors.InMemoryInputDescriptor)11 PageView (org.apache.samza.test.table.TestTableData.PageView)11 DelegatingSystemDescriptor (org.apache.samza.system.descriptors.DelegatingSystemDescriptor)10 GenericInputDescriptor (org.apache.samza.system.descriptors.GenericInputDescriptor)10 Table (org.apache.samza.table.Table)10 TestRunner (org.apache.samza.test.framework.TestRunner)10 EnrichedPageView (org.apache.samza.test.table.TestTableData.EnrichedPageView)10 Profile (org.apache.samza.test.table.TestTableData.Profile)10 Arrays (java.util.Arrays)9 StreamApplicationDescriptor (org.apache.samza.application.descriptors.StreamApplicationDescriptor)9 Assert (org.junit.Assert)9