Search in sources :

Example 6 with StreamStage

use of com.hazelcast.jet.pipeline.StreamStage in project hazelcast by hazelcast.

the class PythonServiceTest method streamStage_mapUsingPython_withHandlerFile.

@Test
@Category(NightlyTest.class)
public void streamStage_mapUsingPython_withHandlerFile() {
    // Given
    PythonServiceConfig cfg = new PythonServiceConfig().setHandlerFile(baseDir + "/echo.py").setHandlerFunction("handle");
    List<String> items = IntStream.range(0, ITEM_COUNT).mapToObj(Integer::toString).collect(toList());
    Pipeline p = Pipeline.create();
    StreamStage<String> stage = p.readFrom(TestSources.items(items)).addTimestamps(x -> 0, 0);
    // When
    StreamStage<String> mapped = stage.apply(mapUsingPython(cfg)).setLocalParallelism(2);
    // Then
    mapped.writeTo(AssertionSinks.assertAnyOrder("Python didn't map the items correctly", items.stream().map(i -> "echo-" + i).collect(toList())));
    instance().getJet().newJob(p).join();
}
Also used : AssertionSinks(com.hazelcast.jet.pipeline.test.AssertionSinks) IntStream(java.util.stream.IntStream) ParallelJVMTest(com.hazelcast.test.annotation.ParallelJVMTest) BeforeClass(org.junit.BeforeClass) QuickTest(com.hazelcast.test.annotation.QuickTest) PythonTransforms.mapUsingPython(com.hazelcast.jet.python.PythonTransforms.mapUsingPython) Collections.singletonList(java.util.Collections.singletonList) BatchStage(com.hazelcast.jet.pipeline.BatchStage) ByteArrayInputStream(java.io.ByteArrayInputStream) After(org.junit.After) Assert.fail(org.junit.Assert.fail) PythonTransforms.mapUsingPythonBatch(com.hazelcast.jet.python.PythonTransforms.mapUsingPythonBatch) Before(org.junit.Before) SimpleTestInClusterSupport(com.hazelcast.jet.SimpleTestInClusterSupport) Config(com.hazelcast.config.Config) StreamStage(com.hazelcast.jet.pipeline.StreamStage) NightlyTest(com.hazelcast.test.annotation.NightlyTest) Pipeline(com.hazelcast.jet.pipeline.Pipeline) Files(java.nio.file.Files) UTF_8(java.nio.charset.StandardCharsets.UTF_8) Sinks(com.hazelcast.jet.pipeline.Sinks) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) CompletionException(java.util.concurrent.CompletionException) Category(org.junit.experimental.categories.Category) File(java.io.File) TestSources(com.hazelcast.jet.pipeline.test.TestSources) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) InputStream(java.io.InputStream) Pipeline(com.hazelcast.jet.pipeline.Pipeline) Category(org.junit.experimental.categories.Category) ParallelJVMTest(com.hazelcast.test.annotation.ParallelJVMTest) QuickTest(com.hazelcast.test.annotation.QuickTest) NightlyTest(com.hazelcast.test.annotation.NightlyTest) Test(org.junit.Test)

Example 7 with StreamStage

use of com.hazelcast.jet.pipeline.StreamStage in project hazelcast by hazelcast.

the class MultiTableCacheIntegrationTest method ordersOfCustomers.

@Test
public void ordersOfCustomers() throws Exception {
    StreamSource<ChangeRecord> source = sourceBuilder("source").setTableWhitelist("inventory.customers", "inventory.orders").build();
    Pipeline pipeline = Pipeline.create();
    StreamStage<ChangeRecord> allRecords = pipeline.readFrom(source).withNativeTimestamps(0);
    allRecords.filter(r -> r.table().equals("customers")).apply(this::fixOrdering).writeTo(Sinks.mapWithEntryProcessor(MAX_CONCURRENT_OPERATIONS, CACHE, record -> (Integer) record.key().toMap().get("id"), CustomerEntryProcessor::new));
    allRecords.filter(r -> r.table().equals("orders")).apply(this::fixOrdering).writeTo(Sinks.mapWithEntryProcessor(MAX_CONCURRENT_OPERATIONS, CACHE, record -> (Integer) record.value().toMap().get("purchaser"), OrderEntryProcessor::new));
    // when
    HazelcastInstance hz = createHazelcastInstances(1)[0];
    Job job = hz.getJet().newJob(pipeline);
    // then
    Map<Integer, OrdersOfCustomer> expected = toMap(new OrdersOfCustomer(new Customer(1001, "Sally", "Thomas", "sally.thomas@acme.com"), new Order(10001, new Date(1452902400000L), 1001, 1, 102)), new OrdersOfCustomer(new Customer(1002, "George", "Bailey", "gbailey@foobar.com"), new Order(10002, new Date(1452988800000L), 1002, 2, 105), new Order(10003, new Date(1455840000000L), 1002, 2, 106)), new OrdersOfCustomer(new Customer(1003, "Edward", "Walker", "ed@walker.com"), new Order(10004, new Date(1456012800000L), 1003, 1, 107)), new OrdersOfCustomer(new Customer(1004, "Anne", "Kretchmar", "annek@noanswer.org")));
    assertEqualsEventually(() -> getIMapContent(hz, CACHE), expected);
    // when
    List<String> batch = new ArrayList<>();
    for (int i = 1; i <= REPEATS; i++) {
        batch.add("UPDATE customers SET first_name='Anne" + i + "' WHERE id=1004");
        batch.add("INSERT INTO customers VALUES (1005, 'Jason', 'Bourne', 'jason@bourne.org')");
        batch.add("DELETE FROM customers WHERE id=1005");
        batch.add("UPDATE orders SET quantity='" + i + "' WHERE id=10004");
        batch.add("DELETE FROM orders WHERE id=10003");
        batch.add("INSERT INTO orders VALUES (10003, '2016-02-19', 1002, 2, 106)");
    }
    executeBatch(batch.toArray(new String[0]));
    // then
    expected = toMap(new OrdersOfCustomer(new Customer(1001, "Sally", "Thomas", "sally.thomas@acme.com"), new Order(10001, new Date(1452902400000L), 1001, 1, 102)), new OrdersOfCustomer(new Customer(1002, "George", "Bailey", "gbailey@foobar.com"), new Order(10002, new Date(1452988800000L), 1002, 2, 105), new Order(10003, new Date(1455840000000L), 1002, 2, 106)), new OrdersOfCustomer(new Customer(1003, "Edward", "Walker", "ed@walker.com"), new Order(10004, new Date(1456012800000L), 1003, REPEATS, 107)), new OrdersOfCustomer(new Customer(1004, "Anne" + REPEATS, "Kretchmar", "annek@noanswer.org")));
    expected.put(1005, new OrdersOfCustomer());
    assertEqualsEventually(() -> getIMapContent(hz, CACHE), expected);
}
Also used : Arrays(java.util.Arrays) Date(java.util.Date) HashMap(java.util.HashMap) StreamSource(com.hazelcast.jet.pipeline.StreamSource) Function(java.util.function.Function) ParsingException(com.hazelcast.jet.cdc.ParsingException) ArrayList(java.util.ArrayList) Operation(com.hazelcast.jet.cdc.Operation) ExceptionUtil.rethrow(com.hazelcast.jet.impl.util.ExceptionUtil.rethrow) Map(java.util.Map) ChangeRecord(com.hazelcast.jet.cdc.ChangeRecord) Nonnull(javax.annotation.Nonnull) Job(com.hazelcast.jet.Job) HazelcastInstance(com.hazelcast.core.HazelcastInstance) StreamStage(com.hazelcast.jet.pipeline.StreamStage) NightlyTest(com.hazelcast.test.annotation.NightlyTest) Pipeline(com.hazelcast.jet.pipeline.Pipeline) Sinks(com.hazelcast.jet.pipeline.Sinks) RecordPart(com.hazelcast.jet.cdc.RecordPart) Test(org.junit.Test) Category(org.junit.experimental.categories.Category) Collectors(java.util.stream.Collectors) Serializable(java.io.Serializable) Objects(java.util.Objects) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) EntryProcessor(com.hazelcast.map.EntryProcessor) Entry(java.util.Map.Entry) TriFunction(com.hazelcast.jet.function.TriFunction) ArrayList(java.util.ArrayList) Date(java.util.Date) Pipeline(com.hazelcast.jet.pipeline.Pipeline) HazelcastInstance(com.hazelcast.core.HazelcastInstance) Job(com.hazelcast.jet.Job) ChangeRecord(com.hazelcast.jet.cdc.ChangeRecord) NightlyTest(com.hazelcast.test.annotation.NightlyTest) Test(org.junit.Test)

Example 8 with StreamStage

use of com.hazelcast.jet.pipeline.StreamStage in project hazelcast by hazelcast.

the class PythonServiceTest method streamStage_mapUsingPython_onAllMembers.

@Test
@Category(NightlyTest.class)
public void streamStage_mapUsingPython_onAllMembers() {
    // Given
    PythonServiceConfig cfg = new PythonServiceConfig().setHandlerFile(baseDir + "/echo.py").setHandlerFunction("handle");
    List<String> items = IntStream.range(0, ITEM_COUNT).mapToObj(Integer::toString).collect(toList());
    Pipeline p = Pipeline.create();
    StreamStage<String> stage = p.readFrom(TestSources.items(items)).addTimestamps(x -> 0, 0);
    // When
    StreamStage<String> mapped = stage.apply(mapUsingPython(x -> x, cfg)).setLocalParallelism(2);
    // Then
    mapped.writeTo(AssertionSinks.assertAnyOrder("Python didn't map the items correctly", items.stream().map(i -> "echo-" + i).collect(toList())));
    instance().getJet().newJob(p).join();
}
Also used : AssertionSinks(com.hazelcast.jet.pipeline.test.AssertionSinks) IntStream(java.util.stream.IntStream) ParallelJVMTest(com.hazelcast.test.annotation.ParallelJVMTest) BeforeClass(org.junit.BeforeClass) QuickTest(com.hazelcast.test.annotation.QuickTest) PythonTransforms.mapUsingPython(com.hazelcast.jet.python.PythonTransforms.mapUsingPython) Collections.singletonList(java.util.Collections.singletonList) BatchStage(com.hazelcast.jet.pipeline.BatchStage) ByteArrayInputStream(java.io.ByteArrayInputStream) After(org.junit.After) Assert.fail(org.junit.Assert.fail) PythonTransforms.mapUsingPythonBatch(com.hazelcast.jet.python.PythonTransforms.mapUsingPythonBatch) Before(org.junit.Before) SimpleTestInClusterSupport(com.hazelcast.jet.SimpleTestInClusterSupport) Config(com.hazelcast.config.Config) StreamStage(com.hazelcast.jet.pipeline.StreamStage) NightlyTest(com.hazelcast.test.annotation.NightlyTest) Pipeline(com.hazelcast.jet.pipeline.Pipeline) Files(java.nio.file.Files) UTF_8(java.nio.charset.StandardCharsets.UTF_8) Sinks(com.hazelcast.jet.pipeline.Sinks) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) CompletionException(java.util.concurrent.CompletionException) Category(org.junit.experimental.categories.Category) File(java.io.File) TestSources(com.hazelcast.jet.pipeline.test.TestSources) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) InputStream(java.io.InputStream) Pipeline(com.hazelcast.jet.pipeline.Pipeline) Category(org.junit.experimental.categories.Category) ParallelJVMTest(com.hazelcast.test.annotation.ParallelJVMTest) QuickTest(com.hazelcast.test.annotation.QuickTest) NightlyTest(com.hazelcast.test.annotation.NightlyTest) Test(org.junit.Test)

Aggregations

StreamStage (com.hazelcast.jet.pipeline.StreamStage)8 Pipeline (com.hazelcast.jet.pipeline.Pipeline)7 Sinks (com.hazelcast.jet.pipeline.Sinks)7 List (java.util.List)7 Test (org.junit.Test)7 Category (org.junit.experimental.categories.Category)7 BatchStage (com.hazelcast.jet.pipeline.BatchStage)6 NightlyTest (com.hazelcast.test.annotation.NightlyTest)6 Config (com.hazelcast.config.Config)5 SimpleTestInClusterSupport (com.hazelcast.jet.SimpleTestInClusterSupport)5 AssertionSinks (com.hazelcast.jet.pipeline.test.AssertionSinks)5 TestSources (com.hazelcast.jet.pipeline.test.TestSources)5 PythonTransforms.mapUsingPython (com.hazelcast.jet.python.PythonTransforms.mapUsingPython)5 PythonTransforms.mapUsingPythonBatch (com.hazelcast.jet.python.PythonTransforms.mapUsingPythonBatch)5 ParallelJVMTest (com.hazelcast.test.annotation.ParallelJVMTest)5 QuickTest (com.hazelcast.test.annotation.QuickTest)5 ByteArrayInputStream (java.io.ByteArrayInputStream)5 File (java.io.File)5 IOException (java.io.IOException)5 InputStream (java.io.InputStream)5