use of com.hazelcast.jet.pipeline.StreamStage in project hazelcast by hazelcast.
the class PythonServiceTest method streamStage_mapUsingPython_withHandlerFile.
@Test
@Category(NightlyTest.class)
public void streamStage_mapUsingPython_withHandlerFile() {
// Given
PythonServiceConfig cfg = new PythonServiceConfig().setHandlerFile(baseDir + "/echo.py").setHandlerFunction("handle");
List<String> items = IntStream.range(0, ITEM_COUNT).mapToObj(Integer::toString).collect(toList());
Pipeline p = Pipeline.create();
StreamStage<String> stage = p.readFrom(TestSources.items(items)).addTimestamps(x -> 0, 0);
// When
StreamStage<String> mapped = stage.apply(mapUsingPython(cfg)).setLocalParallelism(2);
// Then
mapped.writeTo(AssertionSinks.assertAnyOrder("Python didn't map the items correctly", items.stream().map(i -> "echo-" + i).collect(toList())));
instance().getJet().newJob(p).join();
}
use of com.hazelcast.jet.pipeline.StreamStage in project hazelcast by hazelcast.
the class MultiTableCacheIntegrationTest method ordersOfCustomers.
@Test
public void ordersOfCustomers() throws Exception {
StreamSource<ChangeRecord> source = sourceBuilder("source").setTableWhitelist("inventory.customers", "inventory.orders").build();
Pipeline pipeline = Pipeline.create();
StreamStage<ChangeRecord> allRecords = pipeline.readFrom(source).withNativeTimestamps(0);
allRecords.filter(r -> r.table().equals("customers")).apply(this::fixOrdering).writeTo(Sinks.mapWithEntryProcessor(MAX_CONCURRENT_OPERATIONS, CACHE, record -> (Integer) record.key().toMap().get("id"), CustomerEntryProcessor::new));
allRecords.filter(r -> r.table().equals("orders")).apply(this::fixOrdering).writeTo(Sinks.mapWithEntryProcessor(MAX_CONCURRENT_OPERATIONS, CACHE, record -> (Integer) record.value().toMap().get("purchaser"), OrderEntryProcessor::new));
// when
HazelcastInstance hz = createHazelcastInstances(1)[0];
Job job = hz.getJet().newJob(pipeline);
// then
Map<Integer, OrdersOfCustomer> expected = toMap(new OrdersOfCustomer(new Customer(1001, "Sally", "Thomas", "sally.thomas@acme.com"), new Order(10001, new Date(1452902400000L), 1001, 1, 102)), new OrdersOfCustomer(new Customer(1002, "George", "Bailey", "gbailey@foobar.com"), new Order(10002, new Date(1452988800000L), 1002, 2, 105), new Order(10003, new Date(1455840000000L), 1002, 2, 106)), new OrdersOfCustomer(new Customer(1003, "Edward", "Walker", "ed@walker.com"), new Order(10004, new Date(1456012800000L), 1003, 1, 107)), new OrdersOfCustomer(new Customer(1004, "Anne", "Kretchmar", "annek@noanswer.org")));
assertEqualsEventually(() -> getIMapContent(hz, CACHE), expected);
// when
List<String> batch = new ArrayList<>();
for (int i = 1; i <= REPEATS; i++) {
batch.add("UPDATE customers SET first_name='Anne" + i + "' WHERE id=1004");
batch.add("INSERT INTO customers VALUES (1005, 'Jason', 'Bourne', 'jason@bourne.org')");
batch.add("DELETE FROM customers WHERE id=1005");
batch.add("UPDATE orders SET quantity='" + i + "' WHERE id=10004");
batch.add("DELETE FROM orders WHERE id=10003");
batch.add("INSERT INTO orders VALUES (10003, '2016-02-19', 1002, 2, 106)");
}
executeBatch(batch.toArray(new String[0]));
// then
expected = toMap(new OrdersOfCustomer(new Customer(1001, "Sally", "Thomas", "sally.thomas@acme.com"), new Order(10001, new Date(1452902400000L), 1001, 1, 102)), new OrdersOfCustomer(new Customer(1002, "George", "Bailey", "gbailey@foobar.com"), new Order(10002, new Date(1452988800000L), 1002, 2, 105), new Order(10003, new Date(1455840000000L), 1002, 2, 106)), new OrdersOfCustomer(new Customer(1003, "Edward", "Walker", "ed@walker.com"), new Order(10004, new Date(1456012800000L), 1003, REPEATS, 107)), new OrdersOfCustomer(new Customer(1004, "Anne" + REPEATS, "Kretchmar", "annek@noanswer.org")));
expected.put(1005, new OrdersOfCustomer());
assertEqualsEventually(() -> getIMapContent(hz, CACHE), expected);
}
use of com.hazelcast.jet.pipeline.StreamStage in project hazelcast by hazelcast.
the class PythonServiceTest method streamStage_mapUsingPython_onAllMembers.
@Test
@Category(NightlyTest.class)
public void streamStage_mapUsingPython_onAllMembers() {
// Given
PythonServiceConfig cfg = new PythonServiceConfig().setHandlerFile(baseDir + "/echo.py").setHandlerFunction("handle");
List<String> items = IntStream.range(0, ITEM_COUNT).mapToObj(Integer::toString).collect(toList());
Pipeline p = Pipeline.create();
StreamStage<String> stage = p.readFrom(TestSources.items(items)).addTimestamps(x -> 0, 0);
// When
StreamStage<String> mapped = stage.apply(mapUsingPython(x -> x, cfg)).setLocalParallelism(2);
// Then
mapped.writeTo(AssertionSinks.assertAnyOrder("Python didn't map the items correctly", items.stream().map(i -> "echo-" + i).collect(toList())));
instance().getJet().newJob(p).join();
}
Aggregations