Examples with StreamSource com.hazelcast.jet.pipeline.StreamSource used on opensource projects

Example 6 with StreamSource

use of com.hazelcast.jet.pipeline.StreamSource in project hazelcast by hazelcast.

the class DebeziumCdcIntegrationTest method mysql.

@Test
public void mysql() throws Exception {
    Assume.assumeFalse("https://github.com/hazelcast/hazelcast-jet/issues/2623, " + "https://github.com/hazelcast/hazelcast/issues/18800", System.getProperty("java.version").matches("^1[56].*"));
    MySQLContainer<?> container = mySqlContainer();
    try {
        container.start();
        // given
        List<String> expectedRecords = Arrays.asList("1001/0:INSERT:Customer {id=1001, firstName=Sally, lastName=Thomas, email=sally.thomas@acme.com}", "1002/0:INSERT:Customer {id=1002, firstName=George, lastName=Bailey, email=gbailey@foobar.com}", "1003/0:INSERT:Customer {id=1003, firstName=Edward, lastName=Walker, email=ed@walker.com}", "1004/0:INSERT:Customer {id=1004, firstName=Anne, lastName=Kretchmar, email=annek@noanswer.org}", "1004/1:UPDATE:Customer {id=1004, firstName=Anne Marie, lastName=Kretchmar, email=annek@noanswer.org}", "1005/0:INSERT:Customer {id=1005, firstName=Jason, lastName=Bourne, email=jason@bourne.org}", "1005/1:DELETE:Customer {id=1005, firstName=Jason, lastName=Bourne, email=jason@bourne.org}");
        StreamSource<ChangeRecord> source = mySqlSource(container);
        Pipeline pipeline = Pipeline.create();
        pipeline.readFrom(source).withNativeTimestamps(0).<ChangeRecord>customTransform("filter_timestamps", filterTimestampsProcessorSupplier()).groupingKey(record -> (Integer) record.key().toMap().get("id")).mapStateful(LongAccumulator::new, (accumulator, customerId, record) -> {
            long count = accumulator.get();
            accumulator.add(1);
            Operation operation = record.operation();
            RecordPart value = record.value();
            Customer customer = value.toObject(Customer.class);
            return entry(customerId + "/" + count, operation + ":" + customer);
        }).setLocalParallelism(1).writeTo(Sinks.map("results"));
        // when
        HazelcastInstance hz = createHazelcastInstances(2)[0];
        Job job = hz.getJet().newJob(pipeline);
        // then
        assertEqualsEventually(() -> hz.getMap("results").size(), 4);
        // when
        try (Connection connection = getMySqlConnection(container.withDatabaseName("inventory").getJdbcUrl(), container.getUsername(), container.getPassword())) {
            Statement statement = connection.createStatement();
            statement.addBatch("UPDATE customers SET first_name='Anne Marie' WHERE id=1004");
            statement.addBatch("INSERT INTO customers VALUES (1005, 'Jason', 'Bourne', 'jason@bourne.org')");
            statement.addBatch("DELETE FROM customers WHERE id=1005");
            statement.executeBatch();
        }
        // then
        try {
            assertEqualsEventually(() -> mapResultsToSortedList(hz.getMap("results")), expectedRecords);
        } finally {
            job.cancel();
            assertJobStatusEventually(job, JobStatus.FAILED);
        }
    } finally {
        container.stop();
    }
}

Example 7 with StreamSource

use of com.hazelcast.jet.pipeline.StreamSource in project hazelcast by hazelcast.

the class MySqlCdcListenBeforeExistIntegrationTest method pipeline.

private Pipeline pipeline(StreamSource<ChangeRecord> source) {
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source).withNativeTimestamps(0).<ChangeRecord>customTransform("filter_timestamps", filterTimestampsProcessorSupplier()).setLocalParallelism(1).groupingKey(record -> (Integer) record.key().toMap().get("id")).mapStateful(LongAccumulator::new, (accumulator, rowId, record) -> {
        long count = accumulator.get();
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        TableRow row = value.toObject(TableRow.class);
        return entry(rowId + "/" + count, operation + ":" + row);
    }).setLocalParallelism(1).peek().writeTo(Sinks.map(SINK_MAP_NAME));
    return pipeline;
}

Example 8 with StreamSource

use of com.hazelcast.jet.pipeline.StreamSource in project hazelcast by hazelcast.

the class MySqlCdcWhiteBlackListIntegrationTest method pipeline.

private Pipeline pipeline(StreamSource<ChangeRecord> source) {
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source).withNativeTimestamps(0).filter(t -> t.database().startsWith(DB_PREFIX)).setLocalParallelism(1).<ChangeRecord>customTransform("filter_timestamps", filterTimestampsProcessorSupplier()).setLocalParallelism(1).groupingKey(record -> (Integer) record.key().toMap().get("id")).mapStateful(LongAccumulator::new, (accumulator, rowId, record) -> {
        long count = accumulator.get();
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        TableRow row = value.toObject(TableRow.class);
        return entry(rowId + "/" + count, operation + ":" + row);
    }).setLocalParallelism(1).writeTo(Sinks.map(SINK_MAP_NAME));
    return pipeline;
}

Example 9 with StreamSource

use of com.hazelcast.jet.pipeline.StreamSource in project hazelcast by hazelcast.

the class AbstractKinesisTest method getPipeline.

protected Pipeline getPipeline(StreamSource<Map.Entry<String, byte[]>> source) {
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source).withNativeTimestamps(0).rebalance(Map.Entry::getKey).map(e -> entry(e.getKey(), Collections.singletonList(new String(e.getValue())))).writeTo(Sinks.mapWithMerging(results, Map.Entry::getKey, Map.Entry::getValue, (l1, l2) -> {
        ArrayList<String> list = new ArrayList<>();
        list.addAll(l1);
        list.addAll(l2);
        return list;
    }));
    return pipeline;
}

Example 10 with StreamSource

use of com.hazelcast.jet.pipeline.StreamSource in project hazelcast by hazelcast.

the class KinesisIntegrationTest method customProjection.

@Test
@Category(SerialTest.class)
public void customProjection() {
    HELPER.createStream(1);
    sendMessages();
    Long expectedPerSequenceNo = 1L;
    try {
        Pipeline pipeline = Pipeline.create();
        StreamSource<String> source = kinesisSource().withProjectionFn((r, s) -> {
            byte[] payload = new byte[r.getData().remaining()];
            r.getData().get(payload);
            return r.getSequenceNumber();
        }).build();
        pipeline.readFrom(source).withoutTimestamps().groupingKey(r -> r).rollingAggregate(counting()).apply(assertCollectedEventually(ASSERT_TRUE_EVENTUALLY_TIMEOUT, results -> {
            assertEquals(MESSAGES, results.size());
            results.forEach(v -> assertEquals(expectedPerSequenceNo, v.getValue()));
        }));
        hz().getJet().newJob(pipeline).join();
        fail("Expected exception not thrown");
    } catch (CompletionException ce) {
        Throwable cause = peel(ce);
        assertTrue(cause instanceof JetException);
        assertTrue(cause.getCause() instanceof AssertionCompletedException);
    }
}