Examples with Pipeline com.hazelcast.jet.pipeline.Pipeline used on opensource projects

Example 91 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class PostgresCdcIntegrationTest method customersPipeline.

@Nonnull
private Pipeline customersPipeline(Long commitPeriod) {
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source("customers", commitPeriod)).withNativeTimestamps(0).<ChangeRecord>customTransform("filter_timestamps", filterTimestampsProcessorSupplier()).groupingKey(record -> (Integer) record.key().toMap().get("id")).mapStateful(LongAccumulator::new, (accumulator, customerId, record) -> {
        String count = format("%05d", accumulator.get());
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        Customer customer = value.toObject(Customer.class);
        return entry(customerId + "/" + count, operation + ":" + customer);
    }).setLocalParallelism(1).writeTo(Sinks.map("results"));
    return pipeline;
}

Example 92 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class PostgresCdcIntegrationTest method cdcMapSink.

@Test
@Category(NightlyTest.class)
public void cdcMapSink() throws Exception {
    // given
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source("customers")).withNativeTimestamps(0).writeTo(CdcSinks.map("cache", r -> r.key().toMap().get("id"), r -> r.value().toObject(Customer.class).toString()));
    // when
    HazelcastInstance hz = createHazelcastInstances(2)[0];
    JobConfig jobConfig = new JobConfig().setProcessingGuarantee(ProcessingGuarantee.AT_LEAST_ONCE);
    Job job = hz.getJet().newJob(pipeline, jobConfig);
    JetTestSupport.assertJobStatusEventually(job, JobStatus.RUNNING);
    // then
    assertEqualsEventually(() -> mapResultsToSortedList(hz.getMap("cache")), Arrays.asList("1001:Customer {id=1001, firstName=Sally, lastName=Thomas, email=sally.thomas@acme.com}", "1002:Customer {id=1002, firstName=George, lastName=Bailey, email=gbailey@foobar.com}", "1003:Customer {id=1003, firstName=Edward, lastName=Walker, email=ed@walker.com}", "1004:Customer {id=1004, firstName=Anne, lastName=Kretchmar, email=annek@noanswer.org}"));
    // when
    job.restart();
    JetTestSupport.assertJobStatusEventually(job, JobStatus.RUNNING);
    executeBatch("UPDATE customers SET first_name='Anne Marie' WHERE id=1004", "INSERT INTO customers VALUES (1005, 'Jason', 'Bourne', 'jason@bourne.org')");
    // then
    assertEqualsEventually(() -> mapResultsToSortedList(hz.getMap("cache")), Arrays.asList("1001:Customer {id=1001, firstName=Sally, lastName=Thomas, email=sally.thomas@acme.com}", "1002:Customer {id=1002, firstName=George, lastName=Bailey, email=gbailey@foobar.com}", "1003:Customer {id=1003, firstName=Edward, lastName=Walker, email=ed@walker.com}", "1004:Customer {id=1004, firstName=Anne Marie, lastName=Kretchmar, email=annek@noanswer.org}", "1005:Customer {id=1005, firstName=Jason, lastName=Bourne, email=jason@bourne.org}"));
    // when
    executeBatch("DELETE FROM customers WHERE id=1005");
    // then
    assertEqualsEventually(() -> mapResultsToSortedList(hz.getMap("cache")), Arrays.asList("1001:Customer {id=1001, firstName=Sally, lastName=Thomas, email=sally.thomas@acme.com}", "1002:Customer {id=1002, firstName=George, lastName=Bailey, email=gbailey@foobar.com}", "1003:Customer {id=1003, firstName=Edward, lastName=Walker, email=ed@walker.com}", "1004:Customer {id=1004, firstName=Anne Marie, lastName=Kretchmar, email=annek@noanswer.org}"));
}

Example 93 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class PostgresCdcAuthAndConnectionIntegrationTest method wrongPassword.

@Test
public void wrongPassword() {
    StreamSource<ChangeRecord> source = PostgresCdcSources.postgres("name").setDatabaseAddress(postgres.getContainerIpAddress()).setDatabasePort(postgres.getMappedPort(POSTGRESQL_PORT)).setDatabaseUser("postgres").setDatabasePassword("wrongPassword").setDatabaseName("postgres").build();
    Pipeline pipeline = pipeline(source);
    HazelcastInstance hz = createHazelcastInstances(2)[0];
    // when
    Job job = hz.getJet().newJob(pipeline);
    // then
    assertThatThrownBy(job::join).hasRootCauseInstanceOf(JetException.class).hasStackTraceContaining("password authentication failed for user \"postgres\"");
}

Example 94 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class PostgresCdcAuthAndConnectionIntegrationTest method pipeline.

private Pipeline pipeline(StreamSource<ChangeRecord> source) {
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source).withoutTimestamps().writeTo(Sinks.logger());
    return pipeline;
}

Example 95 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class DebeziumCdcIntegrationTest method postgres_simpleJson.

@Test
public void postgres_simpleJson() {
    PostgreSQLContainer<?> container = postgresContainer();
    try {
        container.start();
        // given
        List<String> expectedRecords = Arrays.asList("\\{\"id\":1001}:\\{\"before\":null," + "\"after\":\\{\"id\":1001,\"first_name\":\"Sally\",\"last_name\":\"Thomas\"," + "\"email\":\"sally.thomas@acme.com\"}," + "\"source\":\\{\"version\":\"1.2.5.Final\",\"connector\":\"postgresql\"," + "\"name\":\"dbserver1\",\"ts_ms\":[0-9]*,\"snapshot\":\"true\",\"db\":\"postgres\"," + "\"schema\":\"inventory\",\"table\":\"customers\",\"txId\":[0-9]*,\"lsn\":[0-9]*," + "\"xmin\":null},\"op\":\"r\",\"ts_ms\":[0-9]*,\"transaction\":null}", "\\{\"id\":1002}:\\{\"before\":null," + "\"after\":\\{\"id\":1002,\"first_name\":\"George\",\"last_name\":\"Bailey\"," + "\"email\":\"gbailey@foobar.com\"}," + "\"source\":\\{\"version\":\"1.2.5.Final\",\"connector\":\"postgresql\"," + "\"name\":\"dbserver1\",\"ts_ms\":[0-9]*,\"snapshot\":\"true\",\"db\":\"postgres\"," + "\"schema\":\"inventory\",\"table\":\"customers\",\"txId\":[0-9]*,\"lsn\":[0-9]*," + "\"xmin\":null},\"op\":\"r\",\"ts_ms\":[0-9]*,\"transaction\":null}", "\\{\"id\":1003}:\\{\"before\":null," + "\"after\":\\{\"id\":1003,\"first_name\":\"Edward\",\"last_name\":\"Walker\"," + "\"email\":\"ed@walker.com\"}," + "\"source\":\\{\"version\":\"1.2.5.Final\",\"connector\":\"postgresql\"," + "\"name\":\"dbserver1\",\"ts_ms\":[0-9]*,\"snapshot\":\"true\",\"db\":\"postgres\"," + "\"schema\":\"inventory\",\"table\":\"customers\",\"txId\":[0-9]*,\"lsn\":[0-9]*," + "\"xmin\":null},\"op\":\"r\",\"ts_ms\":[0-9]*,\"transaction\":null}", "\\{\"id\":1004}:\\{\"before\":null," + "\"after\":\\{\"id\":1004,\"first_name\":\"Anne\",\"last_name\":\"Kretchmar\"," + "\"email\":\"annek@noanswer.org\"}," + "\"source\":\\{\"version\":\"1.2.5.Final\",\"connector\":\"postgresql\"," + "\"name\":\"dbserver1\",\"ts_ms\":[0-9]*,\"snapshot\":\"last\",\"db\":\"postgres\"," + "\"schema\":\"inventory\",\"table\":\"customers\",\"txId\":[0-9]*,\"lsn\":[0-9]*," + "\"xmin\":null},\"op\":\"r\",\"ts_ms\":[0-9]*,\"transaction\":null}");
        StreamSource<Entry<String, String>> source = DebeziumCdcSources.debeziumJson("postgres", "io.debezium.connector.postgresql.PostgresConnector").setProperty("database.server.name", "dbserver1").setProperty("database.hostname", container.getContainerIpAddress()).setProperty("database.port", Integer.toString(container.getMappedPort(POSTGRESQL_PORT))).setProperty("database.user", "postgres").setProperty("database.password", "postgres").setProperty("database.dbname", "postgres").setProperty("table.whitelist", "inventory.customers").build();
        Pipeline pipeline = Pipeline.create();
        pipeline.readFrom(source).withNativeTimestamps(0).writeTo(Sinks.map("results"));
        // when
        HazelcastInstance hz = createHazelcastInstances(2)[0];
        Job job = hz.getJet().newJob(pipeline);
        // then
        try {
            assertTrueEventually(() -> assertMatch(expectedRecords, mapResultsToSortedList(hz.getMap("results"))));
        } finally {
            job.cancel();
            assertJobStatusEventually(job, JobStatus.FAILED);
        }
    } finally {
        container.stop();
    }
}