Search in sources :

Example 1 with RecordPart

use of com.hazelcast.jet.cdc.RecordPart in project hazelcast by hazelcast.

the class MySqlCdcIntegrationTest method restart.

@Test
@Category(NightlyTest.class)
public void restart() throws Exception {
    // given
    List<String> expectedRecords = Arrays.asList("1004/1:UPDATE:Customer {id=1004, firstName=Anne Marie, lastName=Kretchmar, email=annek@noanswer.org}", "1005/0:INSERT:Customer {id=1005, firstName=Jason, lastName=Bourne, email=jason@bourne.org}", "1005/1:DELETE:Customer {id=1005, firstName=Jason, lastName=Bourne, email=jason@bourne.org}");
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source("customers")).withNativeTimestamps(0).<ChangeRecord>customTransform("filter_timestamps", filterTimestampsProcessorSupplier()).groupingKey(record -> (Integer) record.key().toMap().get("id")).mapStateful(LongAccumulator::new, (accumulator, customerId, record) -> {
        long count = accumulator.get();
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        Customer customer = value.toObject(Customer.class);
        return entry(customerId + "/" + count, operation + ":" + customer);
    }).setLocalParallelism(1).writeTo(Sinks.map("results"));
    // when
    HazelcastInstance hz = createHazelcastInstances(2)[0];
    JobConfig jobConfig = new JobConfig().setProcessingGuarantee(ProcessingGuarantee.AT_LEAST_ONCE);
    Job job = hz.getJet().newJob(pipeline, jobConfig);
    JetTestSupport.assertJobStatusEventually(job, JobStatus.RUNNING);
    assertEqualsEventually(() -> hz.getMap("results").size(), 4);
    // then
    hz.getMap("results").destroy();
    // when
    assertEqualsEventually(() -> hz.getMap("results").size(), 0);
    // then
    job.restart();
    // when
    JetTestSupport.assertJobStatusEventually(job, JobStatus.RUNNING);
    // then update a record
    try (Connection connection = getConnection(mysql, "inventory")) {
        Statement statement = connection.createStatement();
        statement.addBatch("UPDATE customers SET first_name='Anne Marie' WHERE id=1004");
        statement.addBatch("INSERT INTO customers VALUES (1005, 'Jason', 'Bourne', 'jason@bourne.org')");
        statement.addBatch("DELETE FROM customers WHERE id=1005");
        statement.executeBatch();
    }
    // then
    try {
        assertEqualsEventually(() -> mapResultsToSortedList(hz.getMap("results")), expectedRecords);
    } finally {
        job.cancel();
        assertJobStatusEventually(job, JobStatus.FAILED);
    }
}
Also used : JsonProperty(com.fasterxml.jackson.annotation.JsonProperty) Arrays(java.util.Arrays) Connection(java.sql.Connection) QuickTest(com.hazelcast.test.annotation.QuickTest) Date(java.util.Date) StreamSource(com.hazelcast.jet.pipeline.StreamSource) CdcSinks(com.hazelcast.jet.cdc.CdcSinks) ParsingException(com.hazelcast.jet.cdc.ParsingException) Operation(com.hazelcast.jet.cdc.Operation) Util.entry(com.hazelcast.jet.Util.entry) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) ChangeRecord(com.hazelcast.jet.cdc.ChangeRecord) JobStatus(com.hazelcast.jet.core.JobStatus) Nonnull(javax.annotation.Nonnull) Job(com.hazelcast.jet.Job) HazelcastInstance(com.hazelcast.core.HazelcastInstance) NightlyTest(com.hazelcast.test.annotation.NightlyTest) Pipeline(com.hazelcast.jet.pipeline.Pipeline) JetTestSupport(com.hazelcast.jet.core.JetTestSupport) JobConfig(com.hazelcast.jet.config.JobConfig) Sinks(com.hazelcast.jet.pipeline.Sinks) RecordPart(com.hazelcast.jet.cdc.RecordPart) Test(org.junit.Test) Category(org.junit.experimental.categories.Category) Objects(java.util.Objects) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) Statement(java.sql.Statement) ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) Statement(java.sql.Statement) Connection(java.sql.Connection) Operation(com.hazelcast.jet.cdc.Operation) JobConfig(com.hazelcast.jet.config.JobConfig) Pipeline(com.hazelcast.jet.pipeline.Pipeline) LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) HazelcastInstance(com.hazelcast.core.HazelcastInstance) RecordPart(com.hazelcast.jet.cdc.RecordPart) Job(com.hazelcast.jet.Job) ChangeRecord(com.hazelcast.jet.cdc.ChangeRecord) Category(org.junit.experimental.categories.Category) QuickTest(com.hazelcast.test.annotation.QuickTest) NightlyTest(com.hazelcast.test.annotation.NightlyTest) Test(org.junit.Test)

Example 2 with RecordPart

use of com.hazelcast.jet.cdc.RecordPart in project hazelcast by hazelcast.

the class MySqlCdcIntegrationTest method orders.

@Test
@Category(NightlyTest.class)
public void orders() {
    // given
    List<String> expectedRecords = Arrays.asList("10001/0:INSERT:Order {orderNumber=10001, orderDate=" + new Date(1452902400000L) + ", quantity=1, productId=102}", "10002/0:INSERT:Order {orderNumber=10002, orderDate=" + new Date(1452988800000L) + ", quantity=2, productId=105}", "10003/0:INSERT:Order {orderNumber=10003, orderDate=" + new Date(1455840000000L) + ", quantity=2, productId=106}", "10004/0:INSERT:Order {orderNumber=10004, orderDate=" + new Date(1456012800000L) + ", quantity=1, productId=107}");
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source("orders")).withoutTimestamps().groupingKey(MySqlCdcIntegrationTest::getOrderNumber).mapStateful(LongAccumulator::new, (accumulator, orderId, record) -> {
        long count = accumulator.get();
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        Order order = value.toObject(Order.class);
        return entry(orderId + "/" + count, operation + ":" + order);
    }).setLocalParallelism(1).writeTo(Sinks.map("results"));
    // when
    HazelcastInstance hz = createHazelcastInstances(2)[0];
    Job job = hz.getJet().newJob(pipeline);
    // then
    try {
        assertEqualsEventually(() -> mapResultsToSortedList(hz.getMap("results")), expectedRecords);
    } finally {
        job.cancel();
        assertJobStatusEventually(job, JobStatus.FAILED);
    }
}
Also used : LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) HazelcastInstance(com.hazelcast.core.HazelcastInstance) RecordPart(com.hazelcast.jet.cdc.RecordPart) Operation(com.hazelcast.jet.cdc.Operation) Job(com.hazelcast.jet.Job) Date(java.util.Date) Pipeline(com.hazelcast.jet.pipeline.Pipeline) Category(org.junit.experimental.categories.Category) QuickTest(com.hazelcast.test.annotation.QuickTest) NightlyTest(com.hazelcast.test.annotation.NightlyTest) Test(org.junit.Test)

Example 3 with RecordPart

use of com.hazelcast.jet.cdc.RecordPart in project hazelcast by hazelcast.

the class MySqlCdcIntegrationTest method customers.

@Test
@Category(QuickTest.class)
public void customers() throws Exception {
    // given
    List<String> expectedRecords = Arrays.asList("1001/0:INSERT:Customer {id=1001, firstName=Sally, lastName=Thomas, email=sally.thomas@acme.com}", "1002/0:INSERT:Customer {id=1002, firstName=George, lastName=Bailey, email=gbailey@foobar.com}", "1003/0:INSERT:Customer {id=1003, firstName=Edward, lastName=Walker, email=ed@walker.com}", "1004/0:INSERT:Customer {id=1004, firstName=Anne, lastName=Kretchmar, email=annek@noanswer.org}", "1004/1:UPDATE:Customer {id=1004, firstName=Anne Marie, lastName=Kretchmar, email=annek@noanswer.org}", "1005/0:INSERT:Customer {id=1005, firstName=Jason, lastName=Bourne, email=jason@bourne.org}", "1005/1:DELETE:Customer {id=1005, firstName=Jason, lastName=Bourne, email=jason@bourne.org}");
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source("customers")).withNativeTimestamps(0).<ChangeRecord>customTransform("filter_timestamps", filterTimestampsProcessorSupplier()).groupingKey(record -> (Integer) record.key().toMap().get("id")).mapStateful(LongAccumulator::new, (accumulator, customerId, record) -> {
        long count = accumulator.get();
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        Customer customer = value.toObject(Customer.class);
        return entry(customerId + "/" + count, operation + ":" + customer);
    }).setLocalParallelism(1).writeTo(Sinks.map("results"));
    // when
    HazelcastInstance hz = createHazelcastInstances(2)[0];
    Job job = hz.getJet().newJob(pipeline);
    // then
    assertEqualsEventually(() -> hz.getMap("results").size(), 4);
    // when
    try (Connection connection = getConnection(mysql, "inventory")) {
        Statement statement = connection.createStatement();
        statement.addBatch("UPDATE customers SET first_name='Anne Marie' WHERE id=1004");
        statement.addBatch("INSERT INTO customers VALUES (1005, 'Jason', 'Bourne', 'jason@bourne.org')");
        statement.addBatch("DELETE FROM customers WHERE id=1005");
        statement.executeBatch();
    }
    // then
    try {
        assertEqualsEventually(() -> mapResultsToSortedList(hz.getMap("results")), expectedRecords);
    } finally {
        job.cancel();
        assertJobStatusEventually(job, JobStatus.FAILED);
    }
}
Also used : JsonProperty(com.fasterxml.jackson.annotation.JsonProperty) Arrays(java.util.Arrays) Connection(java.sql.Connection) QuickTest(com.hazelcast.test.annotation.QuickTest) Date(java.util.Date) StreamSource(com.hazelcast.jet.pipeline.StreamSource) CdcSinks(com.hazelcast.jet.cdc.CdcSinks) ParsingException(com.hazelcast.jet.cdc.ParsingException) Operation(com.hazelcast.jet.cdc.Operation) Util.entry(com.hazelcast.jet.Util.entry) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) ChangeRecord(com.hazelcast.jet.cdc.ChangeRecord) JobStatus(com.hazelcast.jet.core.JobStatus) Nonnull(javax.annotation.Nonnull) Job(com.hazelcast.jet.Job) HazelcastInstance(com.hazelcast.core.HazelcastInstance) NightlyTest(com.hazelcast.test.annotation.NightlyTest) Pipeline(com.hazelcast.jet.pipeline.Pipeline) JetTestSupport(com.hazelcast.jet.core.JetTestSupport) JobConfig(com.hazelcast.jet.config.JobConfig) Sinks(com.hazelcast.jet.pipeline.Sinks) RecordPart(com.hazelcast.jet.cdc.RecordPart) Test(org.junit.Test) Category(org.junit.experimental.categories.Category) Objects(java.util.Objects) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) Statement(java.sql.Statement) ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) Statement(java.sql.Statement) Connection(java.sql.Connection) Operation(com.hazelcast.jet.cdc.Operation) Pipeline(com.hazelcast.jet.pipeline.Pipeline) LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) HazelcastInstance(com.hazelcast.core.HazelcastInstance) RecordPart(com.hazelcast.jet.cdc.RecordPart) Job(com.hazelcast.jet.Job) ChangeRecord(com.hazelcast.jet.cdc.ChangeRecord) Category(org.junit.experimental.categories.Category) QuickTest(com.hazelcast.test.annotation.QuickTest) NightlyTest(com.hazelcast.test.annotation.NightlyTest) Test(org.junit.Test)

Example 4 with RecordPart

use of com.hazelcast.jet.cdc.RecordPart in project hazelcast by hazelcast.

the class PostgresCdcIntegrationTest method customersPipeline.

@Nonnull
private Pipeline customersPipeline(Long commitPeriod) {
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source("customers", commitPeriod)).withNativeTimestamps(0).<ChangeRecord>customTransform("filter_timestamps", filterTimestampsProcessorSupplier()).groupingKey(record -> (Integer) record.key().toMap().get("id")).mapStateful(LongAccumulator::new, (accumulator, customerId, record) -> {
        String count = format("%05d", accumulator.get());
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        Customer customer = value.toObject(Customer.class);
        return entry(customerId + "/" + count, operation + ":" + customer);
    }).setLocalParallelism(1).writeTo(Sinks.map("results"));
    return pipeline;
}
Also used : JsonProperty(com.fasterxml.jackson.annotation.JsonProperty) Arrays(java.util.Arrays) Connection(java.sql.Connection) Util.uncheckRun(com.hazelcast.jet.impl.util.Util.uncheckRun) QuickTest(com.hazelcast.test.annotation.QuickTest) Date(java.util.Date) JobProxy(com.hazelcast.jet.impl.JobProxy) StreamSource(com.hazelcast.jet.pipeline.StreamSource) CdcSinks(com.hazelcast.jet.cdc.CdcSinks) ParsingException(com.hazelcast.jet.cdc.ParsingException) ArrayList(java.util.ArrayList) SQLException(java.sql.SQLException) Future(java.util.concurrent.Future) Operation(com.hazelcast.jet.cdc.Operation) ResultSet(java.sql.ResultSet) Util.entry(com.hazelcast.jet.Util.entry) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) Assert.fail(org.junit.Assert.fail) ChangeRecord(com.hazelcast.jet.cdc.ChangeRecord) JobStatus(com.hazelcast.jet.core.JobStatus) Nonnull(javax.annotation.Nonnull) Job(com.hazelcast.jet.Job) JobRepository(com.hazelcast.jet.impl.JobRepository) HazelcastInstance(com.hazelcast.core.HazelcastInstance) NightlyTest(com.hazelcast.test.annotation.NightlyTest) Pipeline(com.hazelcast.jet.pipeline.Pipeline) JetTestSupport(com.hazelcast.jet.core.JetTestSupport) JobConfig(com.hazelcast.jet.config.JobConfig) Sinks(com.hazelcast.jet.pipeline.Sinks) RecordPart(com.hazelcast.jet.cdc.RecordPart) Test(org.junit.Test) Category(org.junit.experimental.categories.Category) PreparedStatement(java.sql.PreparedStatement) String.format(java.lang.String.format) Assert.assertNotEquals(org.junit.Assert.assertNotEquals) List(java.util.List) LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) RUNNING(com.hazelcast.jet.core.JobStatus.RUNNING) ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) Assert.assertEquals(org.junit.Assert.assertEquals) IMap(com.hazelcast.map.IMap) LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) RecordPart(com.hazelcast.jet.cdc.RecordPart) Operation(com.hazelcast.jet.cdc.Operation) ChangeRecord(com.hazelcast.jet.cdc.ChangeRecord) Pipeline(com.hazelcast.jet.pipeline.Pipeline) Nonnull(javax.annotation.Nonnull)

Example 5 with RecordPart

use of com.hazelcast.jet.cdc.RecordPart in project hazelcast by hazelcast.

the class PostgresCdcWhiteBlackListIntegrationTest method pipeline.

private Pipeline pipeline(StreamSource<ChangeRecord> source) {
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source).withNativeTimestamps(0).filter(t -> t.schema().startsWith(SCHEMA_PREFIX)).setLocalParallelism(1).<ChangeRecord>customTransform("filter_timestamps", filterTimestampsProcessorSupplier()).setLocalParallelism(1).groupingKey(record -> (Integer) record.key().toMap().get("id")).mapStateful(LongAccumulator::new, (accumulator, rowId, record) -> {
        long count = accumulator.get();
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        TableRow row = value.toObject(TableRow.class);
        return entry(rowId + "/" + count, operation + ":" + row);
    }).setLocalParallelism(1).peek().writeTo(Sinks.map(SINK_MAP_NAME));
    return pipeline;
}
Also used : HazelcastInstance(com.hazelcast.core.HazelcastInstance) NightlyTest(com.hazelcast.test.annotation.NightlyTest) Arrays(java.util.Arrays) Connection(java.sql.Connection) Pipeline(com.hazelcast.jet.pipeline.Pipeline) Sinks(com.hazelcast.jet.pipeline.Sinks) RecordPart(com.hazelcast.jet.cdc.RecordPart) Test(org.junit.Test) StreamSource(com.hazelcast.jet.pipeline.StreamSource) Category(org.junit.experimental.categories.Category) ArrayList(java.util.ArrayList) SQLException(java.sql.SQLException) List(java.util.List) Operation(com.hazelcast.jet.cdc.Operation) LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) Util.entry(com.hazelcast.jet.Util.entry) Statement(java.sql.Statement) ChangeRecord(com.hazelcast.jet.cdc.ChangeRecord) JobStatus(com.hazelcast.jet.core.JobStatus) Job(com.hazelcast.jet.Job) Before(org.junit.Before) RecordPart(com.hazelcast.jet.cdc.RecordPart) Operation(com.hazelcast.jet.cdc.Operation) ChangeRecord(com.hazelcast.jet.cdc.ChangeRecord) Pipeline(com.hazelcast.jet.pipeline.Pipeline)

Aggregations

LongAccumulator (com.hazelcast.jet.accumulator.LongAccumulator)9 Operation (com.hazelcast.jet.cdc.Operation)9 RecordPart (com.hazelcast.jet.cdc.RecordPart)9 Pipeline (com.hazelcast.jet.pipeline.Pipeline)9 HazelcastInstance (com.hazelcast.core.HazelcastInstance)8 Job (com.hazelcast.jet.Job)8 NightlyTest (com.hazelcast.test.annotation.NightlyTest)8 Test (org.junit.Test)8 Category (org.junit.experimental.categories.Category)8 Util.entry (com.hazelcast.jet.Util.entry)7 ChangeRecord (com.hazelcast.jet.cdc.ChangeRecord)7 JobStatus (com.hazelcast.jet.core.JobStatus)7 Sinks (com.hazelcast.jet.pipeline.Sinks)7 StreamSource (com.hazelcast.jet.pipeline.StreamSource)7 Connection (java.sql.Connection)7 Arrays (java.util.Arrays)7 List (java.util.List)7 Statement (java.sql.Statement)6 SQLException (java.sql.SQLException)5 QuickTest (com.hazelcast.test.annotation.QuickTest)4