Examples with ChangeRecord com.hazelcast.jet.cdc.ChangeRecord used on opensource projects

Example 11 with ChangeRecord

use of com.hazelcast.jet.cdc.ChangeRecord in project hazelcast by hazelcast.

the class PostgresCdcListenBeforeExistsIntegrationTest method listenBeforeColumnExists.

@Test
public void listenBeforeColumnExists() throws Exception {
    // given
    createSchema(SCHEMA);
    createTableWithData(SCHEMA, "someTable");
    insertIntoTable(SCHEMA, "someTable", 1001, "someValue1", "someValue2");
    List<String> expectedRecords = Arrays.asList("1001/0:(SYNC|INSERT):TableRow \\{id=1001, value1=someValue1, value2=someValue2, value3=null\\}", "1002/0:(SYNC|INSERT):TableRow \\{id=1002, value1=someValue4, value2=someValue5, value3=someValue6\\}");
    StreamSource<ChangeRecord> source = sourceBuilder("source").setSchemaWhitelist(SCHEMA).setTableWhitelist(SCHEMA + ".someTable").build();
    Pipeline pipeline = pipeline(source);
    // when
    HazelcastInstance hz = createHazelcastInstances(2)[0];
    Job job = hz.getJet().newJob(pipeline);
    assertJobStatusEventually(job, RUNNING);
    assertReplicationSlotActive();
    try {
        assertTrueEventually(() -> assertMatch(Collections.singletonList("1001/0:(SYNC|INSERT):TableRow \\{id=1001, value1=someValue1, value2=someValue2, value3=null\\}"), mapResultsToSortedList(hz.getMap(SINK_MAP_NAME))));
        // then
        addColumnToTable(SCHEMA, "someTable", "value_3");
        insertIntoTable(SCHEMA, "someTable", 1002, "someValue4", "someValue5", "someValue6");
        assertTrueEventually(() -> assertMatch(expectedRecords, mapResultsToSortedList(hz.getMap(SINK_MAP_NAME))));
    } finally {
        job.cancel();
        assertJobStatusEventually(job, JobStatus.FAILED);
    }
}

Example 12 with ChangeRecord

use of com.hazelcast.jet.cdc.ChangeRecord in project hazelcast by hazelcast.

the class MultiTableCacheIntegrationTest method ordersOfCustomers.

@Test
public void ordersOfCustomers() throws Exception {
    StreamSource<ChangeRecord> source = sourceBuilder("source").setTableWhitelist("inventory.customers", "inventory.orders").build();
    Pipeline pipeline = Pipeline.create();
    StreamStage<ChangeRecord> allRecords = pipeline.readFrom(source).withNativeTimestamps(0);
    allRecords.filter(r -> r.table().equals("customers")).apply(this::fixOrdering).writeTo(Sinks.mapWithEntryProcessor(MAX_CONCURRENT_OPERATIONS, CACHE, record -> (Integer) record.key().toMap().get("id"), CustomerEntryProcessor::new));
    allRecords.filter(r -> r.table().equals("orders")).apply(this::fixOrdering).writeTo(Sinks.mapWithEntryProcessor(MAX_CONCURRENT_OPERATIONS, CACHE, record -> (Integer) record.value().toMap().get("purchaser"), OrderEntryProcessor::new));
    // when
    HazelcastInstance hz = createHazelcastInstances(1)[0];
    Job job = hz.getJet().newJob(pipeline);
    // then
    Map<Integer, OrdersOfCustomer> expected = toMap(new OrdersOfCustomer(new Customer(1001, "Sally", "Thomas", "sally.thomas@acme.com"), new Order(10001, new Date(1452902400000L), 1001, 1, 102)), new OrdersOfCustomer(new Customer(1002, "George", "Bailey", "gbailey@foobar.com"), new Order(10002, new Date(1452988800000L), 1002, 2, 105), new Order(10003, new Date(1455840000000L), 1002, 2, 106)), new OrdersOfCustomer(new Customer(1003, "Edward", "Walker", "ed@walker.com"), new Order(10004, new Date(1456012800000L), 1003, 1, 107)), new OrdersOfCustomer(new Customer(1004, "Anne", "Kretchmar", "annek@noanswer.org")));
    assertEqualsEventually(() -> getIMapContent(hz, CACHE), expected);
    // when
    List<String> batch = new ArrayList<>();
    for (int i = 1; i <= REPEATS; i++) {
        batch.add("UPDATE customers SET first_name='Anne" + i + "' WHERE id=1004");
        batch.add("INSERT INTO customers VALUES (1005, 'Jason', 'Bourne', 'jason@bourne.org')");
        batch.add("DELETE FROM customers WHERE id=1005");
        batch.add("UPDATE orders SET quantity='" + i + "' WHERE id=10004");
        batch.add("DELETE FROM orders WHERE id=10003");
        batch.add("INSERT INTO orders VALUES (10003, '2016-02-19', 1002, 2, 106)");
    }
    executeBatch(batch.toArray(new String[0]));
    // then
    expected = toMap(new OrdersOfCustomer(new Customer(1001, "Sally", "Thomas", "sally.thomas@acme.com"), new Order(10001, new Date(1452902400000L), 1001, 1, 102)), new OrdersOfCustomer(new Customer(1002, "George", "Bailey", "gbailey@foobar.com"), new Order(10002, new Date(1452988800000L), 1002, 2, 105), new Order(10003, new Date(1455840000000L), 1002, 2, 106)), new OrdersOfCustomer(new Customer(1003, "Edward", "Walker", "ed@walker.com"), new Order(10004, new Date(1456012800000L), 1003, REPEATS, 107)), new OrdersOfCustomer(new Customer(1004, "Anne" + REPEATS, "Kretchmar", "annek@noanswer.org")));
    expected.put(1005, new OrdersOfCustomer());
    assertEqualsEventually(() -> getIMapContent(hz, CACHE), expected);
}

Example 13 with ChangeRecord

use of com.hazelcast.jet.cdc.ChangeRecord in project hazelcast by hazelcast.

the class MySqlCdcAuthIntegrationTest method wrongPassword.

@Test
public void wrongPassword() {
    StreamSource<ChangeRecord> source = MySqlCdcSources.mysql("name").setDatabaseAddress(mysql.getContainerIpAddress()).setDatabasePort(mysql.getMappedPort(MYSQL_PORT)).setDatabaseUser("debezium").setDatabasePassword("wrongPassword").setClusterName("dbserver1").build();
    Pipeline pipeline = pipeline(source);
    // when
    HazelcastInstance hz = createHazelcastInstances(2)[0];
    Job job = hz.getJet().newJob(pipeline);
    // then
    assertThatThrownBy(job::join).hasRootCauseInstanceOf(JetException.class).hasStackTraceContaining("Access denied for user");
}

Example 14 with ChangeRecord

use of com.hazelcast.jet.cdc.ChangeRecord in project hazelcast by hazelcast.

the class MySqlCdcListenBeforeExistIntegrationTest method pipeline.

private Pipeline pipeline(StreamSource<ChangeRecord> source) {
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source).withNativeTimestamps(0).<ChangeRecord>customTransform("filter_timestamps", filterTimestampsProcessorSupplier()).setLocalParallelism(1).groupingKey(record -> (Integer) record.key().toMap().get("id")).mapStateful(LongAccumulator::new, (accumulator, rowId, record) -> {
        long count = accumulator.get();
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        TableRow row = value.toObject(TableRow.class);
        return entry(rowId + "/" + count, operation + ":" + row);
    }).setLocalParallelism(1).peek().writeTo(Sinks.map(SINK_MAP_NAME));
    return pipeline;
}

Example 15 with ChangeRecord

use of com.hazelcast.jet.cdc.ChangeRecord in project hazelcast by hazelcast.

the class MySqlCdcListenBeforeExistIntegrationTest method listenBeforeTableExists.

@Test
public void listenBeforeTableExists() throws Exception {
    // given
    createDb(DATABASE);
    List<String> expectedRecords = Collections.singletonList("1001/0:INSERT:TableRow {id=1001, value1=someValue1, value2=someValue2, value3=null}");
    StreamSource<ChangeRecord> source = sourceBuilder("cdcMysql").setDatabaseWhitelist(DATABASE).setTableWhitelist(DATABASE + ".someTable").build();
    Pipeline pipeline = pipeline(source);
    // when
    HazelcastInstance hz = createHazelcastInstances(2)[0];
    Job job = hz.getJet().newJob(pipeline);
    assertJobStatusEventually(job, RUNNING);
    try {
        // then
        createTableWithData(DATABASE, "someTable");
        insertToTable(DATABASE, "someTable", 1001, "someValue1", "someValue2");
        assertEqualsEventually(() -> mapResultsToSortedList(hz.getMap(SINK_MAP_NAME)), expectedRecords);
    } finally {
        job.cancel();
        assertJobStatusEventually(job, JobStatus.FAILED);
    }
}