Examples with ChangeRecord com.hazelcast.jet.cdc.ChangeRecord used on opensource projects

Example 6 with ChangeRecord

use of com.hazelcast.jet.cdc.ChangeRecord in project hazelcast by hazelcast.

the class PostgresCdcAuthAndConnectionIntegrationTest method wrongPassword.

@Test
public void wrongPassword() {
    StreamSource<ChangeRecord> source = PostgresCdcSources.postgres("name").setDatabaseAddress(postgres.getContainerIpAddress()).setDatabasePort(postgres.getMappedPort(POSTGRESQL_PORT)).setDatabaseUser("postgres").setDatabasePassword("wrongPassword").setDatabaseName("postgres").build();
    Pipeline pipeline = pipeline(source);
    HazelcastInstance hz = createHazelcastInstances(2)[0];
    // when
    Job job = hz.getJet().newJob(pipeline);
    // then
    assertThatThrownBy(job::join).hasRootCauseInstanceOf(JetException.class).hasStackTraceContaining("password authentication failed for user \"postgres\"");
}

Example 7 with ChangeRecord

use of com.hazelcast.jet.cdc.ChangeRecord in project hazelcast by hazelcast.

the class PostgresCdcWhiteBlackListIntegrationTest method pipeline.

private Pipeline pipeline(StreamSource<ChangeRecord> source) {
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source).withNativeTimestamps(0).filter(t -> t.schema().startsWith(SCHEMA_PREFIX)).setLocalParallelism(1).<ChangeRecord>customTransform("filter_timestamps", filterTimestampsProcessorSupplier()).setLocalParallelism(1).groupingKey(record -> (Integer) record.key().toMap().get("id")).mapStateful(LongAccumulator::new, (accumulator, rowId, record) -> {
        long count = accumulator.get();
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        TableRow row = value.toObject(TableRow.class);
        return entry(rowId + "/" + count, operation + ":" + row);
    }).setLocalParallelism(1).peek().writeTo(Sinks.map(SINK_MAP_NAME));
    return pipeline;
}

Example 8 with ChangeRecord

use of com.hazelcast.jet.cdc.ChangeRecord in project hazelcast by hazelcast.

the class MySqlCdcListenBeforeExistIntegrationTest method listenBeforeColumnExists.

@Test
public void listenBeforeColumnExists() throws Exception {
    // given
    createDb(DATABASE);
    createTableWithData(DATABASE, "someTable");
    insertToTable(DATABASE, "someTable", 1001, "someValue1", "someValue2");
    List<String> expectedRecords = Arrays.asList("1001/0:INSERT:TableRow {id=1001, value1=someValue1, value2=someValue2, value3=null}", "1002/0:INSERT:TableRow {id=1002, value1=someValue4, value2=someValue5, value3=someValue6}");
    StreamSource<ChangeRecord> source = sourceBuilder("cdcMysql").setDatabaseWhitelist(DATABASE).setTableWhitelist(DATABASE + ".someTable").build();
    Pipeline pipeline = pipeline(source);
    // when
    HazelcastInstance hz = createHazelcastInstances(2)[0];
    Job job = hz.getJet().newJob(pipeline);
    assertJobStatusEventually(job, RUNNING);
    try {
        assertEqualsEventually(() -> mapResultsToSortedList(hz.getMap(SINK_MAP_NAME)), Collections.singletonList("1001/0:INSERT:TableRow {id=1001, value1=someValue1, value2=someValue2, value3=null}"));
        // then
        addColumnToTable(DATABASE, "someTable", "value_3");
        insertToTable(DATABASE, "someTable", 1002, "someValue4", "someValue5", "someValue6");
        assertEqualsEventually(() -> mapResultsToSortedList(hz.getMap(SINK_MAP_NAME)), expectedRecords);
    } finally {
        job.cancel();
        assertJobStatusEventually(job, JobStatus.FAILED);
    }
}

Example 9 with ChangeRecord

use of com.hazelcast.jet.cdc.ChangeRecord in project hazelcast by hazelcast.

the class PostgresCdcListenBeforeExistsIntegrationTest method pipeline.

private Pipeline pipeline(StreamSource<ChangeRecord> source) {
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source).withNativeTimestamps(0).<ChangeRecord>customTransform("filter_timestamps", filterTimestampsProcessorSupplier()).setLocalParallelism(1).groupingKey(record -> (Integer) record.key().toMap().get("id")).mapStateful(LongAccumulator::new, (accumulator, rowId, record) -> {
        long count = accumulator.get();
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        TableRow row = value.toObject(TableRow.class);
        return entry(rowId + "/" + count, operation + ":" + row);
    }).setLocalParallelism(1).peek().writeTo(Sinks.map(SINK_MAP_NAME));
    return pipeline;
}

Example 10 with ChangeRecord

use of com.hazelcast.jet.cdc.ChangeRecord in project hazelcast by hazelcast.

the class PostgresCdcListenBeforeExistsIntegrationTest method listenBeforeSchemaExists.

@Test
public void listenBeforeSchemaExists() throws Exception {
    List<String> expectedRecords = Arrays.asList("1001/0:(SYNC|INSERT):TableRow \\{id=1001, value1=someValue1, value2=someValue2, value3=null\\}");
    StreamSource<ChangeRecord> source = sourceBuilder("source").setSchemaWhitelist(SCHEMA).build();
    Pipeline pipeline = pipeline(source);
    // when
    HazelcastInstance hz = createHazelcastInstances(2)[0];
    Job job = hz.getJet().newJob(pipeline);
    assertJobStatusEventually(job, RUNNING);
    assertReplicationSlotActive();
    try {
        // then
        createSchema(SCHEMA);
        createTableWithData(SCHEMA, "someTable");
        insertIntoTable(SCHEMA, "someTable", 1001, "someValue1", "someValue2");
        assertTrueEventually(() -> assertMatch(expectedRecords, mapResultsToSortedList(hz.getMap(SINK_MAP_NAME))));
    } finally {
        job.cancel();
        assertJobStatusEventually(job, JobStatus.FAILED);
    }
}