use of io.debezium.data.KeyValueStore.Collection in project debezium by debezium.
the class SnapshotReaderIT method shouldCreateSnapshotOfSingleDatabaseWithSchemaChanges.
@Test
public void shouldCreateSnapshotOfSingleDatabaseWithSchemaChanges() throws Exception {
config = simpleConfig().with(MySqlConnectorConfig.INCLUDE_SCHEMA_CHANGES, true).build();
context = new MySqlTaskContext(config);
context.start();
reader = new SnapshotReader("snapshot", context);
reader.uponCompletion(completed::countDown);
reader.generateInsertEvents();
// Start the snapshot ...
reader.start();
// Poll for records ...
// Testing.Print.enable();
List<SourceRecord> records = null;
KeyValueStore store = KeyValueStore.createForTopicsBeginningWith(DATABASE.getServerName() + ".");
SchemaChangeHistory schemaChanges = new SchemaChangeHistory(DATABASE.getServerName());
while ((records = reader.poll()) != null) {
records.forEach(record -> {
VerifyRecord.isValid(record);
store.add(record);
schemaChanges.add(record);
});
}
// The last poll should always return null ...
assertThat(records).isNull();
// There should be 11 schema changes plus 1 SET statement ...
assertThat(schemaChanges.recordCount()).isEqualTo(14);
assertThat(schemaChanges.databaseCount()).isEqualTo(2);
assertThat(schemaChanges.databases()).containsOnly(DATABASE.getDatabaseName(), "");
// Check the records via the store ...
assertThat(store.collectionCount()).isEqualTo(5);
Collection products = store.collection(DATABASE.getDatabaseName(), productsTableName());
assertThat(products.numberOfCreates()).isEqualTo(9);
assertThat(products.numberOfUpdates()).isEqualTo(0);
assertThat(products.numberOfDeletes()).isEqualTo(0);
assertThat(products.numberOfReads()).isEqualTo(0);
assertThat(products.numberOfTombstones()).isEqualTo(0);
assertThat(products.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(products.numberOfValueSchemaChanges()).isEqualTo(1);
Collection products_on_hand = store.collection(DATABASE.getDatabaseName(), "products_on_hand");
assertThat(products_on_hand.numberOfCreates()).isEqualTo(9);
assertThat(products_on_hand.numberOfUpdates()).isEqualTo(0);
assertThat(products_on_hand.numberOfDeletes()).isEqualTo(0);
assertThat(products_on_hand.numberOfReads()).isEqualTo(0);
assertThat(products_on_hand.numberOfTombstones()).isEqualTo(0);
assertThat(products_on_hand.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(products_on_hand.numberOfValueSchemaChanges()).isEqualTo(1);
Collection customers = store.collection(DATABASE.getDatabaseName(), "customers");
assertThat(customers.numberOfCreates()).isEqualTo(4);
assertThat(customers.numberOfUpdates()).isEqualTo(0);
assertThat(customers.numberOfDeletes()).isEqualTo(0);
assertThat(customers.numberOfReads()).isEqualTo(0);
assertThat(customers.numberOfTombstones()).isEqualTo(0);
assertThat(customers.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(customers.numberOfValueSchemaChanges()).isEqualTo(1);
Collection orders = store.collection(DATABASE.getDatabaseName(), "orders");
assertThat(orders.numberOfCreates()).isEqualTo(5);
assertThat(orders.numberOfUpdates()).isEqualTo(0);
assertThat(orders.numberOfDeletes()).isEqualTo(0);
assertThat(orders.numberOfReads()).isEqualTo(0);
assertThat(orders.numberOfTombstones()).isEqualTo(0);
assertThat(orders.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(orders.numberOfValueSchemaChanges()).isEqualTo(1);
Collection timetest = store.collection(DATABASE.getDatabaseName(), "dbz_342_timetest");
assertThat(timetest.numberOfCreates()).isEqualTo(1);
assertThat(timetest.numberOfUpdates()).isEqualTo(0);
assertThat(timetest.numberOfDeletes()).isEqualTo(0);
assertThat(timetest.numberOfReads()).isEqualTo(0);
assertThat(timetest.numberOfTombstones()).isEqualTo(0);
assertThat(timetest.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(timetest.numberOfValueSchemaChanges()).isEqualTo(1);
final List<Struct> timerecords = new ArrayList<>();
timetest.forEach(val -> {
timerecords.add(((Struct) val.value()).getStruct("after"));
});
Struct after = timerecords.get(0);
assertThat(after.get("c1")).isEqualTo(toMicroSeconds("PT517H51M04.78S"));
assertThat(after.get("c2")).isEqualTo(toMicroSeconds("-PT13H14M50S"));
assertThat(after.get("c3")).isEqualTo(toMicroSeconds("-PT733H0M0.001S"));
assertThat(after.get("c4")).isEqualTo(toMicroSeconds("-PT1H59M59.001S"));
assertThat(after.get("c5")).isEqualTo(toMicroSeconds("-PT838H59M58.999999S"));
// Make sure the snapshot completed ...
if (completed.await(10, TimeUnit.SECONDS)) {
// completed the snapshot ...
Testing.print("completed the snapshot");
} else {
fail("failed to complete the snapshot within 10 seconds");
}
}
use of io.debezium.data.KeyValueStore.Collection in project debezium by debezium.
the class SnapshotReaderIT method shouldCreateSnapshotOfSingleDatabase.
@Test
public void shouldCreateSnapshotOfSingleDatabase() throws Exception {
config = simpleConfig().build();
context = new MySqlTaskContext(config);
context.start();
reader = new SnapshotReader("snapshot", context);
reader.uponCompletion(completed::countDown);
reader.generateInsertEvents();
// Start the snapshot ...
reader.start();
// Poll for records ...
// Testing.Print.enable();
List<SourceRecord> records = null;
KeyValueStore store = KeyValueStore.createForTopicsBeginningWith(DATABASE.getServerName() + ".");
SchemaChangeHistory schemaChanges = new SchemaChangeHistory(DATABASE.getServerName());
while ((records = reader.poll()) != null) {
records.forEach(record -> {
VerifyRecord.isValid(record);
store.add(record);
schemaChanges.add(record);
});
}
// The last poll should always return null ...
assertThat(records).isNull();
// There should be no schema changes ...
assertThat(schemaChanges.recordCount()).isEqualTo(0);
// Check the records via the store ...
assertThat(store.collectionCount()).isEqualTo(5);
Collection products = store.collection(DATABASE.getDatabaseName(), productsTableName());
assertThat(products.numberOfCreates()).isEqualTo(9);
assertThat(products.numberOfUpdates()).isEqualTo(0);
assertThat(products.numberOfDeletes()).isEqualTo(0);
assertThat(products.numberOfReads()).isEqualTo(0);
assertThat(products.numberOfTombstones()).isEqualTo(0);
assertThat(products.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(products.numberOfValueSchemaChanges()).isEqualTo(1);
Collection products_on_hand = store.collection(DATABASE.getDatabaseName(), "products_on_hand");
assertThat(products_on_hand.numberOfCreates()).isEqualTo(9);
assertThat(products_on_hand.numberOfUpdates()).isEqualTo(0);
assertThat(products_on_hand.numberOfDeletes()).isEqualTo(0);
assertThat(products_on_hand.numberOfReads()).isEqualTo(0);
assertThat(products_on_hand.numberOfTombstones()).isEqualTo(0);
assertThat(products_on_hand.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(products_on_hand.numberOfValueSchemaChanges()).isEqualTo(1);
Collection customers = store.collection(DATABASE.getDatabaseName(), "customers");
assertThat(customers.numberOfCreates()).isEqualTo(4);
assertThat(customers.numberOfUpdates()).isEqualTo(0);
assertThat(customers.numberOfDeletes()).isEqualTo(0);
assertThat(customers.numberOfReads()).isEqualTo(0);
assertThat(customers.numberOfTombstones()).isEqualTo(0);
assertThat(customers.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(customers.numberOfValueSchemaChanges()).isEqualTo(1);
Collection orders = store.collection(DATABASE.getDatabaseName(), "orders");
assertThat(orders.numberOfCreates()).isEqualTo(5);
assertThat(orders.numberOfUpdates()).isEqualTo(0);
assertThat(orders.numberOfDeletes()).isEqualTo(0);
assertThat(orders.numberOfReads()).isEqualTo(0);
assertThat(orders.numberOfTombstones()).isEqualTo(0);
assertThat(orders.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(orders.numberOfValueSchemaChanges()).isEqualTo(1);
Collection timetest = store.collection(DATABASE.getDatabaseName(), "dbz_342_timetest");
assertThat(timetest.numberOfCreates()).isEqualTo(1);
assertThat(timetest.numberOfUpdates()).isEqualTo(0);
assertThat(timetest.numberOfDeletes()).isEqualTo(0);
assertThat(timetest.numberOfReads()).isEqualTo(0);
assertThat(timetest.numberOfTombstones()).isEqualTo(0);
assertThat(timetest.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(timetest.numberOfValueSchemaChanges()).isEqualTo(1);
final List<Struct> timerecords = new ArrayList<>();
timetest.forEach(val -> {
timerecords.add(((Struct) val.value()).getStruct("after"));
});
Struct after = timerecords.get(0);
assertThat(after.get("c1")).isEqualTo(toMicroSeconds("PT517H51M04.78S"));
assertThat(after.get("c2")).isEqualTo(toMicroSeconds("-PT13H14M50S"));
assertThat(after.get("c3")).isEqualTo(toMicroSeconds("-PT733H0M0.001S"));
assertThat(after.get("c4")).isEqualTo(toMicroSeconds("-PT1H59M59.001S"));
assertThat(after.get("c5")).isEqualTo(toMicroSeconds("-PT838H59M58.999999S"));
// Make sure the snapshot completed ...
if (completed.await(10, TimeUnit.SECONDS)) {
// completed the snapshot ...
Testing.print("completed the snapshot");
} else {
fail("failed to complete the snapshot within 10 seconds");
}
}
use of io.debezium.data.KeyValueStore.Collection in project debezium by debezium.
the class BinlogReaderIT method shouldCreateSnapshotOfSingleDatabase.
@Test
public void shouldCreateSnapshotOfSingleDatabase() throws Exception {
config = simpleConfig().build();
context = new MySqlTaskContext(config);
context.start();
// start from beginning
context.source().setBinlogStartPoint("", 0L);
context.initializeHistory();
reader = new BinlogReader("binlog", context);
// Start reading the binlog ...
reader.start();
// Poll for records ...
// Testing.Print.enable();
// only the inserts for our 4 tables in this database and 1 create table
int expected = 9 + 9 + 4 + 5 + 1;
int consumed = consumeAtLeast(expected);
assertThat(consumed).isGreaterThanOrEqualTo(expected);
store.sourceRecords().forEach(System.out::println);
// There should be no schema changes ...
assertThat(schemaChanges.recordCount()).isEqualTo(0);
// Check the records via the store ...
assertThat(store.collectionCount()).isEqualTo(5);
Collection products = store.collection(DATABASE.getDatabaseName(), productsTableName());
assertThat(products.numberOfCreates()).isEqualTo(9);
assertThat(products.numberOfUpdates()).isEqualTo(0);
assertThat(products.numberOfDeletes()).isEqualTo(0);
assertThat(products.numberOfReads()).isEqualTo(0);
assertThat(products.numberOfTombstones()).isEqualTo(0);
assertThat(products.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(products.numberOfValueSchemaChanges()).isEqualTo(1);
Collection products_on_hand = store.collection(DATABASE.getDatabaseName(), "products_on_hand");
assertThat(products_on_hand.numberOfCreates()).isEqualTo(9);
assertThat(products_on_hand.numberOfUpdates()).isEqualTo(0);
assertThat(products_on_hand.numberOfDeletes()).isEqualTo(0);
assertThat(products_on_hand.numberOfReads()).isEqualTo(0);
assertThat(products_on_hand.numberOfTombstones()).isEqualTo(0);
assertThat(products_on_hand.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(products_on_hand.numberOfValueSchemaChanges()).isEqualTo(1);
Collection customers = store.collection(DATABASE.getDatabaseName(), "customers");
assertThat(customers.numberOfCreates()).isEqualTo(4);
assertThat(customers.numberOfUpdates()).isEqualTo(0);
assertThat(customers.numberOfDeletes()).isEqualTo(0);
assertThat(customers.numberOfReads()).isEqualTo(0);
assertThat(customers.numberOfTombstones()).isEqualTo(0);
assertThat(customers.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(customers.numberOfValueSchemaChanges()).isEqualTo(1);
Collection orders = store.collection(DATABASE.getDatabaseName(), "orders");
assertThat(orders.numberOfCreates()).isEqualTo(5);
assertThat(orders.numberOfUpdates()).isEqualTo(0);
assertThat(orders.numberOfDeletes()).isEqualTo(0);
assertThat(orders.numberOfReads()).isEqualTo(0);
assertThat(orders.numberOfTombstones()).isEqualTo(0);
assertThat(orders.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(orders.numberOfValueSchemaChanges()).isEqualTo(1);
}
use of io.debezium.data.KeyValueStore.Collection in project debezium by debezium.
the class SnapshotReaderIT method shouldCreateSnapshotOfSingleDatabaseUsingReadEvents.
@Test
public void shouldCreateSnapshotOfSingleDatabaseUsingReadEvents() throws Exception {
config = simpleConfig().with(MySqlConnectorConfig.DATABASE_WHITELIST, "connector_(.*)_" + DATABASE.getIdentifier()).build();
context = new MySqlTaskContext(config);
context.start();
reader = new SnapshotReader("snapshot", context);
reader.uponCompletion(completed::countDown);
reader.generateReadEvents();
// Start the snapshot ...
reader.start();
// Poll for records ...
// Testing.Print.enable();
List<SourceRecord> records = null;
KeyValueStore store = KeyValueStore.createForTopicsBeginningWith(DATABASE.getServerName() + ".");
SchemaChangeHistory schemaChanges = new SchemaChangeHistory(DATABASE.getServerName());
while ((records = reader.poll()) != null) {
records.forEach(record -> {
VerifyRecord.isValid(record);
store.add(record);
schemaChanges.add(record);
System.out.println(record);
});
}
// The last poll should always return null ...
assertThat(records).isNull();
// There should be no schema changes ...
assertThat(schemaChanges.recordCount()).isEqualTo(0);
// Check the records via the store ...
// 2 databases
assertThat(store.databases()).containsOnly(DATABASE.getDatabaseName(), OTHER_DATABASE.getDatabaseName());
// 2 databases
assertThat(store.collectionCount()).isEqualTo(9);
Collection products = store.collection(DATABASE.getDatabaseName(), productsTableName());
assertThat(products.numberOfCreates()).isEqualTo(0);
assertThat(products.numberOfUpdates()).isEqualTo(0);
assertThat(products.numberOfDeletes()).isEqualTo(0);
assertThat(products.numberOfReads()).isEqualTo(9);
assertThat(products.numberOfTombstones()).isEqualTo(0);
assertThat(products.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(products.numberOfValueSchemaChanges()).isEqualTo(1);
Collection products_on_hand = store.collection(DATABASE.getDatabaseName(), "products_on_hand");
assertThat(products_on_hand.numberOfCreates()).isEqualTo(0);
assertThat(products_on_hand.numberOfUpdates()).isEqualTo(0);
assertThat(products_on_hand.numberOfDeletes()).isEqualTo(0);
assertThat(products_on_hand.numberOfReads()).isEqualTo(9);
assertThat(products_on_hand.numberOfTombstones()).isEqualTo(0);
assertThat(products_on_hand.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(products_on_hand.numberOfValueSchemaChanges()).isEqualTo(1);
Collection customers = store.collection(DATABASE.getDatabaseName(), "customers");
assertThat(customers.numberOfCreates()).isEqualTo(0);
assertThat(customers.numberOfUpdates()).isEqualTo(0);
assertThat(customers.numberOfDeletes()).isEqualTo(0);
assertThat(customers.numberOfReads()).isEqualTo(4);
assertThat(customers.numberOfTombstones()).isEqualTo(0);
assertThat(customers.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(customers.numberOfValueSchemaChanges()).isEqualTo(1);
Collection orders = store.collection(DATABASE.getDatabaseName(), "orders");
assertThat(orders.numberOfCreates()).isEqualTo(0);
assertThat(orders.numberOfUpdates()).isEqualTo(0);
assertThat(orders.numberOfDeletes()).isEqualTo(0);
assertThat(orders.numberOfReads()).isEqualTo(5);
assertThat(orders.numberOfTombstones()).isEqualTo(0);
assertThat(orders.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(orders.numberOfValueSchemaChanges()).isEqualTo(1);
Collection timetest = store.collection(DATABASE.getDatabaseName(), "dbz_342_timetest");
assertThat(timetest.numberOfCreates()).isEqualTo(0);
assertThat(timetest.numberOfUpdates()).isEqualTo(0);
assertThat(timetest.numberOfDeletes()).isEqualTo(0);
assertThat(timetest.numberOfReads()).isEqualTo(1);
assertThat(timetest.numberOfTombstones()).isEqualTo(0);
assertThat(timetest.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(timetest.numberOfValueSchemaChanges()).isEqualTo(1);
final List<Struct> timerecords = new ArrayList<>();
timetest.forEach(val -> {
timerecords.add(((Struct) val.value()).getStruct("after"));
});
Struct after = timerecords.get(0);
assertThat(after.get("c1")).isEqualTo(toMicroSeconds("PT517H51M04.78S"));
assertThat(after.get("c2")).isEqualTo(toMicroSeconds("-PT13H14M50S"));
assertThat(after.get("c3")).isEqualTo(toMicroSeconds("-PT733H0M0.001S"));
assertThat(after.get("c4")).isEqualTo(toMicroSeconds("-PT1H59M59.001S"));
assertThat(after.get("c5")).isEqualTo(toMicroSeconds("-PT838H59M58.999999S"));
// Make sure the snapshot completed ...
if (completed.await(10, TimeUnit.SECONDS)) {
// completed the snapshot ...
Testing.print("completed the snapshot");
} else {
fail("failed to complete the snapshot within 10 seconds");
}
}
use of io.debezium.data.KeyValueStore.Collection in project debezium by debezium.
the class BinlogReaderIT method shouldCreateSnapshotOfSingleDatabaseWithSchemaChanges.
@Test
public void shouldCreateSnapshotOfSingleDatabaseWithSchemaChanges() throws Exception {
config = simpleConfig().with(MySqlConnectorConfig.INCLUDE_SCHEMA_CHANGES, true).build();
context = new MySqlTaskContext(config);
context.start();
// start from beginning
context.source().setBinlogStartPoint("", 0L);
context.initializeHistory();
reader = new BinlogReader("binlog", context);
// Start reading the binlog ...
reader.start();
// Poll for records ...
// Testing.Print.enable();
// 5 tables plus 2 alters
int expectedSchemaChangeCount = 5 + 2;
// only the inserts for our 4 tables in this database, plus
int expected = (9 + 9 + 4 + 5 + 1) + expectedSchemaChangeCount;
// schema changes
int consumed = consumeAtLeast(expected);
assertThat(consumed).isGreaterThanOrEqualTo(expected);
// There should be no schema changes ...
assertThat(schemaChanges.recordCount()).isEqualTo(expectedSchemaChangeCount);
// Check the records via the store ...
assertThat(store.collectionCount()).isEqualTo(5);
Collection products = store.collection(DATABASE.getDatabaseName(), productsTableName());
assertThat(products.numberOfCreates()).isEqualTo(9);
assertThat(products.numberOfUpdates()).isEqualTo(0);
assertThat(products.numberOfDeletes()).isEqualTo(0);
assertThat(products.numberOfReads()).isEqualTo(0);
assertThat(products.numberOfTombstones()).isEqualTo(0);
assertThat(products.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(products.numberOfValueSchemaChanges()).isEqualTo(1);
Collection products_on_hand = store.collection(DATABASE.getDatabaseName(), "products_on_hand");
assertThat(products_on_hand.numberOfCreates()).isEqualTo(9);
assertThat(products_on_hand.numberOfUpdates()).isEqualTo(0);
assertThat(products_on_hand.numberOfDeletes()).isEqualTo(0);
assertThat(products_on_hand.numberOfReads()).isEqualTo(0);
assertThat(products_on_hand.numberOfTombstones()).isEqualTo(0);
assertThat(products_on_hand.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(products_on_hand.numberOfValueSchemaChanges()).isEqualTo(1);
Collection customers = store.collection(DATABASE.getDatabaseName(), "customers");
assertThat(customers.numberOfCreates()).isEqualTo(4);
assertThat(customers.numberOfUpdates()).isEqualTo(0);
assertThat(customers.numberOfDeletes()).isEqualTo(0);
assertThat(customers.numberOfReads()).isEqualTo(0);
assertThat(customers.numberOfTombstones()).isEqualTo(0);
assertThat(customers.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(customers.numberOfValueSchemaChanges()).isEqualTo(1);
Collection orders = store.collection(DATABASE.getDatabaseName(), "orders");
assertThat(orders.numberOfCreates()).isEqualTo(5);
assertThat(orders.numberOfUpdates()).isEqualTo(0);
assertThat(orders.numberOfDeletes()).isEqualTo(0);
assertThat(orders.numberOfReads()).isEqualTo(0);
assertThat(orders.numberOfTombstones()).isEqualTo(0);
assertThat(orders.numberOfKeySchemaChanges()).isEqualTo(1);
assertThat(orders.numberOfValueSchemaChanges()).isEqualTo(1);
}
Aggregations