Examples with RecordsForCollection io.debezium.connector.mongodb.RecordMakers.RecordsForCollection used on opensource projects

Example 6 with RecordsForCollection

use of io.debezium.connector.mongodb.RecordMakers.RecordsForCollection in project debezium by debezium.

the class Replicator method handleOplogEvent.

/**
 * Handle a single oplog event.
 *
 * @param primaryAddress the address of the primary server from which the event was obtained; may not be null
 * @param event the oplog event; may not be null
 * @return {@code true} if additional events should be processed, or {@code false} if the caller should stop
 *         processing events
 */
protected boolean handleOplogEvent(ServerAddress primaryAddress, Document event) {
    logger.debug("Found event: {}", event);
    String ns = event.getString("ns");
    Document object = event.get("o", Document.class);
    if (object == null) {
        logger.warn("Missing 'o' field in event, so skipping {}", event.toJson());
        return true;
    }
    if (ns == null || ns.isEmpty()) {
        // These are replica set events ...
        String msg = object.getString("msg");
        if ("new primary".equals(msg)) {
            AtomicReference<ServerAddress> address = new AtomicReference<>();
            try {
                primaryClient.executeBlocking("conn", mongoClient -> {
                    ServerAddress currentPrimary = mongoClient.getAddress();
                    address.set(currentPrimary);
                });
            } catch (InterruptedException e) {
                logger.error("Get current primary executeBlocking", e);
            }
            ServerAddress serverAddress = address.get();
            if (serverAddress != null && !serverAddress.equals(primaryAddress)) {
                logger.info("Found new primary event in oplog, so stopping use of {} to continue with new primary", primaryAddress);
                // There is a new primary, so stop using this server and instead use the new primary ...
                return false;
            } else {
                logger.info("Found new primary event in oplog, current {} is new primary. " + "Continue to process oplog event.", primaryAddress);
            }
        }
        // Otherwise, ignore this event ...
        logger.debug("Skipping event with no namespace: {}", event.toJson());
        return true;
    }
    int delimIndex = ns.indexOf('.');
    if (delimIndex > 0) {
        assert (delimIndex + 1) < ns.length();
        String dbName = ns.substring(0, delimIndex);
        String collectionName = ns.substring(delimIndex + 1);
        if ("$cmd".equals(collectionName)) {
            // This is a command on a database ...
            // TODO: Probably want to handle some of these when we track creation/removal of collections
            logger.debug("Skipping database command event: {}", event.toJson());
            return true;
        }
        // Otherwise, it is an event on a document in a collection ...
        if (!databaseFilter.test(dbName)) {
            logger.debug("Skipping the event for database {} based on database.whitelist");
            return true;
        }
        CollectionId collectionId = new CollectionId(rsName, dbName, collectionName);
        if (collectionFilter.test(collectionId)) {
            RecordsForCollection factory = recordMakers.forCollection(collectionId);
            try {
                factory.recordEvent(event, clock.currentTimeInMillis());
            } catch (InterruptedException e) {
                Thread.interrupted();
                return false;
            }
        }
    }
    return true;
}

Example 7 with RecordsForCollection

use of io.debezium.connector.mongodb.RecordMakers.RecordsForCollection in project debezium by debezium.

the class UnwrapFromMongoDbEnvelopeTest method shouldGenerateRecordForUpdateEvent.

@Test
public void shouldGenerateRecordForUpdateEvent() throws InterruptedException {
    BsonTimestamp ts = new BsonTimestamp(1000, 1);
    CollectionId collectionId = new CollectionId("rs0", "dbA", "c1");
    ObjectId objId = new ObjectId();
    Document obj = new Document().append("$set", new Document("name", "Sally"));
    // given
    Document event = new Document().append("o", obj).append("o2", objId).append("ns", "dbA.c1").append("ts", ts).append("h", Long.valueOf(12345678)).append("op", "u");
    RecordsForCollection records = recordMakers.forCollection(collectionId);
    records.recordEvent(event, 1002);
    assertThat(produced.size()).isEqualTo(1);
    SourceRecord record = produced.get(0);
    // when
    SourceRecord transformed = transformation.apply(record);
    Struct key = (Struct) transformed.key();
    Struct value = (Struct) transformed.value();
    // then assert key and its schema
    assertThat(key.schema()).isSameAs(transformed.keySchema());
    assertThat(key.schema().field("id").schema()).isEqualTo(SchemaBuilder.OPTIONAL_STRING_SCHEMA);
    assertThat(key.get("id")).isEqualTo(objId.toString());
    // and then assert value and its schema
    assertThat(value.schema()).isSameAs(transformed.valueSchema());
    assertThat(value.get("name")).isEqualTo("Sally");
    assertThat(value.get("id")).isEqualTo(objId.toString());
    assertThat(value.schema().field("id").schema()).isEqualTo(SchemaBuilder.OPTIONAL_STRING_SCHEMA);
    assertThat(value.schema().field("name").schema()).isEqualTo(SchemaBuilder.OPTIONAL_STRING_SCHEMA);
    assertThat(value.schema().fields()).hasSize(2);
}

Example 8 with RecordsForCollection

use of io.debezium.connector.mongodb.RecordMakers.RecordsForCollection in project debezium by debezium.

the class UnwrapFromMongoDbEnvelopeTest method shouldTransformRecordForInsertEventWithComplexIdType.

@Test
public void shouldTransformRecordForInsertEventWithComplexIdType() throws InterruptedException {
    CollectionId collectionId = new CollectionId("rs0", "dbA", "c1");
    BsonTimestamp ts = new BsonTimestamp(1000, 1);
    Document obj = new Document().append("_id", new Document().append("company", 32).append("dept", "home improvement")).append("name", "Sally");
    // given
    Document event = new Document().append("o", obj).append("ns", "dbA.c1").append("ts", ts).append("h", Long.valueOf(12345678)).append("op", "i");
    RecordsForCollection records = recordMakers.forCollection(collectionId);
    records.recordEvent(event, 1002);
    assertThat(produced.size()).isEqualTo(1);
    SourceRecord record = produced.get(0);
    // when
    SourceRecord transformed = transformation.apply(record);
    Struct key = (Struct) transformed.key();
    Struct value = (Struct) transformed.value();
    // then assert key and its schema
    assertThat(key.schema()).isSameAs(transformed.keySchema());
    assertThat(key.schema().field("id").schema().field("company").schema()).isEqualTo(SchemaBuilder.OPTIONAL_INT32_SCHEMA);
    assertThat(key.schema().field("id").schema().field("dept").schema()).isEqualTo(SchemaBuilder.OPTIONAL_STRING_SCHEMA);
    assertThat(((Struct) key.get("id")).get("company")).isEqualTo(32);
    assertThat(((Struct) key.get("id")).get("dept")).isEqualTo("home improvement");
    // and then assert value and its schema
    assertThat(value.schema()).isSameAs(transformed.valueSchema());
    assertThat(((Struct) value.get("id")).get("company")).isEqualTo(32);
    assertThat(((Struct) value.get("id")).get("dept")).isEqualTo("home improvement");
    assertThat(value.get("name")).isEqualTo("Sally");
    assertThat(value.schema().field("id").schema().field("company").schema()).isEqualTo(SchemaBuilder.OPTIONAL_INT32_SCHEMA);
    assertThat(value.schema().field("id").schema().field("dept").schema()).isEqualTo(SchemaBuilder.OPTIONAL_STRING_SCHEMA);
    assertThat(value.schema().field("name").schema()).isEqualTo(SchemaBuilder.OPTIONAL_STRING_SCHEMA);
    assertThat(value.schema().fields()).hasSize(2);
    transformation.close();
}

Example 9 with RecordsForCollection

use of io.debezium.connector.mongodb.RecordMakers.RecordsForCollection in project debezium by debezium.

the class UnwrapFromMongoDbEnvelopeTest method shouldTransformRecordForInsertEvent.

@Test
public void shouldTransformRecordForInsertEvent() throws InterruptedException {
    CollectionId collectionId = new CollectionId("rs0", "dbA", "c1");
    BsonTimestamp ts = new BsonTimestamp(1000, 1);
    ObjectId objId = new ObjectId();
    Document obj = new Document().append("_id", objId).append("name", "Sally").append("phone", 123L).append("active", true).append("scores", Arrays.asList(1.2, 3.4, 5.6));
    // given
    Document event = new Document().append("o", obj).append("ns", "dbA.c1").append("ts", ts).append("h", Long.valueOf(12345678)).append("op", "i");
    RecordsForCollection records = recordMakers.forCollection(collectionId);
    records.recordEvent(event, 1002);
    assertThat(produced.size()).isEqualTo(1);
    SourceRecord record = produced.get(0);
    // when
    SourceRecord transformed = transformation.apply(record);
    Struct key = (Struct) transformed.key();
    Struct value = (Struct) transformed.value();
    // then assert key and its schema
    assertThat(key.schema()).isSameAs(transformed.keySchema());
    assertThat(key.schema().field("id").schema()).isEqualTo(SchemaBuilder.OPTIONAL_STRING_SCHEMA);
    assertThat(key.get("id")).isEqualTo(objId.toString());
    // and then assert value and its schema
    assertThat(value.schema()).isSameAs(transformed.valueSchema());
    assertThat(value.get("name")).isEqualTo("Sally");
    assertThat(value.get("id")).isEqualTo(objId.toString());
    assertThat(value.get("phone")).isEqualTo(123L);
    assertThat(value.get("active")).isEqualTo(true);
    assertThat(value.get("scores")).isEqualTo(Arrays.asList(1.2, 3.4, 5.6));
    assertThat(value.schema().field("id").schema()).isEqualTo(SchemaBuilder.OPTIONAL_STRING_SCHEMA);
    assertThat(value.schema().field("name").schema()).isEqualTo(SchemaBuilder.OPTIONAL_STRING_SCHEMA);
    assertThat(value.schema().field("phone").schema()).isEqualTo(SchemaBuilder.OPTIONAL_INT64_SCHEMA);
    assertThat(value.schema().field("active").schema()).isEqualTo(SchemaBuilder.OPTIONAL_BOOLEAN_SCHEMA);
    assertThat(value.schema().field("scores").schema()).isEqualTo(SchemaBuilder.array(SchemaBuilder.OPTIONAL_FLOAT64_SCHEMA).build());
    assertThat(value.schema().fields()).hasSize(5);
    transformation.close();
}

Example 10 with RecordsForCollection

use of io.debezium.connector.mongodb.RecordMakers.RecordsForCollection in project debezium by debezium.

the class UnwrapFromMongoDbEnvelopeTest method shouldGenerateRecordForDeleteEventWithoutTombstone.

@Test
@FixFor("DBZ-582")
public void shouldGenerateRecordForDeleteEventWithoutTombstone() throws InterruptedException {
    RecordMakers recordMakers = new RecordMakers(source, topicSelector, produced::add, false);
    BsonTimestamp ts = new BsonTimestamp(1000, 1);
    CollectionId collectionId = new CollectionId("rs0", "dbA", "c1");
    ObjectId objId = new ObjectId();
    Document obj = new Document("_id", objId);
    // given
    Document event = new Document().append("o", obj).append("ns", "dbA.c1").append("ts", ts).append("h", Long.valueOf(12345678)).append("op", "d");
    RecordsForCollection records = recordMakers.forCollection(collectionId);
    records.recordEvent(event, 1002);
    assertThat(produced.size()).isEqualTo(1);
    SourceRecord record = produced.get(0);
    // when
    SourceRecord transformed = transformation.apply(record);
    Struct key = (Struct) transformed.key();
    Struct value = (Struct) transformed.value();
    // then assert key and its schema
    assertThat(key.schema()).isSameAs(transformed.keySchema());
    assertThat(key.schema().field("id").schema()).isEqualTo(SchemaBuilder.OPTIONAL_STRING_SCHEMA);
    assertThat(key.get("id")).isEqualTo(objId.toString());
    assertThat(value).isNull();
}