Examples with RecordsForCollection io.debezium.connector.mongodb.RecordMakers.RecordsForCollection used on opensource projects

Example 1 with RecordsForCollection

use of io.debezium.connector.mongodb.RecordMakers.RecordsForCollection in project debezium by debezium.

the class UnwrapFromMongoDbEnvelopeTest method shouldGenerateRecordForDeleteEvent.

@Test
public void shouldGenerateRecordForDeleteEvent() throws InterruptedException {
    BsonTimestamp ts = new BsonTimestamp(1000, 1);
    CollectionId collectionId = new CollectionId("rs0", "dbA", "c1");
    ObjectId objId = new ObjectId();
    Document obj = new Document("_id", objId);
    // given
    Document event = new Document().append("o", obj).append("ns", "dbA.c1").append("ts", ts).append("h", Long.valueOf(12345678)).append("op", "d");
    RecordsForCollection records = recordMakers.forCollection(collectionId);
    records.recordEvent(event, 1002);
    assertThat(produced.size()).isEqualTo(2);
    SourceRecord record = produced.get(0);
    // when
    SourceRecord transformed = transformation.apply(record);
    Struct key = (Struct) transformed.key();
    Struct value = (Struct) transformed.value();
    // then assert key and its schema
    assertThat(key.schema()).isSameAs(transformed.keySchema());
    assertThat(key.schema().field("id").schema()).isEqualTo(SchemaBuilder.OPTIONAL_STRING_SCHEMA);
    assertThat(key.get("id")).isEqualTo(objId.toString());
    assertThat(value).isNull();
}

Example 2 with RecordsForCollection

use of io.debezium.connector.mongodb.RecordMakers.RecordsForCollection in project debezium by debezium.

the class RecordMakersTest method shouldGenerateRecordForDeleteEventWithoutTombstone.

@Test
@FixFor("DBZ-582")
public void shouldGenerateRecordForDeleteEventWithoutTombstone() throws InterruptedException {
    RecordMakers recordMakers = new RecordMakers(source, topicSelector, produced::add, false);
    BsonTimestamp ts = new BsonTimestamp(1000, 1);
    CollectionId collectionId = new CollectionId("rs0", "dbA", "c1");
    ObjectId objId = new ObjectId();
    Document obj = new Document("_id", objId);
    Document event = new Document().append("o", obj).append("ns", "dbA.c1").append("ts", ts).append("h", new Long(12345678)).append("op", "d");
    RecordsForCollection records = recordMakers.forCollection(collectionId);
    records.recordEvent(event, 1002);
    assertThat(produced.size()).isEqualTo(1);
    SourceRecord record = produced.get(0);
    Struct key = (Struct) record.key();
    Struct value = (Struct) record.value();
    assertThat(key.schema()).isSameAs(record.keySchema());
    assertThat(key.get("id")).isEqualTo(JSONSerializers.getStrict().serialize(objId));
    assertThat(value.schema()).isSameAs(record.valueSchema());
    assertThat(value.getString(FieldName.AFTER)).isNull();
    assertThat(value.getString("patch")).isNull();
    assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.DELETE.code());
    assertThat(value.getInt64(FieldName.TIMESTAMP)).isEqualTo(1002L);
    Struct actualSource = value.getStruct(FieldName.SOURCE);
    Struct expectedSource = source.lastOffsetStruct("rs0", collectionId);
    assertThat(actualSource).isEqualTo(expectedSource);
}

Example 3 with RecordsForCollection

use of io.debezium.connector.mongodb.RecordMakers.RecordsForCollection in project debezium by debezium.

the class RecordMakersTest method shouldGenerateRecordsWithCorrectlySerializedId.

@Test
public void shouldGenerateRecordsWithCorrectlySerializedId() throws InterruptedException {
    CollectionId collectionId = new CollectionId("rs0", "dbA", "c1");
    BsonTimestamp ts = new BsonTimestamp(1000, 1);
    // long
    Document obj = new Document().append("_id", Long.valueOf(Integer.MAX_VALUE) + 10).append("name", "Sally");
    Document event = new Document().append("o", obj).append("ns", "dbA.c1").append("ts", ts).append("h", Long.valueOf(12345678)).append("op", "i");
    RecordsForCollection records = recordMakers.forCollection(collectionId);
    records.recordEvent(event, 1002);
    // String
    obj = new Document().append("_id", "123").append("name", "Sally");
    event = new Document().append("o", obj).append("ns", "dbA.c1").append("ts", ts).append("h", Long.valueOf(12345678)).append("op", "i");
    records = recordMakers.forCollection(collectionId);
    records.recordEvent(event, 1003);
    // Complex key type
    obj = new Document().append("_id", new Document().append("company", 32).append("dept", "home improvement")).append("name", "Sally");
    event = new Document().append("o", obj).append("ns", "dbA.c1").append("ts", ts).append("h", Long.valueOf(12345678)).append("op", "i");
    records = recordMakers.forCollection(collectionId);
    records.recordEvent(event, 1004);
    // date
    Calendar cal = Calendar.getInstance();
    cal.set(2017, 9, 19);
    obj = new Document().append("_id", cal.getTime()).append("name", "Sally");
    event = new Document().append("o", obj).append("ns", "dbA.c1").append("ts", ts).append("h", Long.valueOf(12345678)).append("op", "i");
    records = recordMakers.forCollection(collectionId);
    records.recordEvent(event, 1005);
    // Decimal128
    obj = new Document().append("_id", new Decimal128(new BigDecimal("123.45678"))).append("name", "Sally");
    event = new Document().append("o", obj).append("ns", "dbA.c1").append("ts", ts).append("h", Long.valueOf(12345678)).append("op", "i");
    records = recordMakers.forCollection(collectionId);
    records.recordEvent(event, 1004);
    assertThat(produced.size()).isEqualTo(5);
    SourceRecord record = produced.get(0);
    Struct key = (Struct) record.key();
    assertThat(key.get("id")).isEqualTo("2147483657");
    record = produced.get(1);
    key = (Struct) record.key();
    assertThat(key.get("id")).isEqualTo("\"123\"");
    record = produced.get(2);
    key = (Struct) record.key();
    assertThat(key.get("id")).isEqualTo("{ \"company\" : 32 , \"dept\" : \"home improvement\"}");
    record = produced.get(3);
    key = (Struct) record.key();
    // that's actually not what https://docs.mongodb.com/manual/reference/mongodb-extended-json/#date suggests;
    // seems JsonSerializers is not fully compliant with that description
    assertThat(key.get("id")).isEqualTo("{ \"$date\" : " + cal.getTime().getTime() + "}");
    record = produced.get(4);
    key = (Struct) record.key();
    assertThat(key.get("id")).isEqualTo("{ \"$numberDecimal\" : \"123.45678\"}");
}

Example 4 with RecordsForCollection

use of io.debezium.connector.mongodb.RecordMakers.RecordsForCollection in project debezium by debezium.

the class RecordMakersTest method shouldGenerateRecordForInsertEvent.

@Test
public void shouldGenerateRecordForInsertEvent() throws InterruptedException {
    CollectionId collectionId = new CollectionId("rs0", "dbA", "c1");
    BsonTimestamp ts = new BsonTimestamp(1000, 1);
    ObjectId objId = new ObjectId();
    Document obj = new Document().append("_id", objId).append("name", "Sally");
    Document event = new Document().append("o", obj).append("ns", "dbA.c1").append("ts", ts).append("h", Long.valueOf(12345678)).append("op", "i");
    RecordsForCollection records = recordMakers.forCollection(collectionId);
    records.recordEvent(event, 1002);
    assertThat(produced.size()).isEqualTo(1);
    SourceRecord record = produced.get(0);
    Struct key = (Struct) record.key();
    Struct value = (Struct) record.value();
    assertThat(key.schema()).isSameAs(record.keySchema());
    assertThat(key.get("id")).isEqualTo("{ \"$oid\" : \"" + objId + "\"}");
    assertThat(value.schema()).isSameAs(record.valueSchema());
    // assertThat(value.getString(FieldName.BEFORE)).isNull();
    assertThat(value.getString(FieldName.AFTER)).isEqualTo(obj.toJson(WRITER_SETTINGS));
    assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.CREATE.code());
    assertThat(value.getInt64(FieldName.TIMESTAMP)).isEqualTo(1002L);
    Struct actualSource = value.getStruct(FieldName.SOURCE);
    Struct expectedSource = source.lastOffsetStruct("rs0", collectionId);
    assertThat(actualSource).isEqualTo(expectedSource);
}

Example 5 with RecordsForCollection

use of io.debezium.connector.mongodb.RecordMakers.RecordsForCollection in project debezium by debezium.

the class Replicator method copyCollection.

/**
 * Copy the collection, sending to the recorder a record for each document.
 *
 * @param primary the connection to the replica set's primary node; may not be null
 * @param collectionId the identifier of the collection to be copied; may not be null
 * @param timestamp the timestamp in milliseconds at which the copy operation was started
 * @return number of documents that were copied
 * @throws InterruptedException if the thread was interrupted while the copy operation was running
 */
protected long copyCollection(MongoClient primary, CollectionId collectionId, long timestamp) throws InterruptedException {
    RecordsForCollection factory = recordMakers.forCollection(collectionId);
    MongoDatabase db = primary.getDatabase(collectionId.dbName());
    MongoCollection<Document> docCollection = db.getCollection(collectionId.name());
    long counter = 0;
    try (MongoCursor<Document> cursor = docCollection.find().iterator()) {
        while (cursor.hasNext()) {
            Document doc = cursor.next();
            logger.trace("Found existing doc in {}: {}", collectionId, doc);
            counter += factory.recordObject(collectionId, doc, timestamp);
        }
    }
    return counter;
}