Examples with Position org.apache.connect.mongo.replicator.Position used on opensource projects

Example 1 with Position

use of org.apache.connect.mongo.replicator.Position in project rocketmq-externals by apache.

the class MongoTest method testInitSyncCopy.

@Test
public void testInitSyncCopy() throws NoSuchFieldException, IllegalAccessException, InterruptedException {
    MongoCollection<Document> collection = mongoClient.getDatabase("test").getCollection("person");
    collection.deleteMany(new Document());
    int count = 1000;
    List<String> documents = new ArrayList<>(count);
    for (int i = 0; i < count; i++) {
        Document document = new Document();
        document.put("name", "test" + i);
        document.put("age", i);
        document.put("sex", i % 2 == 0 ? "boy" : "girl");
        collection.insertOne(document);
        documents.add(document.getObjectId("_id").toHexString());
    }
    SourceTaskConfig sourceTaskConfig = new SourceTaskConfig();
    Map<String, List<String>> insterest = new HashMap<>();
    List<String> collections = new ArrayList<>();
    collections.add("*");
    insterest.put("test", collections);
    sourceTaskConfig.setInterestDbAndCollection(JSONObject.toJSONString(insterest));
    ReplicaSetConfig replicaSetConfig = new ReplicaSetConfig("", "test", "localhost");
    ReplicaSetsContext replicaSetsContext = new ReplicaSetsContext(sourceTaskConfig);
    ReplicaSet replicaSet = new ReplicaSet(replicaSetConfig, replicaSetsContext);
    Field running = ReplicaSet.class.getDeclaredField("running");
    running.setAccessible(true);
    running.set(replicaSet, new AtomicBoolean(true));
    InitSync initSync = new InitSync(replicaSetConfig, mongoClient, replicaSetsContext, replicaSet);
    initSync.start();
    int syncCount = 0;
    while (syncCount < count) {
        Collection<SourceDataEntry> sourceDataEntries = replicaSetsContext.poll();
        Assert.assertTrue(sourceDataEntries.size() > 0);
        for (SourceDataEntry sourceDataEntry : sourceDataEntries) {
            ByteBuffer sourcePartition = sourceDataEntry.getSourcePartition();
            Assert.assertEquals("test", new String(sourcePartition.array()));
            ByteBuffer sourcePosition = sourceDataEntry.getSourcePosition();
            Position position = new Position();
            position.setInitSync(true);
            position.setTimeStamp(0);
            position.setInc(0);
            Assert.assertEquals(position, JSONObject.parseObject(new String(sourcePosition.array()), Position.class));
            EntryType entryType = sourceDataEntry.getEntryType();
            Assert.assertEquals(EntryType.CREATE, entryType);
            String queueName = sourceDataEntry.getQueueName();
            Assert.assertEquals("test-person", queueName);
            Schema schema = sourceDataEntry.getSchema();
            Assert.assertTrue(schema.getFields().size() == 2);
            Object[] payload = sourceDataEntry.getPayload();
            Assert.assertTrue(payload.length == 2);
            Assert.assertEquals(payload[0].toString(), "test.person");
            Assert.assertTrue(documents.contains(JSONObject.parseObject(payload[1].toString(), Document.class).get("_id", JSONObject.class).getString("$oid")));
            syncCount++;
        }
    }
    Assert.assertTrue(syncCount == count);
}

Example 2 with Position

use of org.apache.connect.mongo.replicator.Position in project rocketmq-externals by apache.

the class MongoSourceTask method start.

@Override
public void start(KeyValue config) {
    try {
        sourceTaskConfig = new SourceTaskConfig();
        sourceTaskConfig.load(config);
        replicaSetsContext = new ReplicaSetsContext(sourceTaskConfig);
        replicaSetManager = ReplicaSetManager.create(sourceTaskConfig.getMongoAddr());
        replicaSetManager.getReplicaConfigByName().forEach((replicaSetName, replicaSetConfig) -> {
            ByteBuffer byteBuffer = this.context.positionStorageReader().getPosition(ByteBuffer.wrap(replicaSetName.getBytes()));
            if (byteBuffer != null && byteBuffer.array().length > 0) {
                String positionJson = new String(byteBuffer.array(), StandardCharsets.UTF_8);
                Position position = JSONObject.parseObject(positionJson, Position.class);
                replicaSetConfig.setPosition(position);
            } else {
                Position position = new Position();
                position.setTimeStamp(sourceTaskConfig.getPositionTimeStamp());
                position.setInc(sourceTaskConfig.getPositionInc());
                position.setInitSync(sourceTaskConfig.isDataSync());
                replicaSetConfig.setPosition(position);
            }
            ReplicaSet replicaSet = new ReplicaSet(replicaSetConfig, replicaSetsContext);
            replicaSetsContext.addReplicaSet(replicaSet);
            replicaSet.start();
        });
    } catch (Throwable throwable) {
        logger.error("task start error", throwable);
        stop();
    }
}

Example 3 with Position

use of org.apache.connect.mongo.replicator.Position in project rocketmq-externals by apache.

the class MongoSourceConnectorTest method testPoll.

@Test
public void testPoll() throws Exception {
    LinkedBlockingQueue<SourceDataEntry> entries = new LinkedBlockingQueue<>();
    ReplicaSetsContext context = new ReplicaSetsContext(sourceTaskConfig);
    Field dataEntryQueue = ReplicaSetsContext.class.getDeclaredField("dataEntryQueue");
    dataEntryQueue.setAccessible(true);
    dataEntryQueue.set(context, entries);
    ReplicationEvent event = new ReplicationEvent();
    event.setOperationType(OperationType.INSERT);
    event.setNamespace("test.person");
    event.setTimestamp(new BsonTimestamp(1565609506, 1));
    event.setDocument(new Document("testKey", "testValue"));
    event.setH(324243242L);
    event.setEventData(Optional.ofNullable(new Document("testEventKey", "testEventValue")));
    event.setObjectId(Optional.empty());
    context.publishEvent(event, new ReplicaSetConfig("", "testReplicaName", "localhost:27027"));
    List<SourceDataEntry> sourceDataEntries = (List<SourceDataEntry>) context.poll();
    Assert.assertTrue(sourceDataEntries.size() == 1);
    SourceDataEntry sourceDataEntry = sourceDataEntries.get(0);
    Assert.assertEquals("test-person", sourceDataEntry.getQueueName());
    ByteBuffer sourcePartition = sourceDataEntry.getSourcePartition();
    Assert.assertEquals("testReplicaName", new String(sourcePartition.array()));
    ByteBuffer sourcePosition = sourceDataEntry.getSourcePosition();
    Position position = JSONObject.parseObject(new String(sourcePosition.array()), Position.class);
    Assert.assertEquals(position.getTimeStamp(), 1565609506);
    Assert.assertEquals(position.getInc(), 1);
    Assert.assertEquals(position.isInitSync(), false);
    EntryType entryType = sourceDataEntry.getEntryType();
    Assert.assertEquals(EntryType.CREATE, entryType);
    String queueName = sourceDataEntry.getQueueName();
    Assert.assertEquals("test-person", queueName);
    Schema schema = sourceDataEntry.getSchema();
    Assert.assertTrue(schema.getFields().size() == 6);
    Object[] payload = sourceDataEntry.getPayload();
    Assert.assertTrue(payload.length == 6);
}

Example 4 with Position

use of org.apache.connect.mongo.replicator.Position in project rocketmq-externals by apache.

the class MongoDataEntry method createPosition.

private static String createPosition(ReplicationEvent event, ReplicaSetConfig replicaSetConfig) {
    Position position = new Position();
    BsonTimestamp timestamp = event.getTimestamp();
    position.setInc(timestamp != null ? timestamp.getInc() : 0);
    position.setTimeStamp(timestamp != null ? timestamp.getTime() : 0);
    position.setInitSync(event.getOperationType().equals(OperationType.CREATED) ? true : false);
    return JSONObject.toJSONString(position);
}