Examples with InitSync org.apache.connect.mongo.initsync.InitSync used on opensource projects

Example 1 with InitSync

use of org.apache.connect.mongo.initsync.InitSync in project rocketmq-externals by apache.

the class ReplicatorTask method run.

@Override
public void run() {
    BsonTimestamp firstAvailablePosition = findFirstOplogPosition();
    Position userConfigOrRuntimePosition = replicaSetConfig.getPosition();
    boolean needDataSync = !userConfigOrRuntimePosition.isValid() || userConfigOrRuntimePosition.isInitSync() || // userConfigOrRuntimePosition.position < firstAvailablePosition maybe lost some operations
    userConfigOrRuntimePosition.converBsonTimeStamp().compareTo(firstAvailablePosition) < 0;
    if (needDataSync) {
        recordLastOplogPosition();
        InitSync initSync = new InitSync(replicaSetConfig, mongoClient, replicaSetsContext, replicaSet);
        initSync.start();
    }
    if (!replicaSet.isRuning() || replicaSetsContext.isInitSyncAbort()) {
        return;
    }
    MongoDatabase localDataBase = mongoClient.getDatabase(Constants.MONGO_LOCAL_DATABASE);
    FindIterable<Document> iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find(Filters.gt("ts", replicaSetConfig.getPosition().converBsonTimeStamp()));
    MongoCursor<Document> cursor = iterable.sort(new Document("$natural", 1)).noCursorTimeout(true).cursorType(CursorType.TailableAwait).batchSize(200).iterator();
    while (replicaSet.isRuning()) {
        try {
            executorCursor(cursor);
        } catch (Exception e) {
            logger.error("replicaSet:{} shutdown.....", replicaSetConfig, e);
        } finally {
            if (cursor != null) {
                cursor.close();
            }
            replicaSet.shutdown();
        }
    }
    logger.info("replicaSet:{}, already shutdown, replicaTask end of life cycle", replicaSetConfig);
}

Example 2 with InitSync

use of org.apache.connect.mongo.initsync.InitSync in project rocketmq-externals by apache.

the class MongoTest method testInitSyncCopy.

@Test
public void testInitSyncCopy() throws NoSuchFieldException, IllegalAccessException, InterruptedException {
    MongoCollection<Document> collection = mongoClient.getDatabase("test").getCollection("person");
    collection.deleteMany(new Document());
    int count = 1000;
    List<String> documents = new ArrayList<>(count);
    for (int i = 0; i < count; i++) {
        Document document = new Document();
        document.put("name", "test" + i);
        document.put("age", i);
        document.put("sex", i % 2 == 0 ? "boy" : "girl");
        collection.insertOne(document);
        documents.add(document.getObjectId("_id").toHexString());
    }
    SourceTaskConfig sourceTaskConfig = new SourceTaskConfig();
    Map<String, List<String>> insterest = new HashMap<>();
    List<String> collections = new ArrayList<>();
    collections.add("*");
    insterest.put("test", collections);
    sourceTaskConfig.setInterestDbAndCollection(JSONObject.toJSONString(insterest));
    ReplicaSetConfig replicaSetConfig = new ReplicaSetConfig("", "test", "localhost");
    ReplicaSetsContext replicaSetsContext = new ReplicaSetsContext(sourceTaskConfig);
    ReplicaSet replicaSet = new ReplicaSet(replicaSetConfig, replicaSetsContext);
    Field running = ReplicaSet.class.getDeclaredField("running");
    running.setAccessible(true);
    running.set(replicaSet, new AtomicBoolean(true));
    InitSync initSync = new InitSync(replicaSetConfig, mongoClient, replicaSetsContext, replicaSet);
    initSync.start();
    int syncCount = 0;
    while (syncCount < count) {
        Collection<SourceDataEntry> sourceDataEntries = replicaSetsContext.poll();
        Assert.assertTrue(sourceDataEntries.size() > 0);
        for (SourceDataEntry sourceDataEntry : sourceDataEntries) {
            ByteBuffer sourcePartition = sourceDataEntry.getSourcePartition();
            Assert.assertEquals("test", new String(sourcePartition.array()));
            ByteBuffer sourcePosition = sourceDataEntry.getSourcePosition();
            Position position = new Position();
            position.setInitSync(true);
            position.setTimeStamp(0);
            position.setInc(0);
            Assert.assertEquals(position, JSONObject.parseObject(new String(sourcePosition.array()), Position.class));
            EntryType entryType = sourceDataEntry.getEntryType();
            Assert.assertEquals(EntryType.CREATE, entryType);
            String queueName = sourceDataEntry.getQueueName();
            Assert.assertEquals("test-person", queueName);
            Schema schema = sourceDataEntry.getSchema();
            Assert.assertTrue(schema.getFields().size() == 2);
            Object[] payload = sourceDataEntry.getPayload();
            Assert.assertTrue(payload.length == 2);
            Assert.assertEquals(payload[0].toString(), "test.person");
            Assert.assertTrue(documents.contains(JSONObject.parseObject(payload[1].toString(), Document.class).get("_id", JSONObject.class).getString("$oid")));
            syncCount++;
        }
    }
    Assert.assertTrue(syncCount == count);
}