Examples with ReplicaSet org.apache.connect.mongo.replicator.ReplicaSet used on opensource projects

Example 1 with ReplicaSet

use of org.apache.connect.mongo.replicator.ReplicaSet in project rocketmq-externals by apache.

the class MongoTest method testInitSyncCopy.

@Test
public void testInitSyncCopy() throws NoSuchFieldException, IllegalAccessException, InterruptedException {
    MongoCollection<Document> collection = mongoClient.getDatabase("test").getCollection("person");
    collection.deleteMany(new Document());
    int count = 1000;
    List<String> documents = new ArrayList<>(count);
    for (int i = 0; i < count; i++) {
        Document document = new Document();
        document.put("name", "test" + i);
        document.put("age", i);
        document.put("sex", i % 2 == 0 ? "boy" : "girl");
        collection.insertOne(document);
        documents.add(document.getObjectId("_id").toHexString());
    }
    SourceTaskConfig sourceTaskConfig = new SourceTaskConfig();
    Map<String, List<String>> insterest = new HashMap<>();
    List<String> collections = new ArrayList<>();
    collections.add("*");
    insterest.put("test", collections);
    sourceTaskConfig.setInterestDbAndCollection(JSONObject.toJSONString(insterest));
    ReplicaSetConfig replicaSetConfig = new ReplicaSetConfig("", "test", "localhost");
    ReplicaSetsContext replicaSetsContext = new ReplicaSetsContext(sourceTaskConfig);
    ReplicaSet replicaSet = new ReplicaSet(replicaSetConfig, replicaSetsContext);
    Field running = ReplicaSet.class.getDeclaredField("running");
    running.setAccessible(true);
    running.set(replicaSet, new AtomicBoolean(true));
    InitSync initSync = new InitSync(replicaSetConfig, mongoClient, replicaSetsContext, replicaSet);
    initSync.start();
    int syncCount = 0;
    while (syncCount < count) {
        Collection<SourceDataEntry> sourceDataEntries = replicaSetsContext.poll();
        Assert.assertTrue(sourceDataEntries.size() > 0);
        for (SourceDataEntry sourceDataEntry : sourceDataEntries) {
            ByteBuffer sourcePartition = sourceDataEntry.getSourcePartition();
            Assert.assertEquals("test", new String(sourcePartition.array()));
            ByteBuffer sourcePosition = sourceDataEntry.getSourcePosition();
            Position position = new Position();
            position.setInitSync(true);
            position.setTimeStamp(0);
            position.setInc(0);
            Assert.assertEquals(position, JSONObject.parseObject(new String(sourcePosition.array()), Position.class));
            EntryType entryType = sourceDataEntry.getEntryType();
            Assert.assertEquals(EntryType.CREATE, entryType);
            String queueName = sourceDataEntry.getQueueName();
            Assert.assertEquals("test-person", queueName);
            Schema schema = sourceDataEntry.getSchema();
            Assert.assertTrue(schema.getFields().size() == 2);
            Object[] payload = sourceDataEntry.getPayload();
            Assert.assertTrue(payload.length == 2);
            Assert.assertEquals(payload[0].toString(), "test.person");
            Assert.assertTrue(documents.contains(JSONObject.parseObject(payload[1].toString(), Document.class).get("_id", JSONObject.class).getString("$oid")));
            syncCount++;
        }
    }
    Assert.assertTrue(syncCount == count);
}

Example 2 with ReplicaSet

use of org.apache.connect.mongo.replicator.ReplicaSet in project rocketmq-externals by apache.

the class ReplicaSetTest method before.

@Before
public void before() {
    this.sourceTaskConfig = new SourceTaskConfig();
    this.replicaSetConfig = new ReplicaSetConfig("shardName1", "", "127.0.0.1:27027");
    this.replicaSetsContext = new ReplicaSetsContext(sourceTaskConfig);
    this.replicaSet = new ReplicaSet(replicaSetConfig, replicaSetsContext);
}

Example 3 with ReplicaSet

use of org.apache.connect.mongo.replicator.ReplicaSet in project rocketmq-externals by apache.

the class MongoSourceTaskTest method testEmptyContextStart.

@Test
public void testEmptyContextStart() throws NoSuchFieldException, IllegalAccessException {
    MongoSourceTask mongoSourceTask = new MongoSourceTask();
    DefaultKeyValue defaultKeyValue = new DefaultKeyValue();
    defaultKeyValue.put("mongoAddr", "test/127.0.0.1:27027");
    defaultKeyValue.put("positionTimeStamp", "11111111");
    defaultKeyValue.put("positionInc", "111");
    defaultKeyValue.put("serverSelectionTimeoutMS", "10");
    defaultKeyValue.put("dataSync", "true");
    Field context = SourceTask.class.getDeclaredField("context");
    context.setAccessible(true);
    context.set(mongoSourceTask, emptyTaskContext());
    mongoSourceTask.start(defaultKeyValue);
    Field replicaSetsContext = MongoSourceTask.class.getDeclaredField("replicaSetsContext");
    replicaSetsContext.setAccessible(true);
    ReplicaSetsContext setsContext = (ReplicaSetsContext) replicaSetsContext.get(mongoSourceTask);
    Field replicaSets = ReplicaSetsContext.class.getDeclaredField("replicaSets");
    replicaSets.setAccessible(true);
    List<ReplicaSet> replicaSetList = (List<ReplicaSet>) replicaSets.get(setsContext);
    Assert.assertTrue(replicaSetList.size() == 1);
    ReplicaSet replicaSet = replicaSetList.get(0);
    Field replicaSetConfig = ReplicaSet.class.getDeclaredField("replicaSetConfig");
    replicaSetConfig.setAccessible(true);
    ReplicaSetConfig replicaSetConfig1 = (ReplicaSetConfig) replicaSetConfig.get(replicaSet);
    Assert.assertTrue(StringUtils.equals(replicaSetConfig1.getReplicaSetName(), "test"));
    Assert.assertTrue(StringUtils.equals(replicaSetConfig1.getHost(), "127.0.0.1:27027"));
    Assert.assertTrue(replicaSetConfig1.getPosition().getTimeStamp() == 11111111);
    Assert.assertTrue(replicaSetConfig1.getPosition().getInc() == 111);
    Assert.assertTrue(replicaSetConfig1.getPosition().isInitSync());
}

Example 4 with ReplicaSet

use of org.apache.connect.mongo.replicator.ReplicaSet in project rocketmq-externals by apache.

the class MongoSourceTask method start.

@Override
public void start(KeyValue config) {
    try {
        sourceTaskConfig = new SourceTaskConfig();
        sourceTaskConfig.load(config);
        replicaSetsContext = new ReplicaSetsContext(sourceTaskConfig);
        replicaSetManager = ReplicaSetManager.create(sourceTaskConfig.getMongoAddr());
        replicaSetManager.getReplicaConfigByName().forEach((replicaSetName, replicaSetConfig) -> {
            ByteBuffer byteBuffer = this.context.positionStorageReader().getPosition(ByteBuffer.wrap(replicaSetName.getBytes()));
            if (byteBuffer != null && byteBuffer.array().length > 0) {
                String positionJson = new String(byteBuffer.array(), StandardCharsets.UTF_8);
                Position position = JSONObject.parseObject(positionJson, Position.class);
                replicaSetConfig.setPosition(position);
            } else {
                Position position = new Position();
                position.setTimeStamp(sourceTaskConfig.getPositionTimeStamp());
                position.setInc(sourceTaskConfig.getPositionInc());
                position.setInitSync(sourceTaskConfig.isDataSync());
                replicaSetConfig.setPosition(position);
            }
            ReplicaSet replicaSet = new ReplicaSet(replicaSetConfig, replicaSetsContext);
            replicaSetsContext.addReplicaSet(replicaSet);
            replicaSet.start();
        });
    } catch (Throwable throwable) {
        logger.error("task start error", throwable);
        stop();
    }
}

Example 5 with ReplicaSet

use of org.apache.connect.mongo.replicator.ReplicaSet in project rocketmq-externals by apache.

the class MongoSourceTaskTest method testContextStart.

@Test
public void testContextStart() throws NoSuchFieldException, IllegalAccessException {
    MongoSourceTask mongoSourceTask = new MongoSourceTask();
    DefaultKeyValue defaultKeyValue = new DefaultKeyValue();
    defaultKeyValue.put("mongoAddr", "test/127.0.0.1:27027");
    defaultKeyValue.put("serverSelectionTimeoutMS", "10");
    Field context = SourceTask.class.getDeclaredField("context");
    context.setAccessible(true);
    context.set(mongoSourceTask, TaskContext());
    mongoSourceTask.start(defaultKeyValue);
    Field replicaSetsContext = MongoSourceTask.class.getDeclaredField("replicaSetsContext");
    replicaSetsContext.setAccessible(true);
    ReplicaSetsContext setsContext = (ReplicaSetsContext) replicaSetsContext.get(mongoSourceTask);
    Field replicaSets = ReplicaSetsContext.class.getDeclaredField("replicaSets");
    replicaSets.setAccessible(true);
    List<ReplicaSet> replicaSetList = (List<ReplicaSet>) replicaSets.get(setsContext);
    Assert.assertTrue(replicaSetList.size() == 1);
    ReplicaSet replicaSet = replicaSetList.get(0);
    Field replicaSetConfig = ReplicaSet.class.getDeclaredField("replicaSetConfig");
    replicaSetConfig.setAccessible(true);
    ReplicaSetConfig replicaSetConfig1 = (ReplicaSetConfig) replicaSetConfig.get(replicaSet);
    Assert.assertTrue(StringUtils.equals(replicaSetConfig1.getReplicaSetName(), "test"));
    Assert.assertTrue(StringUtils.equals(replicaSetConfig1.getHost(), "127.0.0.1:27027"));
    Assert.assertTrue(replicaSetConfig1.getPosition().getTimeStamp() == 22222222);
    Assert.assertTrue(replicaSetConfig1.getPosition().getInc() == 222);
    Assert.assertTrue(!replicaSetConfig1.getPosition().isInitSync());
}