Examples with SourceDataEntry io.openmessaging.connector.api.data.SourceDataEntry used on opensource projects

Example 1 with SourceDataEntry

use of io.openmessaging.connector.api.data.SourceDataEntry in project rocketmq-externals by apache.

the class ActivemqSourceTask method poll.

@Override
public Collection<SourceDataEntry> poll() {
    List<SourceDataEntry> res = new ArrayList<>();
    try {
        Message message = replicator.getQueue().poll(1000, TimeUnit.MILLISECONDS);
        if (message != null) {
            Object[] payload = new Object[] { config.getDestinationType(), config.getDestinationName(), getMessageContent(message) };
            SourceDataEntry sourceDataEntry = new SourceDataEntry(sourcePartition, null, System.currentTimeMillis(), EntryType.CREATE, null, null, payload);
            res.add(sourceDataEntry);
        }
    } catch (Exception e) {
        log.error("activemq task poll error, current config:" + JSON.toJSONString(config), e);
    }
    return res;
}

Example 2 with SourceDataEntry

use of io.openmessaging.connector.api.data.SourceDataEntry in project rocketmq-externals by apache.

the class ActivemqSourceTaskTest method test.

// @Test
public void test() throws InterruptedException {
    KeyValue kv = new DefaultKeyValue();
    kv.put("activemqUrl", "tcp://112.74.48.251:6166");
    kv.put("destinationType", "queue");
    kv.put("destinationName", "test-queue");
    ActivemqSourceTask task = new ActivemqSourceTask();
    task.start(kv);
    for (int i = 0; i < 20; ) {
        Collection<SourceDataEntry> sourceDataEntry = task.poll();
        i = i + sourceDataEntry.size();
        System.out.println(sourceDataEntry);
    }
    Thread.sleep(20000);
}

Example 3 with SourceDataEntry

use of io.openmessaging.connector.api.data.SourceDataEntry in project rocketmq-externals by apache.

the class ActivemqSourceTaskTest method pollTest.

@Test
public void pollTest() throws Exception {
    ActivemqSourceTask task = new ActivemqSourceTask();
    TextMessage textMessage = new ActiveMQTextMessage();
    textMessage.setText("hello rocketmq");
    Replicator replicatorObject = Mockito.mock(Replicator.class);
    BlockingQueue<Message> queue = new LinkedBlockingQueue<>();
    Mockito.when(replicatorObject.getQueue()).thenReturn(queue);
    Field replicator = ActivemqSourceTask.class.getDeclaredField("replicator");
    replicator.setAccessible(true);
    replicator.set(task, replicatorObject);
    Field config = ActivemqSourceTask.class.getDeclaredField("config");
    config.setAccessible(true);
    config.set(task, new Config());
    queue.put(textMessage);
    Collection<SourceDataEntry> list = task.poll();
    Assert.assertEquals(list.size(), 1);
    list = task.poll();
    Assert.assertEquals(list.size(), 0);
}

Example 4 with SourceDataEntry

use of io.openmessaging.connector.api.data.SourceDataEntry in project rocketmq-externals by apache.

the class BaseJmsSourceTask method poll.

@Override
public Collection<SourceDataEntry> poll() {
    List<SourceDataEntry> res = new ArrayList<>();
    try {
        Message message = replicator.getQueue().poll(1000, TimeUnit.MILLISECONDS);
        if (message != null) {
            Object[] payload = new Object[] { config.getDestinationType(), config.getDestinationName(), getMessageContent(message) };
            SourceDataEntry sourceDataEntry = new SourceDataEntry(sourcePartition, null, System.currentTimeMillis(), EntryType.CREATE, null, null, payload);
            res.add(sourceDataEntry);
        }
    } catch (Exception e) {
        log.error("activemq task poll error, current config:" + JSON.toJSONString(config), e);
    }
    return res;
}

Example 5 with SourceDataEntry

use of io.openmessaging.connector.api.data.SourceDataEntry in project rocketmq-externals by apache.

the class MongoTest method testInitSyncCopy.

@Test
public void testInitSyncCopy() throws NoSuchFieldException, IllegalAccessException, InterruptedException {
    MongoCollection<Document> collection = mongoClient.getDatabase("test").getCollection("person");
    collection.deleteMany(new Document());
    int count = 1000;
    List<String> documents = new ArrayList<>(count);
    for (int i = 0; i < count; i++) {
        Document document = new Document();
        document.put("name", "test" + i);
        document.put("age", i);
        document.put("sex", i % 2 == 0 ? "boy" : "girl");
        collection.insertOne(document);
        documents.add(document.getObjectId("_id").toHexString());
    }
    SourceTaskConfig sourceTaskConfig = new SourceTaskConfig();
    Map<String, List<String>> insterest = new HashMap<>();
    List<String> collections = new ArrayList<>();
    collections.add("*");
    insterest.put("test", collections);
    sourceTaskConfig.setInterestDbAndCollection(JSONObject.toJSONString(insterest));
    ReplicaSetConfig replicaSetConfig = new ReplicaSetConfig("", "test", "localhost");
    ReplicaSetsContext replicaSetsContext = new ReplicaSetsContext(sourceTaskConfig);
    ReplicaSet replicaSet = new ReplicaSet(replicaSetConfig, replicaSetsContext);
    Field running = ReplicaSet.class.getDeclaredField("running");
    running.setAccessible(true);
    running.set(replicaSet, new AtomicBoolean(true));
    InitSync initSync = new InitSync(replicaSetConfig, mongoClient, replicaSetsContext, replicaSet);
    initSync.start();
    int syncCount = 0;
    while (syncCount < count) {
        Collection<SourceDataEntry> sourceDataEntries = replicaSetsContext.poll();
        Assert.assertTrue(sourceDataEntries.size() > 0);
        for (SourceDataEntry sourceDataEntry : sourceDataEntries) {
            ByteBuffer sourcePartition = sourceDataEntry.getSourcePartition();
            Assert.assertEquals("test", new String(sourcePartition.array()));
            ByteBuffer sourcePosition = sourceDataEntry.getSourcePosition();
            Position position = new Position();
            position.setInitSync(true);
            position.setTimeStamp(0);
            position.setInc(0);
            Assert.assertEquals(position, JSONObject.parseObject(new String(sourcePosition.array()), Position.class));
            EntryType entryType = sourceDataEntry.getEntryType();
            Assert.assertEquals(EntryType.CREATE, entryType);
            String queueName = sourceDataEntry.getQueueName();
            Assert.assertEquals("test-person", queueName);
            Schema schema = sourceDataEntry.getSchema();
            Assert.assertTrue(schema.getFields().size() == 2);
            Object[] payload = sourceDataEntry.getPayload();
            Assert.assertTrue(payload.length == 2);
            Assert.assertEquals(payload[0].toString(), "test.person");
            Assert.assertTrue(documents.contains(JSONObject.parseObject(payload[1].toString(), Document.class).get("_id", JSONObject.class).getString("$oid")));
            syncCount++;
        }
    }
    Assert.assertTrue(syncCount == count);
}