Search in sources :

Example 1 with KeyValue

use of io.openmessaging.KeyValue in project rocketmq-externals by apache.

the class ActivemqSourceTaskTest method test.

// @Test
public void test() throws InterruptedException {
    KeyValue kv = new DefaultKeyValue();
    kv.put("activemqUrl", "tcp://112.74.48.251:6166");
    kv.put("destinationType", "queue");
    kv.put("destinationName", "test-queue");
    ActivemqSourceTask task = new ActivemqSourceTask();
    task.start(kv);
    for (int i = 0; i < 20; ) {
        Collection<SourceDataEntry> sourceDataEntry = task.poll();
        i = i + sourceDataEntry.size();
        System.out.println(sourceDataEntry);
    }
    Thread.sleep(20000);
}
Also used : DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) KeyValue(io.openmessaging.KeyValue) DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) SourceDataEntry(io.openmessaging.connector.api.data.SourceDataEntry)

Example 2 with KeyValue

use of io.openmessaging.KeyValue in project rocketmq-externals by apache.

the class DivideTaskByTopic method divideSinkTaskByTopic.

private List<KeyValue> divideSinkTaskByTopic(DbConnectorConfig dbConnectorConfig, TaskDivideConfig tdc) {
    List<KeyValue> config = new ArrayList<KeyValue>();
    int parallelism = tdc.getTaskParallelism();
    int id = -1;
    Set<String> topicRouteSet = ((SinkDbConnectorConfig) dbConnectorConfig).getWhiteTopics();
    Map<Integer, StringBuilder> taskTopicList = new HashMap<>();
    for (String topicName : topicRouteSet) {
        int ind = ++id % parallelism;
        if (!taskTopicList.containsKey(ind)) {
            taskTopicList.put(ind, new StringBuilder(topicName));
        } else {
            taskTopicList.get(ind).append(",").append(topicName);
        }
    }
    for (int i = 0; i < parallelism; i++) {
        KeyValue keyValue = new DefaultKeyValue();
        keyValue.put(Config.CONN_DB_IP, tdc.getDbUrl());
        keyValue.put(Config.CONN_DB_PORT, tdc.getDbPort());
        keyValue.put(Config.CONN_DB_USERNAME, tdc.getDbUserName());
        keyValue.put(Config.CONN_DB_PASSWORD, tdc.getDbPassword());
        keyValue.put(Config.CONN_DB_DATACENTER, tdc.getLocalDataCenter());
        keyValue.put(Config.CONN_TOPIC_NAMES, taskTopicList.get(i).toString());
        keyValue.put(Config.CONN_DATA_TYPE, tdc.getDataType());
        keyValue.put(Config.CONN_SOURCE_RECORD_CONVERTER, tdc.getSrcRecordConverter());
        keyValue.put(Config.CONN_DB_MODE, tdc.getMode());
        config.add(keyValue);
    }
    return config;
}
Also used : DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) KeyValue(io.openmessaging.KeyValue)

Example 3 with KeyValue

use of io.openmessaging.KeyValue in project rocketmq-externals by apache.

the class DivideTaskByQueue method divideSinkTaskByQueue.

public List<KeyValue> divideSinkTaskByQueue(DbConnectorConfig dbConnectorConfig, TaskDivideConfig tdc) {
    List<KeyValue> config = new ArrayList<KeyValue>();
    int parallelism = tdc.getTaskParallelism();
    Map<Integer, List<TaskTopicInfo>> queueTopicList = new HashMap<Integer, List<TaskTopicInfo>>();
    Map<String, Set<TaskTopicInfo>> topicRouteMap = ((SinkDbConnectorConfig) dbConnectorConfig).getTopicRouteMap();
    int id = -1;
    for (String t : topicRouteMap.keySet()) {
        for (TaskTopicInfo taskTopicInfo : topicRouteMap.get(t)) {
            int ind = ++id % parallelism;
            if (!queueTopicList.containsKey(ind)) {
                queueTopicList.put(ind, new ArrayList<TaskTopicInfo>());
            }
            queueTopicList.get(ind).add(taskTopicInfo);
        }
    }
    for (int i = 0; i < parallelism; i++) {
        KeyValue keyValue = new DefaultKeyValue();
        keyValue.put(Config.CONN_DB_IP, tdc.getDbUrl());
        keyValue.put(Config.CONN_DB_PORT, tdc.getDbPort());
        keyValue.put(Config.CONN_DB_USERNAME, tdc.getDbUserName());
        keyValue.put(Config.CONN_DB_PASSWORD, tdc.getDbPassword());
        keyValue.put(Config.CONN_TOPIC_NAMES, JSONObject.toJSONString(queueTopicList.get(i)));
        keyValue.put(Config.CONN_DATA_TYPE, tdc.getDataType());
        keyValue.put(Config.CONN_SOURCE_RECORD_CONVERTER, tdc.getSrcRecordConverter());
        keyValue.put(Config.CONN_DB_MODE, tdc.getMode());
        config.add(keyValue);
    }
    return config;
}
Also used : DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) KeyValue(io.openmessaging.KeyValue) Set(java.util.Set) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) List(java.util.List) ArrayList(java.util.ArrayList)

Example 4 with KeyValue

use of io.openmessaging.KeyValue in project rocketmq-externals by apache.

the class BaseJmsSourceConnectorTest method taskConfigsTest.

@Test
public void taskConfigsTest() {
    assertEquals(connector.taskConfigs().get(0), null);
    KeyValue keyValue = new DefaultKeyValue();
    for (String requestKey : Config.REQUEST_CONFIG) {
        keyValue.put(requestKey, requestKey);
    }
    connector.verifyAndSetConfig(keyValue);
    assertEquals(connector.taskConfigs().get(0), keyValue);
}
Also used : DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) KeyValue(io.openmessaging.KeyValue) Test(org.junit.Test)

Example 5 with KeyValue

use of io.openmessaging.KeyValue in project rocketmq-externals by apache.

the class DivideTaskByTopic method divideSinkTaskByTopic.

private List<KeyValue> divideSinkTaskByTopic(DbConnectorConfig dbConnectorConfig, TaskDivideConfig tdc) {
    List<KeyValue> config = new ArrayList<KeyValue>();
    int parallelism = tdc.getTaskParallelism();
    int id = -1;
    Set<String> topicRouteSet = ((SinkDbConnectorConfig) dbConnectorConfig).getWhiteTopics();
    Map<Integer, StringBuilder> taskTopicList = new HashMap<>();
    for (String topicName : topicRouteSet) {
        int ind = ++id % parallelism;
        if (!taskTopicList.containsKey(ind)) {
            taskTopicList.put(ind, new StringBuilder(topicName));
        } else {
            taskTopicList.get(ind).append(",").append(topicName);
        }
    }
    for (int i = 0; i < parallelism; i++) {
        KeyValue keyValue = new DefaultKeyValue();
        keyValue.put(Config.CONN_DB_IP, tdc.getDbUrl());
        keyValue.put(Config.CONN_DB_PORT, tdc.getDbPort());
        keyValue.put(Config.CONN_DB_USERNAME, tdc.getDbUserName());
        keyValue.put(Config.CONN_DB_PASSWORD, tdc.getDbPassword());
        keyValue.put(Config.CONN_TOPIC_NAMES, taskTopicList.get(i).toString());
        keyValue.put(Config.CONN_DATA_TYPE, tdc.getDataType());
        keyValue.put(Config.CONN_SOURCE_RECORD_CONVERTER, tdc.getSrcRecordConverter());
        keyValue.put(Config.CONN_DB_MODE, tdc.getMode());
        config.add(keyValue);
    }
    return config;
}
Also used : DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) KeyValue(io.openmessaging.KeyValue)

Aggregations

KeyValue (io.openmessaging.KeyValue)39 DefaultKeyValue (io.openmessaging.internal.DefaultKeyValue)29 Test (org.junit.Test)14 ArrayList (java.util.ArrayList)9 Map (java.util.Map)6 HashMap (java.util.HashMap)5 List (java.util.List)5 ConnectKeyValue (org.apache.rocketmq.connect.runtime.common.ConnectKeyValue)5 BytesMessage (io.openmessaging.BytesMessage)4 SourceDataEntry (io.openmessaging.connector.api.data.SourceDataEntry)4 PositionStorageReader (io.openmessaging.connector.api.PositionStorageReader)3 SourceTaskContext (io.openmessaging.connector.api.source.SourceTaskContext)3 ByteBuffer (java.nio.ByteBuffer)3 QueueMetaData (io.openmessaging.connector.api.common.QueueMetaData)2 SinkTaskContext (io.openmessaging.connector.api.sink.SinkTaskContext)2 BytesMessageImpl (io.openmessaging.rocketmq.domain.BytesMessageImpl)2 Set (java.util.Set)2 MQClientException (org.apache.rocketmq.client.exception.MQClientException)2 MessageQueue (org.apache.rocketmq.common.message.MessageQueue)2 RemotingException (org.apache.rocketmq.remoting.exception.RemotingException)2