use of io.openmessaging.KeyValue in project rocketmq-externals by apache.
the class ActivemqSourceTaskTest method test.
// @Test
public void test() throws InterruptedException {
KeyValue kv = new DefaultKeyValue();
kv.put("activemqUrl", "tcp://112.74.48.251:6166");
kv.put("destinationType", "queue");
kv.put("destinationName", "test-queue");
ActivemqSourceTask task = new ActivemqSourceTask();
task.start(kv);
for (int i = 0; i < 20; ) {
Collection<SourceDataEntry> sourceDataEntry = task.poll();
i = i + sourceDataEntry.size();
System.out.println(sourceDataEntry);
}
Thread.sleep(20000);
}
use of io.openmessaging.KeyValue in project rocketmq-externals by apache.
the class DivideTaskByTopic method divideSinkTaskByTopic.
private List<KeyValue> divideSinkTaskByTopic(DbConnectorConfig dbConnectorConfig, TaskDivideConfig tdc) {
List<KeyValue> config = new ArrayList<KeyValue>();
int parallelism = tdc.getTaskParallelism();
int id = -1;
Set<String> topicRouteSet = ((SinkDbConnectorConfig) dbConnectorConfig).getWhiteTopics();
Map<Integer, StringBuilder> taskTopicList = new HashMap<>();
for (String topicName : topicRouteSet) {
int ind = ++id % parallelism;
if (!taskTopicList.containsKey(ind)) {
taskTopicList.put(ind, new StringBuilder(topicName));
} else {
taskTopicList.get(ind).append(",").append(topicName);
}
}
for (int i = 0; i < parallelism; i++) {
KeyValue keyValue = new DefaultKeyValue();
keyValue.put(Config.CONN_DB_IP, tdc.getDbUrl());
keyValue.put(Config.CONN_DB_PORT, tdc.getDbPort());
keyValue.put(Config.CONN_DB_USERNAME, tdc.getDbUserName());
keyValue.put(Config.CONN_DB_PASSWORD, tdc.getDbPassword());
keyValue.put(Config.CONN_DB_DATACENTER, tdc.getLocalDataCenter());
keyValue.put(Config.CONN_TOPIC_NAMES, taskTopicList.get(i).toString());
keyValue.put(Config.CONN_DATA_TYPE, tdc.getDataType());
keyValue.put(Config.CONN_SOURCE_RECORD_CONVERTER, tdc.getSrcRecordConverter());
keyValue.put(Config.CONN_DB_MODE, tdc.getMode());
config.add(keyValue);
}
return config;
}
use of io.openmessaging.KeyValue in project rocketmq-externals by apache.
the class DivideTaskByQueue method divideSinkTaskByQueue.
public List<KeyValue> divideSinkTaskByQueue(DbConnectorConfig dbConnectorConfig, TaskDivideConfig tdc) {
List<KeyValue> config = new ArrayList<KeyValue>();
int parallelism = tdc.getTaskParallelism();
Map<Integer, List<TaskTopicInfo>> queueTopicList = new HashMap<Integer, List<TaskTopicInfo>>();
Map<String, Set<TaskTopicInfo>> topicRouteMap = ((SinkDbConnectorConfig) dbConnectorConfig).getTopicRouteMap();
int id = -1;
for (String t : topicRouteMap.keySet()) {
for (TaskTopicInfo taskTopicInfo : topicRouteMap.get(t)) {
int ind = ++id % parallelism;
if (!queueTopicList.containsKey(ind)) {
queueTopicList.put(ind, new ArrayList<TaskTopicInfo>());
}
queueTopicList.get(ind).add(taskTopicInfo);
}
}
for (int i = 0; i < parallelism; i++) {
KeyValue keyValue = new DefaultKeyValue();
keyValue.put(Config.CONN_DB_IP, tdc.getDbUrl());
keyValue.put(Config.CONN_DB_PORT, tdc.getDbPort());
keyValue.put(Config.CONN_DB_USERNAME, tdc.getDbUserName());
keyValue.put(Config.CONN_DB_PASSWORD, tdc.getDbPassword());
keyValue.put(Config.CONN_TOPIC_NAMES, JSONObject.toJSONString(queueTopicList.get(i)));
keyValue.put(Config.CONN_DATA_TYPE, tdc.getDataType());
keyValue.put(Config.CONN_SOURCE_RECORD_CONVERTER, tdc.getSrcRecordConverter());
keyValue.put(Config.CONN_DB_MODE, tdc.getMode());
config.add(keyValue);
}
return config;
}
use of io.openmessaging.KeyValue in project rocketmq-externals by apache.
the class BaseJmsSourceConnectorTest method taskConfigsTest.
@Test
public void taskConfigsTest() {
assertEquals(connector.taskConfigs().get(0), null);
KeyValue keyValue = new DefaultKeyValue();
for (String requestKey : Config.REQUEST_CONFIG) {
keyValue.put(requestKey, requestKey);
}
connector.verifyAndSetConfig(keyValue);
assertEquals(connector.taskConfigs().get(0), keyValue);
}
use of io.openmessaging.KeyValue in project rocketmq-externals by apache.
the class DivideTaskByTopic method divideSinkTaskByTopic.
private List<KeyValue> divideSinkTaskByTopic(DbConnectorConfig dbConnectorConfig, TaskDivideConfig tdc) {
List<KeyValue> config = new ArrayList<KeyValue>();
int parallelism = tdc.getTaskParallelism();
int id = -1;
Set<String> topicRouteSet = ((SinkDbConnectorConfig) dbConnectorConfig).getWhiteTopics();
Map<Integer, StringBuilder> taskTopicList = new HashMap<>();
for (String topicName : topicRouteSet) {
int ind = ++id % parallelism;
if (!taskTopicList.containsKey(ind)) {
taskTopicList.put(ind, new StringBuilder(topicName));
} else {
taskTopicList.get(ind).append(",").append(topicName);
}
}
for (int i = 0; i < parallelism; i++) {
KeyValue keyValue = new DefaultKeyValue();
keyValue.put(Config.CONN_DB_IP, tdc.getDbUrl());
keyValue.put(Config.CONN_DB_PORT, tdc.getDbPort());
keyValue.put(Config.CONN_DB_USERNAME, tdc.getDbUserName());
keyValue.put(Config.CONN_DB_PASSWORD, tdc.getDbPassword());
keyValue.put(Config.CONN_TOPIC_NAMES, taskTopicList.get(i).toString());
keyValue.put(Config.CONN_DATA_TYPE, tdc.getDataType());
keyValue.put(Config.CONN_SOURCE_RECORD_CONVERTER, tdc.getSrcRecordConverter());
keyValue.put(Config.CONN_DB_MODE, tdc.getMode());
config.add(keyValue);
}
return config;
}
Aggregations