use of io.openmessaging.KeyValue in project rocketmq-externals by apache.
the class DivideTaskByTopic method divideSourceTaskByTopic.
private List<KeyValue> divideSourceTaskByTopic(DbConnectorConfig dbConnectorConfig, TaskDivideConfig tdc) {
List<KeyValue> config = new ArrayList<KeyValue>();
int parallelism = tdc.getTaskParallelism();
int id = -1;
Map<String, String> topicRouteMap = ((SourceDbConnectorConfig) dbConnectorConfig).getWhiteTopics();
Map<Integer, String> taskTopicList = new HashMap<>();
Map<Integer, Map<String, Map<String, String>>> taskWhiteList = new HashMap<>();
for (Map.Entry<String, String> entry : topicRouteMap.entrySet()) {
int ind = ++id % parallelism;
if (!taskWhiteList.containsKey(ind)) {
taskWhiteList.put(ind, new HashMap<>());
}
String dbKey = entry.getKey().split("-")[0];
String tableKey = entry.getKey().split("-")[1];
taskTopicList.put(ind, tableKey);
String filter = entry.getValue();
Map<String, String> tableMap = new HashMap<>();
tableMap.put(tableKey, filter);
if (!taskWhiteList.get(ind).containsKey(dbKey)) {
taskWhiteList.get(ind).put(dbKey, tableMap);
} else {
taskWhiteList.get(ind).get(dbKey).putAll(tableMap);
}
}
for (int i = 0; i < parallelism; i++) {
KeyValue keyValue = new DefaultKeyValue();
keyValue.put(Config.CONN_DB_IP, tdc.getDbUrl());
keyValue.put(Config.CONN_DB_PORT, tdc.getDbPort());
keyValue.put(Config.CONN_DB_USERNAME, tdc.getDbUserName());
keyValue.put(Config.CONN_DB_PASSWORD, tdc.getDbPassword());
keyValue.put(Config.CONN_WHITE_LIST, JSONObject.toJSONString(taskWhiteList.get(i)));
keyValue.put(Config.CONN_TOPIC_NAMES, taskTopicList.get(i));
keyValue.put(Config.CONN_DATA_TYPE, tdc.getDataType());
keyValue.put(Config.CONN_SOURCE_RECORD_CONVERTER, tdc.getSrcRecordConverter());
keyValue.put(Config.CONN_DB_MODE, tdc.getMode());
config.add(keyValue);
}
return config;
}
use of io.openmessaging.KeyValue in project rocketmq-externals by apache.
the class JdbcSourceConnectorTest method taskConfigsTest.
@Test
public void taskConfigsTest() {
assertEquals(connector.taskConfigs().get(0), null);
KeyValue keyValue = new DefaultKeyValue();
for (String requestKey : Config.REQUEST_CONFIG) {
keyValue.put(requestKey, requestKey);
}
connector.verifyAndSetConfig(keyValue);
assertEquals(connector.taskConfigs().get(0), keyValue);
}
use of io.openmessaging.KeyValue in project rocketmq-externals by apache.
the class KafkaSourceConnector method taskConfigs.
@Override
public List<KeyValue> taskConfigs() {
if (connectConfig == null) {
return new ArrayList<KeyValue>();
}
log.info("Source Connector taskConfigs enter");
List<KeyValue> configs = new ArrayList<>();
int task_num = connectConfig.getInt(ConfigDefine.TASK_NUM);
log.info("Source Connector taskConfigs: task_num:" + task_num);
for (int i = 0; i < task_num; ++i) {
KeyValue config = new DefaultKeyValue();
config.put(ConfigDefine.BOOTSTRAP_SERVER, connectConfig.getString(ConfigDefine.BOOTSTRAP_SERVER));
config.put(ConfigDefine.TOPICS, connectConfig.getString(ConfigDefine.TOPICS));
config.put(ConfigDefine.GROUP_ID, connectConfig.getString(ConfigDefine.GROUP_ID));
config.put(ConfigDefine.CONNECTOR_CLASS, connectConfig.getString(ConfigDefine.CONNECTOR_CLASS));
config.put(ConfigDefine.SOURCE_RECORD_CONVERTER, connectConfig.getString(ConfigDefine.SOURCE_RECORD_CONVERTER));
configs.add(config);
}
return configs;
}
use of io.openmessaging.KeyValue in project rocketmq-externals by apache.
the class KafkaSourceConnectorTest method verifyAndSetConfigTest.
@Test
public void verifyAndSetConfigTest() {
KeyValue keyValue = new DefaultKeyValue();
for (String requestKey : ConfigDefine.REQUEST_CONFIG) {
assertEquals(connector.verifyAndSetConfig(keyValue), "Request Config key: " + requestKey);
keyValue.put(requestKey, requestKey);
}
assertEquals(connector.verifyAndSetConfig(keyValue), "");
}
use of io.openmessaging.KeyValue in project rocketmq-externals by apache.
the class KafkaSourceConnectorTest method taskConfigsTest.
@Test
public void taskConfigsTest() {
assertEquals(connector.taskConfigs().size(), 0);
KeyValue keyValue = new DefaultKeyValue();
for (String requestKey : ConfigDefine.REQUEST_CONFIG) {
keyValue.put(requestKey, requestKey);
}
keyValue.put(ConfigDefine.TASK_NUM, 1);
connector.verifyAndSetConfig(keyValue);
assertEquals(connector.taskConfigs().get(0).getString(ConfigDefine.TOPICS), keyValue.getString(ConfigDefine.TOPICS));
}
Aggregations