use of io.openmessaging.internal.DefaultKeyValue in project rocketmq-externals by apache.
the class RedisSourceTaskTest method initKeyValue.
private void initKeyValue() {
this.keyValue = new DefaultKeyValue();
this.keyValue.put("redisAddr", "127.0.0.1");
this.keyValue.put("redisPort", "6379");
this.keyValue.put("redisPassword", "");
}
use of io.openmessaging.internal.DefaultKeyValue in project rocketmq-externals by apache.
the class PropertyToObjectUtilsTest method test.
@Test
public void test() {
User user = new User();
user.setName("LLL");
user.setAge(100);
user.setScore(365);
user.setAverage(66.66);
user.setMath(78L);
user.setHigh(false);
KeyValue pair = new DefaultKeyValue();
pair.put("name", "LLL");
pair.put("age", "100");
pair.put("score", "365");
pair.put("average", "66.66");
pair.put("math", "78");
pair.put("high", "false");
User user1 = new User();
Exception ex = null;
try {
PropertyToObjectUtils.properties2Object(pair, user1);
} catch (InvocationTargetException e) {
ex = e;
} catch (IllegalAccessException e) {
ex = e;
}
Assert.assertEquals(user, user1);
Assert.assertNull(ex);
}
use of io.openmessaging.internal.DefaultKeyValue in project rocketmq-externals by apache.
the class Utils method groupPartitions.
public static List<KeyValue> groupPartitions(List<String> elements, int numGroups, RmqConnectorConfig tdc) {
if (numGroups <= 0)
throw new IllegalArgumentException("Number of groups must be positive.");
List<KeyValue> result = new ArrayList<>(numGroups);
// Each group has either n+1 or n raw partitions
int perGroup = elements.size() / numGroups;
int leftover = elements.size() - (numGroups * perGroup);
int assigned = 0;
for (int group = 0; group < numGroups; group++) {
int numThisGroup = group < leftover ? perGroup + 1 : perGroup;
KeyValue keyValue = new DefaultKeyValue();
List<String> groupList = new ArrayList<>();
for (int i = 0; i < numThisGroup; i++) {
groupList.add(elements.get(assigned));
assigned++;
}
keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSrcNamesrvs());
keyValue.put(TaskConfigEnum.TASK_SOURCE_CLUSTER.getKey(), tdc.getSrcCluster());
keyValue.put(TaskConfigEnum.TASK_OFFSET_SYNC_TOPIC.getKey(), tdc.getOffsetSyncTopic());
keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.OFFSET.ordinal());
keyValue.put(TaskConfigEnum.TASK_GROUP_INFO.getKey(), JSONObject.toJSONString(groupList));
keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getConverter());
result.add(keyValue);
log.debug("allocate group partition: {}", keyValue);
}
return result;
}
use of io.openmessaging.internal.DefaultKeyValue in project rocketmq-externals by apache.
the class DivideTaskByConsistentHash method divide.
@Override
public List<KeyValue> divide(Map<String, Set<TaskTopicInfo>> topicMap, TaskDivideConfig tdc) {
List<KeyValue> config = new ArrayList<>();
int parallelism = tdc.getTaskParallelism();
Map<Integer, List<TaskTopicInfo>> queueTopicList = new HashMap<>();
int id = -1;
Collection<ClientNode> cidNodes = new ArrayList<>();
for (int i = 0; i < parallelism; i++) {
cidNodes.add(new ClientNode(i, Integer.toString(i)));
queueTopicList.put(i, new ArrayList<>());
}
ConsistentHashRouter<ClientNode> router = new ConsistentHashRouter<>(cidNodes, cidNodes.size());
for (String t : topicMap.keySet()) {
for (TaskTopicInfo queue : topicMap.get(t)) {
ClientNode clientNode = router.routeNode(queue.toString());
if (clientNode != null) {
queueTopicList.get(clientNode.index).add(queue);
}
}
}
for (int i = 0; i < parallelism; i++) {
KeyValue keyValue = new DefaultKeyValue();
keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(queueTopicList.get(i)));
keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
keyValue.put(TaskConfigEnum.TASK_SOURCE_ACL_ENABLE.getKey(), String.valueOf(tdc.isSrcAclEnable()));
keyValue.put(TaskConfigEnum.TASK_SOURCE_ACCESS_KEY.getKey(), tdc.getSrcAccessKey());
keyValue.put(TaskConfigEnum.TASK_SOURCE_SECRET_KEY.getKey(), tdc.getSrcSecretKey());
config.add(keyValue);
}
return config;
}
use of io.openmessaging.internal.DefaultKeyValue in project rocketmq-externals by apache.
the class DivideTaskByTopic method divide.
@Override
public List<KeyValue> divide(Map<String, Set<TaskTopicInfo>> topicRouteMap, TaskDivideConfig tdc) {
List<KeyValue> config = new ArrayList<KeyValue>();
int parallelism = tdc.getTaskParallelism();
int id = -1;
Map<Integer, List<TaskTopicInfo>> taskTopicList = new HashMap<Integer, List<TaskTopicInfo>>();
for (Map.Entry<String, Set<TaskTopicInfo>> entry : topicRouteMap.entrySet()) {
int ind = ++id % parallelism;
if (!taskTopicList.containsKey(ind)) {
taskTopicList.put(ind, new ArrayList<TaskTopicInfo>());
}
taskTopicList.get(ind).addAll(entry.getValue());
}
for (int i = 0; i < parallelism; i++) {
KeyValue keyValue = new DefaultKeyValue();
keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(taskTopicList.get(i)));
keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
keyValue.put(TaskConfigEnum.TASK_SOURCE_ACL_ENABLE.getKey(), String.valueOf(tdc.isSrcAclEnable()));
keyValue.put(TaskConfigEnum.TASK_SOURCE_ACCESS_KEY.getKey(), tdc.getSrcAccessKey());
keyValue.put(TaskConfigEnum.TASK_SOURCE_SECRET_KEY.getKey(), tdc.getSrcSecretKey());
config.add(keyValue);
}
return config;
}
Aggregations