Search in sources :

Example 16 with DefaultKeyValue

use of io.openmessaging.internal.DefaultKeyValue in project rocketmq-externals by apache.

the class RedisSourceTaskTest method initKeyValue.

private void initKeyValue() {
    this.keyValue = new DefaultKeyValue();
    this.keyValue.put("redisAddr", "127.0.0.1");
    this.keyValue.put("redisPort", "6379");
    this.keyValue.put("redisPassword", "");
}
Also used : DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue)

Example 17 with DefaultKeyValue

use of io.openmessaging.internal.DefaultKeyValue in project rocketmq-externals by apache.

the class PropertyToObjectUtilsTest method test.

@Test
public void test() {
    User user = new User();
    user.setName("LLL");
    user.setAge(100);
    user.setScore(365);
    user.setAverage(66.66);
    user.setMath(78L);
    user.setHigh(false);
    KeyValue pair = new DefaultKeyValue();
    pair.put("name", "LLL");
    pair.put("age", "100");
    pair.put("score", "365");
    pair.put("average", "66.66");
    pair.put("math", "78");
    pair.put("high", "false");
    User user1 = new User();
    Exception ex = null;
    try {
        PropertyToObjectUtils.properties2Object(pair, user1);
    } catch (InvocationTargetException e) {
        ex = e;
    } catch (IllegalAccessException e) {
        ex = e;
    }
    Assert.assertEquals(user, user1);
    Assert.assertNull(ex);
}
Also used : DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) KeyValue(io.openmessaging.KeyValue) InvocationTargetException(java.lang.reflect.InvocationTargetException) InvocationTargetException(java.lang.reflect.InvocationTargetException) Test(org.junit.Test)

Example 18 with DefaultKeyValue

use of io.openmessaging.internal.DefaultKeyValue in project rocketmq-externals by apache.

the class Utils method groupPartitions.

public static List<KeyValue> groupPartitions(List<String> elements, int numGroups, RmqConnectorConfig tdc) {
    if (numGroups <= 0)
        throw new IllegalArgumentException("Number of groups must be positive.");
    List<KeyValue> result = new ArrayList<>(numGroups);
    // Each group has either n+1 or n raw partitions
    int perGroup = elements.size() / numGroups;
    int leftover = elements.size() - (numGroups * perGroup);
    int assigned = 0;
    for (int group = 0; group < numGroups; group++) {
        int numThisGroup = group < leftover ? perGroup + 1 : perGroup;
        KeyValue keyValue = new DefaultKeyValue();
        List<String> groupList = new ArrayList<>();
        for (int i = 0; i < numThisGroup; i++) {
            groupList.add(elements.get(assigned));
            assigned++;
        }
        keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSrcNamesrvs());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_CLUSTER.getKey(), tdc.getSrcCluster());
        keyValue.put(TaskConfigEnum.TASK_OFFSET_SYNC_TOPIC.getKey(), tdc.getOffsetSyncTopic());
        keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.OFFSET.ordinal());
        keyValue.put(TaskConfigEnum.TASK_GROUP_INFO.getKey(), JSONObject.toJSONString(groupList));
        keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getConverter());
        result.add(keyValue);
        log.debug("allocate group partition: {}", keyValue);
    }
    return result;
}
Also used : DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) KeyValue(io.openmessaging.KeyValue) DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) ArrayList(java.util.ArrayList)

Example 19 with DefaultKeyValue

use of io.openmessaging.internal.DefaultKeyValue in project rocketmq-externals by apache.

the class DivideTaskByConsistentHash method divide.

@Override
public List<KeyValue> divide(Map<String, Set<TaskTopicInfo>> topicMap, TaskDivideConfig tdc) {
    List<KeyValue> config = new ArrayList<>();
    int parallelism = tdc.getTaskParallelism();
    Map<Integer, List<TaskTopicInfo>> queueTopicList = new HashMap<>();
    int id = -1;
    Collection<ClientNode> cidNodes = new ArrayList<>();
    for (int i = 0; i < parallelism; i++) {
        cidNodes.add(new ClientNode(i, Integer.toString(i)));
        queueTopicList.put(i, new ArrayList<>());
    }
    ConsistentHashRouter<ClientNode> router = new ConsistentHashRouter<>(cidNodes, cidNodes.size());
    for (String t : topicMap.keySet()) {
        for (TaskTopicInfo queue : topicMap.get(t)) {
            ClientNode clientNode = router.routeNode(queue.toString());
            if (clientNode != null) {
                queueTopicList.get(clientNode.index).add(queue);
            }
        }
    }
    for (int i = 0; i < parallelism; i++) {
        KeyValue keyValue = new DefaultKeyValue();
        keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
        keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
        keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(queueTopicList.get(i)));
        keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ACL_ENABLE.getKey(), String.valueOf(tdc.isSrcAclEnable()));
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ACCESS_KEY.getKey(), tdc.getSrcAccessKey());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_SECRET_KEY.getKey(), tdc.getSrcSecretKey());
        config.add(keyValue);
    }
    return config;
}
Also used : KeyValue(io.openmessaging.KeyValue) DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) ConsistentHashRouter(org.apache.rocketmq.common.consistenthash.ConsistentHashRouter) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) TaskTopicInfo(org.apache.rocketmq.replicator.config.TaskTopicInfo) ArrayList(java.util.ArrayList) List(java.util.List)

Example 20 with DefaultKeyValue

use of io.openmessaging.internal.DefaultKeyValue in project rocketmq-externals by apache.

the class DivideTaskByTopic method divide.

@Override
public List<KeyValue> divide(Map<String, Set<TaskTopicInfo>> topicRouteMap, TaskDivideConfig tdc) {
    List<KeyValue> config = new ArrayList<KeyValue>();
    int parallelism = tdc.getTaskParallelism();
    int id = -1;
    Map<Integer, List<TaskTopicInfo>> taskTopicList = new HashMap<Integer, List<TaskTopicInfo>>();
    for (Map.Entry<String, Set<TaskTopicInfo>> entry : topicRouteMap.entrySet()) {
        int ind = ++id % parallelism;
        if (!taskTopicList.containsKey(ind)) {
            taskTopicList.put(ind, new ArrayList<TaskTopicInfo>());
        }
        taskTopicList.get(ind).addAll(entry.getValue());
    }
    for (int i = 0; i < parallelism; i++) {
        KeyValue keyValue = new DefaultKeyValue();
        keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
        keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
        keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(taskTopicList.get(i)));
        keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ACL_ENABLE.getKey(), String.valueOf(tdc.isSrcAclEnable()));
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ACCESS_KEY.getKey(), tdc.getSrcAccessKey());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_SECRET_KEY.getKey(), tdc.getSrcSecretKey());
        config.add(keyValue);
    }
    return config;
}
Also used : DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) KeyValue(io.openmessaging.KeyValue) Set(java.util.Set) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) DefaultKeyValue(io.openmessaging.internal.DefaultKeyValue) List(java.util.List) ArrayList(java.util.ArrayList) Map(java.util.Map) HashMap(java.util.HashMap)

Aggregations

DefaultKeyValue (io.openmessaging.internal.DefaultKeyValue)32 KeyValue (io.openmessaging.KeyValue)27 Test (org.junit.Test)14 ArrayList (java.util.ArrayList)8 List (java.util.List)7 HashMap (java.util.HashMap)4 Field (java.lang.reflect.Field)3 SourceDataEntry (io.openmessaging.connector.api.data.SourceDataEntry)2 Map (java.util.Map)2 Set (java.util.Set)2 MongoSourceTask (org.apache.connect.mongo.connector.MongoSourceTask)2 ReplicaSet (org.apache.connect.mongo.replicator.ReplicaSet)2 ReplicaSetConfig (org.apache.connect.mongo.replicator.ReplicaSetConfig)2 ReplicaSetsContext (org.apache.connect.mongo.replicator.ReplicaSetsContext)2 TaskTopicInfo (org.apache.rocketmq.replicator.config.TaskTopicInfo)2 Before (org.junit.Before)2 RedisURI (com.moilioncircle.redis.replicator.RedisURI)1 InvocationTargetException (java.lang.reflect.InvocationTargetException)1 ByteBuffer (java.nio.ByteBuffer)1 LinkedList (java.util.LinkedList)1