Examples with KeyValue io.openmessaging.KeyValue used on opensource projects

Example 21 with KeyValue

use of io.openmessaging.KeyValue in project rocketmq-externals by apache.

the class DivideTaskByTopic method divide.

@Override
public List<KeyValue> divide(Map<String, Set<TaskTopicInfo>> topicRouteMap, TaskDivideConfig tdc) {
    List<KeyValue> config = new ArrayList<KeyValue>();
    int parallelism = tdc.getTaskParallelism();
    int id = -1;
    Map<Integer, List<TaskTopicInfo>> taskTopicList = new HashMap<Integer, List<TaskTopicInfo>>();
    for (Map.Entry<String, Set<TaskTopicInfo>> entry : topicRouteMap.entrySet()) {
        int ind = ++id % parallelism;
        if (!taskTopicList.containsKey(ind)) {
            taskTopicList.put(ind, new ArrayList<TaskTopicInfo>());
        }
        taskTopicList.get(ind).addAll(entry.getValue());
    }
    for (int i = 0; i < parallelism; i++) {
        KeyValue keyValue = new DefaultKeyValue();
        keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
        keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
        keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(taskTopicList.get(i)));
        keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ACL_ENABLE.getKey(), String.valueOf(tdc.isSrcAclEnable()));
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ACCESS_KEY.getKey(), tdc.getSrcAccessKey());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_SECRET_KEY.getKey(), tdc.getSrcSecretKey());
        config.add(keyValue);
    }
    return config;
}

Example 22 with KeyValue

use of io.openmessaging.KeyValue in project rocketmq-externals by apache.

the class ConnectKeyValueTest method testKeyValueOperations.

@Test
public void testKeyValueOperations() {
    KeyValue keyValue = new ConnectKeyValue();
    keyValue.put("StringKey", "StringValue");
    keyValue.put("IntegerKey", 0);
    keyValue.put("LongKey", 1L);
    keyValue.put("DoubleKey", 5.2);
    assertEquals("StringValue", keyValue.getString("StringKey"));
    assertEquals(0, keyValue.getInt("IntegerKey"));
    assertEquals(1L, keyValue.getLong("LongKey"));
    assertEquals(5.2, keyValue.getDouble("DoubleKey"), 0.0);
    assertEquals("StringValue1", keyValue.getString("StringKey1", "StringValue1"));
    assertEquals(2, keyValue.getInt("IntegerKey1", 2));
    assertEquals(2L, keyValue.getLong("LongKey1", 2L));
    assertEquals(5.0, keyValue.getDouble("DoubleKey1", 5.0), 0.0);
    Set<String> keySet = keyValue.keySet();
    Set<String> compareKeySet = new HashSet<>();
    compareKeySet.add("StringKey");
    compareKeySet.add("IntegerKey");
    compareKeySet.add("LongKey");
    compareKeySet.add("DoubleKey");
    assertEquals(keySet, compareKeySet);
    assertTrue(keyValue.containsKey("StringKey"));
    assertTrue(keyValue.containsKey("IntegerKey"));
    assertTrue(keyValue.containsKey("LongKey"));
    assertTrue(keyValue.containsKey("DoubleKey"));
}

Example 23 with KeyValue

use of io.openmessaging.KeyValue in project rocketmq-externals by apache.

the class RedisSourceConnectorTest method testConnector.

@Test
public void testConnector() {
    RedisSourceConnector connector = new RedisSourceConnector();
    connector.verifyAndSetConfig(this.keyValue);
    Class cl = connector.taskClass();
    Assert.assertNotNull(cl);
    List<KeyValue> keyValues = connector.taskConfigs();
    Assert.assertNotNull(keyValues);
    connector.start();
    connector.pause();
    connector.resume();
    connector.stop();
}

Example 24 with KeyValue

use of io.openmessaging.KeyValue in project rocketmq-externals by apache.

the class HudiSinkConnector method taskConfigs.

@Override
public List<KeyValue> taskConfigs() {
    log.info("List.start");
    if (!configValid) {
        return new ArrayList<KeyValue>();
    }
    startMQAdminTools();
    buildRoute();
    DefaultKeyValue defaultKeyValue = new DefaultKeyValue();
    defaultKeyValue.put(HudiConnectConfig.CONN_HUDI_TABLE_PATH, sinkConnectConfig.getTablePath());
    defaultKeyValue.put(HudiConnectConfig.CONN_HUDI_TABLE_NAME, sinkConnectConfig.getTableName());
    defaultKeyValue.put(HudiConnectConfig.CONN_HUDI_INSERT_SHUFFLE_PARALLELISM, sinkConnectConfig.getInsertShuffleParallelism());
    defaultKeyValue.put(HudiConnectConfig.CONN_HUDI_UPSERT_SHUFFLE_PARALLELISM, sinkConnectConfig.getUpsertShuffleParallelism());
    defaultKeyValue.put(HudiConnectConfig.CONN_HUDI_DELETE_PARALLELISM, sinkConnectConfig.getDeleteParallelism());
    defaultKeyValue.put(HudiConnectConfig.CONN_SOURCE_RECORD_CONVERTER, sinkConnectConfig.getSrcRecordConverter());
    defaultKeyValue.put(HudiConnectConfig.CONN_TOPIC_NAMES, sinkConnectConfig.getTopicNames());
    defaultKeyValue.put(HudiConnectConfig.CONN_SCHEMA_PATH, sinkConnectConfig.getSchemaPath());
    defaultKeyValue.put(HudiConnectConfig.CONN_TASK_PARALLELISM, sinkConnectConfig.getTaskParallelism());
    defaultKeyValue.put(HudiConnectConfig.CONN_TASK_DIVIDE_STRATEGY, sinkConnectConfig.getTaskDivideStrategy());
    defaultKeyValue.put(HudiConnectConfig.CONN_WHITE_LIST, JSONObject.toJSONString(sinkConnectConfig.getWhiteList()));
    defaultKeyValue.put(HudiConnectConfig.CONN_SCHEMA_PATH, sinkConnectConfig.getSchemaPath());
    defaultKeyValue.put(HudiConnectConfig.CONN_TOPIC_ROUTE_INFO, JSONObject.toJSONString(sinkConnectConfig.getTopicRouteMap()));
    log.info("taskConfig : " + defaultKeyValue + ", sinkConnectConfig : " + sinkConnectConfig);
    ITaskDivideStrategy strategy = TaskDivideStrategyFactory.getInstance();
    List<KeyValue> taskConfigs = strategy.divide(defaultKeyValue);
    return taskConfigs;
}

Example 25 with KeyValue

use of io.openmessaging.KeyValue in project rocketmq-externals by apache.

the class TaskDivideByQueueStrategy method divide.

@Override
public List<KeyValue> divide(KeyValue source) {
    List<KeyValue> config = new ArrayList<KeyValue>();
    int parallelism = source.getInt(HudiConnectConfig.CONN_TASK_PARALLELISM);
    Map<String, MessageQueue> topicRouteInfos = (Map<String, MessageQueue>) JSONObject.parse(source.getString(HudiConnectConfig.CONN_TOPIC_ROUTE_INFO));
    int id = 0;
    List<List<String>> taskTopicQueues = new ArrayList<>(parallelism);
    for (Map.Entry<String, MessageQueue> topicQueue : topicRouteInfos.entrySet()) {
        MessageQueue messageQueue = topicQueue.getValue();
        String topicQueueStr = messageQueue.getTopic() + "," + messageQueue.getBrokerName() + "," + messageQueue.getQueueId();
        int ind = ++id % parallelism;
        if (taskTopicQueues.get(ind) != null) {
            List<String> taskTopicQueue = new LinkedList<>();
            taskTopicQueue.add(topicQueueStr);
            taskTopicQueues.add(ind, taskTopicQueue);
        } else {
            List<String> taskTopicQueue = taskTopicQueues.get(ind);
            taskTopicQueue.add(topicQueueStr);
        }
    }
    for (int i = 0; i < parallelism; i++) {
        // build single task queue config; format is topicName1,brokerName1,queueId1;topicName1,brokerName1,queueId2
        String singleTaskTopicQueueStr = "";
        List<String> singleTaskTopicQueues = taskTopicQueues.get(i);
        for (String singleTopicQueue : singleTaskTopicQueues) {
            singleTaskTopicQueueStr += singleTopicQueue + ";";
        }
        singleTaskTopicQueueStr = singleTaskTopicQueueStr.substring(0, singleTaskTopicQueueStr.length() - 1);
        // fill connect config;
        KeyValue keyValue = new DefaultKeyValue();
        keyValue.put(HudiConnectConfig.CONN_TOPIC_QUEUES, singleTaskTopicQueueStr);
        keyValue.put(HudiConnectConfig.CONN_HUDI_TABLE_PATH, source.getString(HudiConnectConfig.CONN_HUDI_TABLE_PATH));
        keyValue.put(HudiConnectConfig.CONN_HUDI_TABLE_NAME, source.getString(HudiConnectConfig.CONN_HUDI_TABLE_NAME));
        keyValue.put(HudiConnectConfig.CONN_HUDI_INSERT_SHUFFLE_PARALLELISM, source.getInt(HudiConnectConfig.CONN_HUDI_INSERT_SHUFFLE_PARALLELISM));
        keyValue.put(HudiConnectConfig.CONN_HUDI_UPSERT_SHUFFLE_PARALLELISM, source.getInt(HudiConnectConfig.CONN_HUDI_UPSERT_SHUFFLE_PARALLELISM));
        keyValue.put(HudiConnectConfig.CONN_HUDI_DELETE_PARALLELISM, source.getInt(HudiConnectConfig.CONN_HUDI_DELETE_PARALLELISM));
        keyValue.put(HudiConnectConfig.CONN_SOURCE_RECORD_CONVERTER, source.getString(HudiConnectConfig.CONN_SOURCE_RECORD_CONVERTER));
        keyValue.put(HudiConnectConfig.CONN_SCHEMA_PATH, source.getString(HudiConnectConfig.CONN_SCHEMA_PATH));
        keyValue.put(HudiConnectConfig.CONN_TASK_PARALLELISM, source.getInt(HudiConnectConfig.CONN_TASK_PARALLELISM));
        keyValue.put(HudiConnectConfig.CONN_SCHEMA_PATH, source.getString(HudiConnectConfig.CONN_SCHEMA_PATH));
        config.add(keyValue);
    }
    return config;
}