Examples with DefaultKeyValue io.openmessaging.internal.DefaultKeyValue used on opensource projects

Example 21 with DefaultKeyValue

use of io.openmessaging.internal.DefaultKeyValue in project rocketmq-externals by apache.

the class RedisSourceConnectorTest method initKeyValue.

@Before
public void initKeyValue() {
    this.keyValue = new DefaultKeyValue();
    this.keyValue.put("redisAddr", "127.0.0.1");
    this.keyValue.put("redisPort", "6379");
    this.keyValue.put("redisPassword", "");
}

Example 22 with DefaultKeyValue

use of io.openmessaging.internal.DefaultKeyValue in project rocketmq-externals by apache.

the class HudiSinkConnector method taskConfigs.

@Override
public List<KeyValue> taskConfigs() {
    log.info("List.start");
    if (!configValid) {
        return new ArrayList<KeyValue>();
    }
    startMQAdminTools();
    buildRoute();
    DefaultKeyValue defaultKeyValue = new DefaultKeyValue();
    defaultKeyValue.put(HudiConnectConfig.CONN_HUDI_TABLE_PATH, sinkConnectConfig.getTablePath());
    defaultKeyValue.put(HudiConnectConfig.CONN_HUDI_TABLE_NAME, sinkConnectConfig.getTableName());
    defaultKeyValue.put(HudiConnectConfig.CONN_HUDI_INSERT_SHUFFLE_PARALLELISM, sinkConnectConfig.getInsertShuffleParallelism());
    defaultKeyValue.put(HudiConnectConfig.CONN_HUDI_UPSERT_SHUFFLE_PARALLELISM, sinkConnectConfig.getUpsertShuffleParallelism());
    defaultKeyValue.put(HudiConnectConfig.CONN_HUDI_DELETE_PARALLELISM, sinkConnectConfig.getDeleteParallelism());
    defaultKeyValue.put(HudiConnectConfig.CONN_SOURCE_RECORD_CONVERTER, sinkConnectConfig.getSrcRecordConverter());
    defaultKeyValue.put(HudiConnectConfig.CONN_TOPIC_NAMES, sinkConnectConfig.getTopicNames());
    defaultKeyValue.put(HudiConnectConfig.CONN_SCHEMA_PATH, sinkConnectConfig.getSchemaPath());
    defaultKeyValue.put(HudiConnectConfig.CONN_TASK_PARALLELISM, sinkConnectConfig.getTaskParallelism());
    defaultKeyValue.put(HudiConnectConfig.CONN_TASK_DIVIDE_STRATEGY, sinkConnectConfig.getTaskDivideStrategy());
    defaultKeyValue.put(HudiConnectConfig.CONN_WHITE_LIST, JSONObject.toJSONString(sinkConnectConfig.getWhiteList()));
    defaultKeyValue.put(HudiConnectConfig.CONN_SCHEMA_PATH, sinkConnectConfig.getSchemaPath());
    defaultKeyValue.put(HudiConnectConfig.CONN_TOPIC_ROUTE_INFO, JSONObject.toJSONString(sinkConnectConfig.getTopicRouteMap()));
    log.info("taskConfig : " + defaultKeyValue + ", sinkConnectConfig : " + sinkConnectConfig);
    ITaskDivideStrategy strategy = TaskDivideStrategyFactory.getInstance();
    List<KeyValue> taskConfigs = strategy.divide(defaultKeyValue);
    return taskConfigs;
}

Example 23 with DefaultKeyValue

use of io.openmessaging.internal.DefaultKeyValue in project rocketmq-externals by apache.

the class TaskDivideByQueueStrategy method divide.

@Override
public List<KeyValue> divide(KeyValue source) {
    List<KeyValue> config = new ArrayList<KeyValue>();
    int parallelism = source.getInt(HudiConnectConfig.CONN_TASK_PARALLELISM);
    Map<String, MessageQueue> topicRouteInfos = (Map<String, MessageQueue>) JSONObject.parse(source.getString(HudiConnectConfig.CONN_TOPIC_ROUTE_INFO));
    int id = 0;
    List<List<String>> taskTopicQueues = new ArrayList<>(parallelism);
    for (Map.Entry<String, MessageQueue> topicQueue : topicRouteInfos.entrySet()) {
        MessageQueue messageQueue = topicQueue.getValue();
        String topicQueueStr = messageQueue.getTopic() + "," + messageQueue.getBrokerName() + "," + messageQueue.getQueueId();
        int ind = ++id % parallelism;
        if (taskTopicQueues.get(ind) != null) {
            List<String> taskTopicQueue = new LinkedList<>();
            taskTopicQueue.add(topicQueueStr);
            taskTopicQueues.add(ind, taskTopicQueue);
        } else {
            List<String> taskTopicQueue = taskTopicQueues.get(ind);
            taskTopicQueue.add(topicQueueStr);
        }
    }
    for (int i = 0; i < parallelism; i++) {
        // build single task queue config; format is topicName1,brokerName1,queueId1;topicName1,brokerName1,queueId2
        String singleTaskTopicQueueStr = "";
        List<String> singleTaskTopicQueues = taskTopicQueues.get(i);
        for (String singleTopicQueue : singleTaskTopicQueues) {
            singleTaskTopicQueueStr += singleTopicQueue + ";";
        }
        singleTaskTopicQueueStr = singleTaskTopicQueueStr.substring(0, singleTaskTopicQueueStr.length() - 1);
        // fill connect config;
        KeyValue keyValue = new DefaultKeyValue();
        keyValue.put(HudiConnectConfig.CONN_TOPIC_QUEUES, singleTaskTopicQueueStr);
        keyValue.put(HudiConnectConfig.CONN_HUDI_TABLE_PATH, source.getString(HudiConnectConfig.CONN_HUDI_TABLE_PATH));
        keyValue.put(HudiConnectConfig.CONN_HUDI_TABLE_NAME, source.getString(HudiConnectConfig.CONN_HUDI_TABLE_NAME));
        keyValue.put(HudiConnectConfig.CONN_HUDI_INSERT_SHUFFLE_PARALLELISM, source.getInt(HudiConnectConfig.CONN_HUDI_INSERT_SHUFFLE_PARALLELISM));
        keyValue.put(HudiConnectConfig.CONN_HUDI_UPSERT_SHUFFLE_PARALLELISM, source.getInt(HudiConnectConfig.CONN_HUDI_UPSERT_SHUFFLE_PARALLELISM));
        keyValue.put(HudiConnectConfig.CONN_HUDI_DELETE_PARALLELISM, source.getInt(HudiConnectConfig.CONN_HUDI_DELETE_PARALLELISM));
        keyValue.put(HudiConnectConfig.CONN_SOURCE_RECORD_CONVERTER, source.getString(HudiConnectConfig.CONN_SOURCE_RECORD_CONVERTER));
        keyValue.put(HudiConnectConfig.CONN_SCHEMA_PATH, source.getString(HudiConnectConfig.CONN_SCHEMA_PATH));
        keyValue.put(HudiConnectConfig.CONN_TASK_PARALLELISM, source.getInt(HudiConnectConfig.CONN_TASK_PARALLELISM));
        keyValue.put(HudiConnectConfig.CONN_SCHEMA_PATH, source.getString(HudiConnectConfig.CONN_SCHEMA_PATH));
        config.add(keyValue);
    }
    return config;
}

Example 24 with DefaultKeyValue

use of io.openmessaging.internal.DefaultKeyValue in project rocketmq-externals by apache.

the class DivideTaskByTopic method divideSourceTaskByTopic.

private List<KeyValue> divideSourceTaskByTopic(DbConnectorConfig dbConnectorConfig, TaskDivideConfig tdc) {
    List<KeyValue> config = new ArrayList<KeyValue>();
    int parallelism = tdc.getTaskParallelism();
    int id = -1;
    Map<String, String> topicRouteMap = ((SourceDbConnectorConfig) dbConnectorConfig).getWhiteTopics();
    Map<Integer, Map<String, Map<String, String>>> taskTopicList = new HashMap<>();
    for (Map.Entry<String, String> entry : topicRouteMap.entrySet()) {
        int ind = ++id % parallelism;
        if (!taskTopicList.containsKey(ind)) {
            taskTopicList.put(ind, new HashMap<>());
        }
        String dbKey = entry.getKey().split("-")[0];
        String tableKey = entry.getKey().split("-")[1];
        String filter = entry.getValue();
        Map<String, String> tableMap = new HashMap<>();
        tableMap.put(tableKey, filter);
        if (!taskTopicList.get(ind).containsKey(dbKey)) {
            taskTopicList.get(ind).put(dbKey, tableMap);
        } else {
            taskTopicList.get(ind).get(dbKey).putAll(tableMap);
        }
    }
    for (int i = 0; i < parallelism; i++) {
        KeyValue keyValue = new DefaultKeyValue();
        keyValue.put(Config.CONN_DB_IP, tdc.getDbUrl());
        keyValue.put(Config.CONN_DB_PORT, tdc.getDbPort());
        keyValue.put(Config.CONN_DB_USERNAME, tdc.getDbUserName());
        keyValue.put(Config.CONN_DB_PASSWORD, tdc.getDbPassword());
        keyValue.put(Config.CONN_DB_DATACENTER, tdc.getLocalDataCenter());
        keyValue.put(Config.CONN_WHITE_LIST, JSONObject.toJSONString(taskTopicList.get(i)));
        keyValue.put(Config.CONN_DATA_TYPE, tdc.getDataType());
        keyValue.put(Config.CONN_SOURCE_RECORD_CONVERTER, tdc.getSrcRecordConverter());
        keyValue.put(Config.CONN_DB_MODE, tdc.getMode());
        config.add(keyValue);
    }
    return config;
}

Example 25 with DefaultKeyValue

use of io.openmessaging.internal.DefaultKeyValue in project rocketmq-externals by apache.

the class JdbcSourceConnectorTest method verifyAndSetConfigTest.

@Test
public void verifyAndSetConfigTest() {
    KeyValue keyValue = new DefaultKeyValue();
    for (String requestKey : Config.REQUEST_CONFIG) {
        assertEquals(connector.verifyAndSetConfig(keyValue), "Request config key: " + requestKey);
        keyValue.put(requestKey, requestKey);
    }
    assertEquals(connector.verifyAndSetConfig(keyValue), "");
}