Examples with TaskTopicInfo org.apache.rocketmq.replicator.config.TaskTopicInfo used on opensource projects

Example 1 with TaskTopicInfo

use of org.apache.rocketmq.replicator.config.TaskTopicInfo in project rocketmq-externals by apache.

the class RmqSourceTask method start.

@Override
public void start(KeyValue config) {
    ConfigUtil.load(config, this.config);
    RPCHook rpcHook = null;
    if (this.config.isSrcAclEnable()) {
        rpcHook = new AclClientRPCHook(new SessionCredentials(this.config.getSrcAccessKey(), this.config.getSrcSecretKey()));
    }
    this.consumer = new DefaultMQPullConsumer(rpcHook);
    this.consumer.setConsumerGroup(this.taskId);
    this.consumer.setNamesrvAddr(this.config.getSourceRocketmq());
    this.consumer.setInstanceName(Utils.createInstanceName(this.config.getSourceRocketmq()));
    List<TaskTopicInfo> topicList = JSONObject.parseArray(this.config.getTaskTopicList(), TaskTopicInfo.class);
    try {
        if (topicList == null) {
            throw new IllegalStateException("topicList is null");
        }
        this.consumer.start();
        List<TaskTopicInfo> topicListFilter = new ArrayList<>();
        for (TaskTopicInfo tti : topicList) {
            Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues(tti.getTopic());
            for (MessageQueue mq : mqs) {
                if (tti.getBrokerName().equals(mq.getBrokerName()) && tti.getQueueId() == mq.getQueueId()) {
                    topicListFilter.add(tti);
                    break;
                }
            }
        }
        PositionStorageReader positionStorageReader = this.context.positionStorageReader();
        mqOffsetMap.putAll(getPositionMapWithCheck(topicListFilter, positionStorageReader, this.TIMEOUT, TimeUnit.MILLISECONDS));
        started = true;
    } catch (Exception e) {
        log.error("Consumer of task {} start failed.", this.taskId, e);
        throw new IllegalStateException(String.format("Consumer of task %s start failed.", this.taskId));
    }
    log.info("RocketMQ source task started");
}

Example 2 with TaskTopicInfo

use of org.apache.rocketmq.replicator.config.TaskTopicInfo in project rocketmq-externals by apache.

the class RmqSourceTask method getPositionMap.

public Map<TaskTopicInfo, Long> getPositionMap(List<TaskTopicInfo> taskList, PositionStorageReader positionStorageReader) {
    Map<TaskTopicInfo, Long> positionMap = new HashMap<>();
    for (TaskTopicInfo tti : taskList) {
        ByteBuffer positionInfo = positionStorageReader.getPosition(ByteBuffer.wrap(RmqConstants.getPartition(tti.getTopic(), tti.getBrokerName(), String.valueOf(tti.getQueueId())).getBytes(StandardCharsets.UTF_8)));
        if (null != positionInfo && positionInfo.array().length > 0) {
            String positionJson = new String(positionInfo.array(), StandardCharsets.UTF_8);
            JSONObject jsonObject = JSONObject.parseObject(positionJson);
            positionMap.put(tti, jsonObject.getLong(RmqConstants.NEXT_POSITION));
        } else {
            positionMap.put(tti, 0L);
        }
    }
    return positionMap;
}

Example 3 with TaskTopicInfo

use of org.apache.rocketmq.replicator.config.TaskTopicInfo in project rocketmq-externals by apache.

the class RmqSourceReplicator method buildRoute.

public void buildRoute() {
    List<Pattern> patterns = new ArrayList<Pattern>();
    String srcCluster = this.replicatorConfig.getSrcCluster();
    try {
        Set<String> targetTopicSet = fetchTargetTopics();
        for (String topic : this.replicatorConfig.getWhiteList()) {
            Pattern pattern = Pattern.compile(topic);
            patterns.add(pattern);
        }
        TopicList topics = srcMQAdminExt.fetchAllTopicList();
        for (String topic : topics.getTopicList()) {
            if (topic.equals(ConfigDefine.CONN_STORE_TOPIC)) {
                continue;
            }
            if (!syncRETRY && topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
                continue;
            }
            if (!syncDLQ && topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)) {
                continue;
            }
            for (Pattern pattern : patterns) {
                Matcher matcher = pattern.matcher(topic);
                if (matcher.matches()) {
                    String targetTopic = generateTargetTopic(topic);
                    if (!targetTopicSet.contains(targetTopic)) {
                        ensureTargetTopic(topic, targetTopic);
                    }
                    // different from BrokerData with cluster field, which can ensure the brokerData is from expected cluster.
                    // QueueData use brokerName as unique info on cluster of rocketmq. so when we want to get QueueData of
                    // expected cluster, we should get brokerNames of expected cluster, and then filter queueDatas.
                    List<BrokerData> brokerList = Utils.examineBrokerData(this.srcMQAdminExt, topic, srcCluster);
                    Set<String> brokerNameSet = new HashSet<String>();
                    for (BrokerData b : brokerList) {
                        brokerNameSet.add(b.getBrokerName());
                    }
                    TopicRouteData topicRouteData = srcMQAdminExt.examineTopicRouteInfo(topic);
                    if (!topicRouteMap.containsKey(topic)) {
                        topicRouteMap.put(topic, new HashSet<>(16));
                    }
                    for (QueueData qd : topicRouteData.getQueueDatas()) {
                        if (brokerNameSet.contains(qd.getBrokerName())) {
                            for (int i = 0; i < qd.getReadQueueNums(); i++) {
                                TaskTopicInfo taskTopicInfo = new TaskTopicInfo(topic, qd.getBrokerName(), i, targetTopic);
                                topicRouteMap.get(topic).add(taskTopicInfo);
                            }
                        }
                    }
                }
            }
        }
    } catch (Exception e) {
        log.error("Fetch topic list error.", e);
    }
}

Example 4 with TaskTopicInfo

use of org.apache.rocketmq.replicator.config.TaskTopicInfo in project rocketmq-externals by apache.

the class DivideTaskByConsistentHash method divide.

@Override
public List<KeyValue> divide(Map<String, Set<TaskTopicInfo>> topicMap, TaskDivideConfig tdc) {
    List<KeyValue> config = new ArrayList<>();
    int parallelism = tdc.getTaskParallelism();
    Map<Integer, List<TaskTopicInfo>> queueTopicList = new HashMap<>();
    int id = -1;
    Collection<ClientNode> cidNodes = new ArrayList<>();
    for (int i = 0; i < parallelism; i++) {
        cidNodes.add(new ClientNode(i, Integer.toString(i)));
        queueTopicList.put(i, new ArrayList<>());
    }
    ConsistentHashRouter<ClientNode> router = new ConsistentHashRouter<>(cidNodes, cidNodes.size());
    for (String t : topicMap.keySet()) {
        for (TaskTopicInfo queue : topicMap.get(t)) {
            ClientNode clientNode = router.routeNode(queue.toString());
            if (clientNode != null) {
                queueTopicList.get(clientNode.index).add(queue);
            }
        }
    }
    for (int i = 0; i < parallelism; i++) {
        KeyValue keyValue = new DefaultKeyValue();
        keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
        keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
        keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(queueTopicList.get(i)));
        keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ACL_ENABLE.getKey(), String.valueOf(tdc.isSrcAclEnable()));
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ACCESS_KEY.getKey(), tdc.getSrcAccessKey());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_SECRET_KEY.getKey(), tdc.getSrcSecretKey());
        config.add(keyValue);
    }
    return config;
}

Example 5 with TaskTopicInfo

use of org.apache.rocketmq.replicator.config.TaskTopicInfo in project rocketmq-externals by apache.

the class DivideTaskByQueue method divide.

@Override
public List<KeyValue> divide(Map<String, Set<TaskTopicInfo>> topicRouteMap, TaskDivideConfig tdc) {
    List<KeyValue> config = new ArrayList<KeyValue>();
    int parallelism = tdc.getTaskParallelism();
    Map<Integer, List<TaskTopicInfo>> queueTopicList = new HashMap<Integer, List<TaskTopicInfo>>();
    int id = -1;
    for (String t : topicRouteMap.keySet()) {
        for (TaskTopicInfo taskTopicInfo : topicRouteMap.get(t)) {
            int ind = ++id % parallelism;
            if (!queueTopicList.containsKey(ind)) {
                queueTopicList.put(ind, new ArrayList<TaskTopicInfo>());
            }
            queueTopicList.get(ind).add(taskTopicInfo);
        }
    }
    for (int i = 0; i < parallelism; i++) {
        KeyValue keyValue = new DefaultKeyValue();
        keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
        keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
        keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(queueTopicList.get(i)));
        keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ACL_ENABLE.getKey(), String.valueOf(tdc.isSrcAclEnable()));
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ACCESS_KEY.getKey(), tdc.getSrcAccessKey());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_SECRET_KEY.getKey(), tdc.getSrcSecretKey());
        config.add(keyValue);
    }
    return config;
}