Examples with KeyValue io.openmessaging.KeyValue used on opensource projects

Example 16 with KeyValue

use of io.openmessaging.KeyValue in project rocketmq-externals by apache.

the class WorkerSourceTask method run.

/**
 * Start a source task, and send data entry to MQ cyclically.
 */
@Override
public void run() {
    try {
        producer.start();
        log.info("Source task producer start.");
        state.compareAndSet(WorkerTaskState.NEW, WorkerTaskState.PENDING);
        sourceTask.initialize(new SourceTaskContext() {

            @Override
            public PositionStorageReader positionStorageReader() {
                return positionStorageReader;
            }

            @Override
            public KeyValue configs() {
                return taskConfig;
            }
        });
        sourceTask.start(taskConfig);
        state.compareAndSet(WorkerTaskState.PENDING, WorkerTaskState.RUNNING);
        log.info("Source task start, config:{}", JSON.toJSONString(taskConfig));
        while (WorkerState.STARTED == workerState.get() && WorkerTaskState.RUNNING == state.get()) {
            try {
                Collection<SourceDataEntry> toSendEntries = sourceTask.poll();
                if (null != toSendEntries && toSendEntries.size() > 0) {
                    sendRecord(toSendEntries);
                }
            } catch (Exception e) {
                log.error("Source task runtime exception", e);
                state.set(WorkerTaskState.ERROR);
            }
        }
        sourceTask.stop();
        state.compareAndSet(WorkerTaskState.STOPPING, WorkerTaskState.STOPPED);
        log.info("Source task stop, config:{}", JSON.toJSONString(taskConfig));
    } catch (Exception e) {
        log.error("Run task failed.", e);
        state.set(WorkerTaskState.ERROR);
    } finally {
        if (producer != null) {
            producer.shutdown();
            log.info("Source task producer shutdown.");
        }
    }
}

Example 17 with KeyValue

use of io.openmessaging.KeyValue in project rocketmq-externals by apache.

the class PropertyToObjectUtilsTest method test.

@Test
public void test() {
    User user = new User();
    user.setName("LLL");
    user.setAge(100);
    user.setScore(365);
    user.setAverage(66.66);
    user.setMath(78L);
    user.setHigh(false);
    KeyValue pair = new DefaultKeyValue();
    pair.put("name", "LLL");
    pair.put("age", "100");
    pair.put("score", "365");
    pair.put("average", "66.66");
    pair.put("math", "78");
    pair.put("high", "false");
    User user1 = new User();
    Exception ex = null;
    try {
        PropertyToObjectUtils.properties2Object(pair, user1);
    } catch (InvocationTargetException e) {
        ex = e;
    } catch (IllegalAccessException e) {
        ex = e;
    }
    Assert.assertEquals(user, user1);
    Assert.assertNull(ex);
}

Example 18 with KeyValue

use of io.openmessaging.KeyValue in project rocketmq-externals by apache.

the class WorkerSinkTask method run.

/**
 * Start a sink task, and receive data entry from MQ cyclically.
 */
@Override
public void run() {
    try {
        consumer.start();
        log.info("Sink task consumer start.");
        state.compareAndSet(WorkerTaskState.NEW, WorkerTaskState.PENDING);
        sinkTask.initialize(new SinkTaskContext() {

            @Override
            public void resetOffset(QueueMetaData queueMetaData, Long offset) {
                String shardingKey = queueMetaData.getShardingKey();
                String queueName = queueMetaData.getQueueName();
                if (StringUtils.isNotEmpty(shardingKey) && StringUtils.isNotEmpty(queueName)) {
                    String[] s = shardingKey.split(COMMA);
                    if (s.length == 2 && StringUtils.isNotEmpty(s[0]) && StringUtils.isNotEmpty(s[1])) {
                        String brokerName = s[0];
                        Integer queueId = Integer.valueOf(s[1]);
                        MessageQueue messageQueue = new MessageQueue(queueName, brokerName, queueId);
                        messageQueuesOffsetMap.put(messageQueue, offset);
                        offsetManagementService.putPosition(convertToByteBufferKey(messageQueue), convertToByteBufferValue(offset));
                        return;
                    }
                }
                log.warn("Missing parameters, queueMetaData {}", queueMetaData);
            }

            @Override
            public void resetOffset(Map<QueueMetaData, Long> offsets) {
                for (Map.Entry<QueueMetaData, Long> entry : offsets.entrySet()) {
                    String shardingKey = entry.getKey().getShardingKey();
                    String queueName = entry.getKey().getQueueName();
                    if (StringUtils.isNotEmpty(shardingKey) && StringUtils.isNotEmpty(queueName)) {
                        String[] s = shardingKey.split(COMMA);
                        if (s.length == 2 && StringUtils.isNotEmpty(s[0]) && StringUtils.isNotEmpty(s[1])) {
                            String brokerName = s[0];
                            Integer queueId = Integer.valueOf(s[1]);
                            MessageQueue messageQueue = new MessageQueue(queueName, brokerName, queueId);
                            messageQueuesOffsetMap.put(messageQueue, entry.getValue());
                            offsetManagementService.putPosition(convertToByteBufferKey(messageQueue), convertToByteBufferValue(entry.getValue()));
                            continue;
                        }
                    }
                    log.warn("Missing parameters, queueMetaData {}", entry.getKey());
                }
            }

            @Override
            public void pause(List<QueueMetaData> queueMetaDatas) {
                if (null != queueMetaDatas && queueMetaDatas.size() > 0) {
                    for (QueueMetaData queueMetaData : queueMetaDatas) {
                        String shardingKey = queueMetaData.getShardingKey();
                        String queueName = queueMetaData.getQueueName();
                        if (StringUtils.isNotEmpty(shardingKey) && StringUtils.isNotEmpty(queueName)) {
                            String[] s = shardingKey.split(COMMA);
                            if (s.length == 2 && StringUtils.isNotEmpty(s[0]) && StringUtils.isNotEmpty(s[1])) {
                                String brokerName = s[0];
                                Integer queueId = Integer.valueOf(s[1]);
                                MessageQueue messageQueue = new MessageQueue(queueName, brokerName, queueId);
                                messageQueuesStateMap.put(messageQueue, QueueState.PAUSE);
                                continue;
                            }
                        }
                        log.warn("Missing parameters, queueMetaData {}", queueMetaData);
                    }
                }
            }

            @Override
            public void resume(List<QueueMetaData> queueMetaDatas) {
                if (null != queueMetaDatas && queueMetaDatas.size() > 0) {
                    for (QueueMetaData queueMetaData : queueMetaDatas) {
                        String shardingKey = queueMetaData.getShardingKey();
                        String queueName = queueMetaData.getQueueName();
                        if (StringUtils.isNotEmpty(shardingKey) && StringUtils.isNotEmpty(queueName)) {
                            String[] s = shardingKey.split(COMMA);
                            if (s.length == 2 && StringUtils.isNotEmpty(s[0]) && StringUtils.isNotEmpty(s[1])) {
                                String brokerName = s[0];
                                Integer queueId = Integer.valueOf(s[1]);
                                MessageQueue messageQueue = new MessageQueue(queueName, brokerName, queueId);
                                messageQueuesStateMap.remove(messageQueue);
                                continue;
                            }
                        }
                        log.warn("Missing parameters, queueMetaData {}", queueMetaData);
                    }
                }
            }

            @Override
            public KeyValue configs() {
                return taskConfig;
            }
        });
        String topicNamesStr = taskConfig.getString(QUEUENAMES_CONFIG);
        String topicQueuesStr = taskConfig.getString(TOPIC_QUEUES_CONFIG);
        if (!StringUtils.isEmpty(topicNamesStr)) {
            String[] topicNames = topicNamesStr.split(COMMA);
            for (String topicName : topicNames) {
                final Set<MessageQueue> messageQueues = consumer.fetchSubscribeMessageQueues(topicName);
                for (MessageQueue messageQueue : messageQueues) {
                    final long offset = consumer.searchOffset(messageQueue, TIMEOUT);
                    messageQueuesOffsetMap.put(messageQueue, offset);
                }
                messageQueues.addAll(messageQueues);
            }
            log.debug("{} Initializing and starting task for topicNames {}", this, topicNames);
        } else if (!StringUtils.isEmpty(topicQueuesStr)) {
            String[] topicQueues = topicQueuesStr.split(SEMICOLON);
            for (String messageQueueStr : topicQueues) {
                String[] items = messageQueueStr.split(COMMA);
                if (items.length != 3) {
                    log.error("Topic queue format error, topicQueueStr : " + topicNamesStr);
                    return;
                }
                MessageQueue messageQueue = new MessageQueue(items[0], items[1], Integer.valueOf(items[2]));
                final long offset = consumer.searchOffset(messageQueue, TIMEOUT);
                messageQueuesOffsetMap.put(messageQueue, offset);
            }
        } else {
            log.error("Lack of sink comsume topicNames config");
            state.set(WorkerTaskState.ERROR);
            return;
        }
        for (Map.Entry<MessageQueue, Long> entry : messageQueuesOffsetMap.entrySet()) {
            MessageQueue messageQueue = entry.getKey();
            ByteBuffer byteBuffer = offsetStorageReader.getPosition(convertToByteBufferKey(messageQueue));
            if (null != byteBuffer) {
                messageQueuesOffsetMap.put(messageQueue, convertToOffset(byteBuffer));
            }
        }
        sinkTask.start(taskConfig);
        // we assume executed here means we are safe
        log.info("Sink task start, config:{}", JSON.toJSONString(taskConfig));
        state.compareAndSet(WorkerTaskState.PENDING, WorkerTaskState.RUNNING);
        while (WorkerState.STARTED == workerState.get() && WorkerTaskState.RUNNING == state.get()) {
            // this method can block up to 3 minutes long
            pullMessageFromQueues();
        }
        sinkTask.stop();
        state.compareAndSet(WorkerTaskState.STOPPING, WorkerTaskState.STOPPED);
        log.info("Sink task stop, config:{}", JSON.toJSONString(taskConfig));
    } catch (Exception e) {
        log.error("Run task failed.", e);
        state.set(WorkerTaskState.ERROR);
    } finally {
        if (consumer != null) {
            consumer.shutdown();
            log.info("Sink task consumer shutdown.");
        }
    }
}

Example 19 with KeyValue

use of io.openmessaging.KeyValue in project rocketmq-externals by apache.

the class Utils method groupPartitions.

public static List<KeyValue> groupPartitions(List<String> elements, int numGroups, RmqConnectorConfig tdc) {
    if (numGroups <= 0)
        throw new IllegalArgumentException("Number of groups must be positive.");
    List<KeyValue> result = new ArrayList<>(numGroups);
    // Each group has either n+1 or n raw partitions
    int perGroup = elements.size() / numGroups;
    int leftover = elements.size() - (numGroups * perGroup);
    int assigned = 0;
    for (int group = 0; group < numGroups; group++) {
        int numThisGroup = group < leftover ? perGroup + 1 : perGroup;
        KeyValue keyValue = new DefaultKeyValue();
        List<String> groupList = new ArrayList<>();
        for (int i = 0; i < numThisGroup; i++) {
            groupList.add(elements.get(assigned));
            assigned++;
        }
        keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSrcNamesrvs());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_CLUSTER.getKey(), tdc.getSrcCluster());
        keyValue.put(TaskConfigEnum.TASK_OFFSET_SYNC_TOPIC.getKey(), tdc.getOffsetSyncTopic());
        keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.OFFSET.ordinal());
        keyValue.put(TaskConfigEnum.TASK_GROUP_INFO.getKey(), JSONObject.toJSONString(groupList));
        keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getConverter());
        result.add(keyValue);
        log.debug("allocate group partition: {}", keyValue);
    }
    return result;
}

Example 20 with KeyValue

use of io.openmessaging.KeyValue in project rocketmq-externals by apache.

the class DivideTaskByConsistentHash method divide.

@Override
public List<KeyValue> divide(Map<String, Set<TaskTopicInfo>> topicMap, TaskDivideConfig tdc) {
    List<KeyValue> config = new ArrayList<>();
    int parallelism = tdc.getTaskParallelism();
    Map<Integer, List<TaskTopicInfo>> queueTopicList = new HashMap<>();
    int id = -1;
    Collection<ClientNode> cidNodes = new ArrayList<>();
    for (int i = 0; i < parallelism; i++) {
        cidNodes.add(new ClientNode(i, Integer.toString(i)));
        queueTopicList.put(i, new ArrayList<>());
    }
    ConsistentHashRouter<ClientNode> router = new ConsistentHashRouter<>(cidNodes, cidNodes.size());
    for (String t : topicMap.keySet()) {
        for (TaskTopicInfo queue : topicMap.get(t)) {
            ClientNode clientNode = router.routeNode(queue.toString());
            if (clientNode != null) {
                queueTopicList.get(clientNode.index).add(queue);
            }
        }
    }
    for (int i = 0; i < parallelism; i++) {
        KeyValue keyValue = new DefaultKeyValue();
        keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
        keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
        keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(queueTopicList.get(i)));
        keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ACL_ENABLE.getKey(), String.valueOf(tdc.isSrcAclEnable()));
        keyValue.put(TaskConfigEnum.TASK_SOURCE_ACCESS_KEY.getKey(), tdc.getSrcAccessKey());
        keyValue.put(TaskConfigEnum.TASK_SOURCE_SECRET_KEY.getKey(), tdc.getSrcSecretKey());
        config.add(keyValue);
    }
    return config;
}