Examples with PositionStorageReader io.openmessaging.connector.api.PositionStorageReader used on opensource projects

Example 1 with PositionStorageReader

use of io.openmessaging.connector.api.PositionStorageReader in project rocketmq-externals by apache.

the class RedisSourceTaskTest method initAndStartTask.

@Before
public void initAndStartTask() {
    try {
        initKeyValue();
        this.task = new RedisSourceTask();
        this.task.initialize(new SourceTaskContext() {

            @Override
            public PositionStorageReader positionStorageReader() {
                return new PositionStorageReader() {

                    @Override
                    public ByteBuffer getPosition(ByteBuffer byteBuffer) {
                        return null;
                    }

                    @Override
                    public Map<ByteBuffer, ByteBuffer> getPositions(Collection<ByteBuffer> collection) {
                        return null;
                    }
                };
            }

            @Override
            public KeyValue configs() {
                return keyValue;
            }
        });
        this.task.start(this.keyValue);
    } catch (JedisConnectionException e) {
    }
}

Example 2 with PositionStorageReader

use of io.openmessaging.connector.api.PositionStorageReader in project rocketmq-externals by apache.

the class WorkerSourceTask method run.

/**
 * Start a source task, and send data entry to MQ cyclically.
 */
@Override
public void run() {
    try {
        producer.start();
        log.info("Source task producer start.");
        state.compareAndSet(WorkerTaskState.NEW, WorkerTaskState.PENDING);
        sourceTask.initialize(new SourceTaskContext() {

            @Override
            public PositionStorageReader positionStorageReader() {
                return positionStorageReader;
            }

            @Override
            public KeyValue configs() {
                return taskConfig;
            }
        });
        sourceTask.start(taskConfig);
        state.compareAndSet(WorkerTaskState.PENDING, WorkerTaskState.RUNNING);
        log.info("Source task start, config:{}", JSON.toJSONString(taskConfig));
        while (WorkerState.STARTED == workerState.get() && WorkerTaskState.RUNNING == state.get()) {
            try {
                Collection<SourceDataEntry> toSendEntries = sourceTask.poll();
                if (null != toSendEntries && toSendEntries.size() > 0) {
                    sendRecord(toSendEntries);
                }
            } catch (Exception e) {
                log.error("Source task runtime exception", e);
                state.set(WorkerTaskState.ERROR);
            }
        }
        sourceTask.stop();
        state.compareAndSet(WorkerTaskState.STOPPING, WorkerTaskState.STOPPED);
        log.info("Source task stop, config:{}", JSON.toJSONString(taskConfig));
    } catch (Exception e) {
        log.error("Run task failed.", e);
        state.set(WorkerTaskState.ERROR);
    } finally {
        if (producer != null) {
            producer.shutdown();
            log.info("Source task producer shutdown.");
        }
    }
}

Example 3 with PositionStorageReader

use of io.openmessaging.connector.api.PositionStorageReader in project rocketmq-externals by apache.

the class RmqSourceTask method start.

@Override
public void start(KeyValue config) {
    ConfigUtil.load(config, this.config);
    RPCHook rpcHook = null;
    if (this.config.isSrcAclEnable()) {
        rpcHook = new AclClientRPCHook(new SessionCredentials(this.config.getSrcAccessKey(), this.config.getSrcSecretKey()));
    }
    this.consumer = new DefaultMQPullConsumer(rpcHook);
    this.consumer.setConsumerGroup(this.taskId);
    this.consumer.setNamesrvAddr(this.config.getSourceRocketmq());
    this.consumer.setInstanceName(Utils.createInstanceName(this.config.getSourceRocketmq()));
    List<TaskTopicInfo> topicList = JSONObject.parseArray(this.config.getTaskTopicList(), TaskTopicInfo.class);
    try {
        if (topicList == null) {
            throw new IllegalStateException("topicList is null");
        }
        this.consumer.start();
        List<TaskTopicInfo> topicListFilter = new ArrayList<>();
        for (TaskTopicInfo tti : topicList) {
            Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues(tti.getTopic());
            for (MessageQueue mq : mqs) {
                if (tti.getBrokerName().equals(mq.getBrokerName()) && tti.getQueueId() == mq.getQueueId()) {
                    topicListFilter.add(tti);
                    break;
                }
            }
        }
        PositionStorageReader positionStorageReader = this.context.positionStorageReader();
        mqOffsetMap.putAll(getPositionMapWithCheck(topicListFilter, positionStorageReader, this.TIMEOUT, TimeUnit.MILLISECONDS));
        started = true;
    } catch (Exception e) {
        log.error("Consumer of task {} start failed.", this.taskId, e);
        throw new IllegalStateException(String.format("Consumer of task %s start failed.", this.taskId));
    }
    log.info("RocketMQ source task started");
}

Example 4 with PositionStorageReader

use of io.openmessaging.connector.api.PositionStorageReader in project rocketmq-externals by apache.

the class WorkerDirectTask method startSourceTask.

private void startSourceTask() {
    state.compareAndSet(WorkerTaskState.NEW, WorkerTaskState.PENDING);
    sourceTask.initialize(new SourceTaskContext() {

        @Override
        public PositionStorageReader positionStorageReader() {
            return positionStorageReader;
        }

        @Override
        public KeyValue configs() {
            return taskConfig;
        }
    });
    sourceTask.start(taskConfig);
    state.compareAndSet(WorkerTaskState.PENDING, WorkerTaskState.RUNNING);
    log.info("Source task start, config:{}", JSON.toJSONString(taskConfig));
}