use of io.openmessaging.connector.api.sink.SinkTaskContext in project rocketmq-externals by apache.
the class WorkerDirectTask method starkSinkTask.
private void starkSinkTask() {
sinkTask.initialize(new SinkTaskContext() {
@Override
public KeyValue configs() {
return taskConfig;
}
@Override
public void resetOffset(QueueMetaData queueMetaData, Long offset) {
}
@Override
public void resetOffset(Map<QueueMetaData, Long> offsets) {
}
@Override
public void pause(List<QueueMetaData> queues) {
}
@Override
public void resume(List<QueueMetaData> queues) {
}
});
sinkTask.start(taskConfig);
log.info("Sink task start, config:{}", JSON.toJSONString(taskConfig));
}
use of io.openmessaging.connector.api.sink.SinkTaskContext in project rocketmq-externals by apache.
the class WorkerSinkTask method run.
/**
* Start a sink task, and receive data entry from MQ cyclically.
*/
@Override
public void run() {
try {
consumer.start();
log.info("Sink task consumer start.");
state.compareAndSet(WorkerTaskState.NEW, WorkerTaskState.PENDING);
sinkTask.initialize(new SinkTaskContext() {
@Override
public void resetOffset(QueueMetaData queueMetaData, Long offset) {
String shardingKey = queueMetaData.getShardingKey();
String queueName = queueMetaData.getQueueName();
if (StringUtils.isNotEmpty(shardingKey) && StringUtils.isNotEmpty(queueName)) {
String[] s = shardingKey.split(COMMA);
if (s.length == 2 && StringUtils.isNotEmpty(s[0]) && StringUtils.isNotEmpty(s[1])) {
String brokerName = s[0];
Integer queueId = Integer.valueOf(s[1]);
MessageQueue messageQueue = new MessageQueue(queueName, brokerName, queueId);
messageQueuesOffsetMap.put(messageQueue, offset);
offsetManagementService.putPosition(convertToByteBufferKey(messageQueue), convertToByteBufferValue(offset));
return;
}
}
log.warn("Missing parameters, queueMetaData {}", queueMetaData);
}
@Override
public void resetOffset(Map<QueueMetaData, Long> offsets) {
for (Map.Entry<QueueMetaData, Long> entry : offsets.entrySet()) {
String shardingKey = entry.getKey().getShardingKey();
String queueName = entry.getKey().getQueueName();
if (StringUtils.isNotEmpty(shardingKey) && StringUtils.isNotEmpty(queueName)) {
String[] s = shardingKey.split(COMMA);
if (s.length == 2 && StringUtils.isNotEmpty(s[0]) && StringUtils.isNotEmpty(s[1])) {
String brokerName = s[0];
Integer queueId = Integer.valueOf(s[1]);
MessageQueue messageQueue = new MessageQueue(queueName, brokerName, queueId);
messageQueuesOffsetMap.put(messageQueue, entry.getValue());
offsetManagementService.putPosition(convertToByteBufferKey(messageQueue), convertToByteBufferValue(entry.getValue()));
continue;
}
}
log.warn("Missing parameters, queueMetaData {}", entry.getKey());
}
}
@Override
public void pause(List<QueueMetaData> queueMetaDatas) {
if (null != queueMetaDatas && queueMetaDatas.size() > 0) {
for (QueueMetaData queueMetaData : queueMetaDatas) {
String shardingKey = queueMetaData.getShardingKey();
String queueName = queueMetaData.getQueueName();
if (StringUtils.isNotEmpty(shardingKey) && StringUtils.isNotEmpty(queueName)) {
String[] s = shardingKey.split(COMMA);
if (s.length == 2 && StringUtils.isNotEmpty(s[0]) && StringUtils.isNotEmpty(s[1])) {
String brokerName = s[0];
Integer queueId = Integer.valueOf(s[1]);
MessageQueue messageQueue = new MessageQueue(queueName, brokerName, queueId);
messageQueuesStateMap.put(messageQueue, QueueState.PAUSE);
continue;
}
}
log.warn("Missing parameters, queueMetaData {}", queueMetaData);
}
}
}
@Override
public void resume(List<QueueMetaData> queueMetaDatas) {
if (null != queueMetaDatas && queueMetaDatas.size() > 0) {
for (QueueMetaData queueMetaData : queueMetaDatas) {
String shardingKey = queueMetaData.getShardingKey();
String queueName = queueMetaData.getQueueName();
if (StringUtils.isNotEmpty(shardingKey) && StringUtils.isNotEmpty(queueName)) {
String[] s = shardingKey.split(COMMA);
if (s.length == 2 && StringUtils.isNotEmpty(s[0]) && StringUtils.isNotEmpty(s[1])) {
String brokerName = s[0];
Integer queueId = Integer.valueOf(s[1]);
MessageQueue messageQueue = new MessageQueue(queueName, brokerName, queueId);
messageQueuesStateMap.remove(messageQueue);
continue;
}
}
log.warn("Missing parameters, queueMetaData {}", queueMetaData);
}
}
}
@Override
public KeyValue configs() {
return taskConfig;
}
});
String topicNamesStr = taskConfig.getString(QUEUENAMES_CONFIG);
String topicQueuesStr = taskConfig.getString(TOPIC_QUEUES_CONFIG);
if (!StringUtils.isEmpty(topicNamesStr)) {
String[] topicNames = topicNamesStr.split(COMMA);
for (String topicName : topicNames) {
final Set<MessageQueue> messageQueues = consumer.fetchSubscribeMessageQueues(topicName);
for (MessageQueue messageQueue : messageQueues) {
final long offset = consumer.searchOffset(messageQueue, TIMEOUT);
messageQueuesOffsetMap.put(messageQueue, offset);
}
messageQueues.addAll(messageQueues);
}
log.debug("{} Initializing and starting task for topicNames {}", this, topicNames);
} else if (!StringUtils.isEmpty(topicQueuesStr)) {
String[] topicQueues = topicQueuesStr.split(SEMICOLON);
for (String messageQueueStr : topicQueues) {
String[] items = messageQueueStr.split(COMMA);
if (items.length != 3) {
log.error("Topic queue format error, topicQueueStr : " + topicNamesStr);
return;
}
MessageQueue messageQueue = new MessageQueue(items[0], items[1], Integer.valueOf(items[2]));
final long offset = consumer.searchOffset(messageQueue, TIMEOUT);
messageQueuesOffsetMap.put(messageQueue, offset);
}
} else {
log.error("Lack of sink comsume topicNames config");
state.set(WorkerTaskState.ERROR);
return;
}
for (Map.Entry<MessageQueue, Long> entry : messageQueuesOffsetMap.entrySet()) {
MessageQueue messageQueue = entry.getKey();
ByteBuffer byteBuffer = offsetStorageReader.getPosition(convertToByteBufferKey(messageQueue));
if (null != byteBuffer) {
messageQueuesOffsetMap.put(messageQueue, convertToOffset(byteBuffer));
}
}
sinkTask.start(taskConfig);
// we assume executed here means we are safe
log.info("Sink task start, config:{}", JSON.toJSONString(taskConfig));
state.compareAndSet(WorkerTaskState.PENDING, WorkerTaskState.RUNNING);
while (WorkerState.STARTED == workerState.get() && WorkerTaskState.RUNNING == state.get()) {
// this method can block up to 3 minutes long
pullMessageFromQueues();
}
sinkTask.stop();
state.compareAndSet(WorkerTaskState.STOPPING, WorkerTaskState.STOPPED);
log.info("Sink task stop, config:{}", JSON.toJSONString(taskConfig));
} catch (Exception e) {
log.error("Run task failed.", e);
state.set(WorkerTaskState.ERROR);
} finally {
if (consumer != null) {
consumer.shutdown();
log.info("Sink task consumer shutdown.");
}
}
}
Aggregations