use of io.openmessaging.connector.api.data.Schema in project rocketmq-externals by apache.
the class RedisEntryConverter method getRedisSchema.
private Schema getRedisSchema(FieldType valueType) {
Schema schema = new Schema();
schema.setDataSource(Options.REDIS_DATASOURCE.name());
List<Field> fields = new ArrayList<>();
fields.add(new Field(0, Options.REDIS_COMMAND.name(), FieldType.STRING));
fields.add(new Field(1, Options.REDIS_KEY.name(), FieldType.STRING));
fields.add(new Field(2, Options.REDIS_VALUE.name(), valueType));
fields.add(new Field(3, Options.REDIS_PARAMS.name(), FieldType.MAP));
schema.setFields(fields);
return schema;
}
use of io.openmessaging.connector.api.data.Schema in project rocketmq-externals by apache.
the class WorkerSourceTask method sendRecord.
/**
* Send list of sourceDataEntries to MQ.
*
* @param sourceDataEntries
*/
private void sendRecord(Collection<SourceDataEntry> sourceDataEntries) {
for (SourceDataEntry sourceDataEntry : sourceDataEntries) {
ByteBuffer partition = sourceDataEntry.getSourcePartition();
Optional<ByteBuffer> opartition = Optional.ofNullable(partition);
ByteBuffer position = sourceDataEntry.getSourcePosition();
Optional<ByteBuffer> oposition = Optional.ofNullable(position);
sourceDataEntry.setSourcePartition(null);
sourceDataEntry.setSourcePosition(null);
Message sourceMessage = new Message();
sourceMessage.setTopic(sourceDataEntry.getQueueName());
if (null == recordConverter || recordConverter instanceof RocketMQConverter) {
if (StringUtils.isNotEmpty(sourceDataEntry.getShardingKey())) {
MessageAccessor.putProperty(sourceMessage, RuntimeConfigDefine.CONNECT_SHARDINGKEY, sourceDataEntry.getShardingKey());
}
if (StringUtils.isNotEmpty(sourceDataEntry.getQueueName())) {
MessageAccessor.putProperty(sourceMessage, RuntimeConfigDefine.CONNECT_TOPICNAME, sourceDataEntry.getQueueName());
}
if (opartition.isPresent()) {
MessageAccessor.putProperty(sourceMessage, RuntimeConfigDefine.CONNECT_SOURCE_PARTITION, new String(opartition.get().array()));
}
if (oposition.isPresent()) {
MessageAccessor.putProperty(sourceMessage, RuntimeConfigDefine.CONNECT_SOURCE_POSITION, new String(oposition.get().array()));
}
EntryType entryType = sourceDataEntry.getEntryType();
Optional<EntryType> oentryType = Optional.ofNullable(entryType);
if (oentryType.isPresent()) {
MessageAccessor.putProperty(sourceMessage, RuntimeConfigDefine.CONNECT_ENTRYTYPE, oentryType.get().name());
}
Long timestamp = sourceDataEntry.getTimestamp();
Optional<Long> otimestamp = Optional.ofNullable(timestamp);
if (otimestamp.isPresent()) {
MessageAccessor.putProperty(sourceMessage, RuntimeConfigDefine.CONNECT_TIMESTAMP, otimestamp.get().toString());
}
Schema schema = sourceDataEntry.getSchema();
Optional<Schema> oschema = Optional.ofNullable(schema);
if (oschema.isPresent()) {
MessageAccessor.putProperty(sourceMessage, RuntimeConfigDefine.CONNECT_SCHEMA, JSON.toJSONString(oschema.get()));
}
Object[] payload = sourceDataEntry.getPayload();
if (null != payload && null != payload[0]) {
Object object = payload[0];
final byte[] messageBody = (String.valueOf(object)).getBytes();
if (messageBody.length > RuntimeConfigDefine.MAX_MESSAGE_SIZE) {
log.error("Send record, message size is greater than {} bytes, payload: {}", RuntimeConfigDefine.MAX_MESSAGE_SIZE, sourceDataEntry.getPayload());
return;
}
sourceMessage.setBody(messageBody);
}
} else {
byte[] payload = recordConverter.objectToByte(sourceDataEntry.getPayload());
Object[] newPayload = new Object[1];
newPayload[0] = Base64.getEncoder().encodeToString(payload);
sourceDataEntry.setPayload(newPayload);
final byte[] messageBody = JSON.toJSONString(sourceDataEntry).getBytes();
if (messageBody.length > RuntimeConfigDefine.MAX_MESSAGE_SIZE) {
log.error("Send record, message size is greater than {} bytes, payload: {}", RuntimeConfigDefine.MAX_MESSAGE_SIZE, sourceDataEntry.getPayload());
return;
}
sourceMessage.setBody(messageBody);
}
try {
producer.send(sourceMessage, new SendCallback() {
@Override
public void onSuccess(org.apache.rocketmq.client.producer.SendResult result) {
log.info("Successful send message to RocketMQ:{}", result.getMsgId());
try {
if (null != partition && null != position) {
positionManagementService.putPosition(partition, position);
}
} catch (Exception e) {
log.error("Source task save position info failed.", e);
}
}
@Override
public void onException(Throwable throwable) {
if (null != throwable) {
log.error("Source task send record failed {}.", throwable);
}
}
});
} catch (MQClientException e) {
log.error("Send message error. message: {}, error info: {}.", sourceMessage, e);
} catch (RemotingException e) {
log.error("Send message error. message: {}, error info: {}.", sourceMessage, e);
} catch (InterruptedException e) {
log.error("Send message error. message: {}, error info: {}.", sourceMessage, e);
}
}
}
use of io.openmessaging.connector.api.data.Schema in project rocketmq-externals by apache.
the class MetaSourceTask method poll.
@Override
public Collection<SourceDataEntry> poll() {
log.debug("polling...");
List<String> groups = JSONObject.parseArray(this.config.getTaskGroupList(), String.class);
if (groups == null) {
log.info("no group in task.");
try {
Thread.sleep(TimeUnit.SECONDS.toMillis(10));
} catch (InterruptedException e) {
throw new IllegalStateException(e);
}
return Collections.emptyList();
}
List<SourceDataEntry> res = new ArrayList<>();
for (String group : groups) {
ConsumeStats stats;
try {
stats = this.srcMQAdminExt.examineConsumeStats(group);
} catch (Exception e) {
log.error("admin get consumer info failed for consumer groups: " + group, e);
continue;
}
for (Map.Entry<MessageQueue, OffsetWrapper> offsetTable : stats.getOffsetTable().entrySet()) {
MessageQueue mq = offsetTable.getKey();
long srcOffset = offsetTable.getValue().getConsumerOffset();
long targetOffset = this.store.convertTargetOffset(mq, group, srcOffset);
JSONObject jsonObject = new JSONObject();
jsonObject.put(RmqConstants.NEXT_POSITION, srcOffset);
Schema schema = new Schema();
schema.setDataSource(this.config.getSourceRocketmq());
schema.setName(mq.getTopic());
schema.setFields(new ArrayList<>());
schema.getFields().add(new Field(0, FieldName.OFFSET.getKey(), FieldType.INT64));
DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
dataEntryBuilder.timestamp(System.currentTimeMillis()).queue(this.config.getStoreTopic()).entryType(EntryType.UPDATE);
dataEntryBuilder.putFiled(FieldName.OFFSET.getKey(), targetOffset);
SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(ByteBuffer.wrap(RmqConstants.getPartition(mq.getTopic(), mq.getBrokerName(), String.valueOf(mq.getQueueId())).getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(jsonObject.toJSONString().getBytes(StandardCharsets.UTF_8)));
String targetTopic = new StringBuilder().append(group).append("-").append(mq.getTopic()).append("-").append(mq.getQueueId()).toString();
sourceDataEntry.setQueueName(targetTopic);
res.add(sourceDataEntry);
}
}
return res;
}
use of io.openmessaging.connector.api.data.Schema in project rocketmq-externals by apache.
the class FileSourceTask method poll.
@Override
public Collection<SourceDataEntry> poll() {
log.info("Start a poll stream is null:{}", stream == null);
if (stream == null) {
try {
stream = Files.newInputStream(Paths.get(fileConfig.getFilename()));
ByteBuffer positionInfo;
positionInfo = this.context.positionStorageReader().getPosition(ByteBuffer.wrap(FileConstants.getPartition(fileConfig.getFilename()).getBytes(Charset.defaultCharset())));
if (positionInfo != null) {
log.info("positionInfo is not null!");
String positionJson = new String(positionInfo.array(), Charset.defaultCharset());
JSONObject jsonObject = JSONObject.parseObject(positionJson);
Object lastRecordedOffset = jsonObject.getLong(FileConstants.NEXT_POSITION);
if (lastRecordedOffset != null && !(lastRecordedOffset instanceof Long))
throw new ConnectException(-1, "Offset position is the incorrect type");
if (lastRecordedOffset != null) {
log.debug("Found previous offset, trying to skip to file offset {}", lastRecordedOffset);
long skipLeft = (Long) lastRecordedOffset;
while (skipLeft > 0) {
try {
long skipped = stream.skip(skipLeft);
skipLeft -= skipped;
} catch (IOException e) {
log.error("Error while trying to seek to previous offset in file {}: ", fileConfig.getFilename(), e);
throw new ConnectException(-1, e);
}
}
log.debug("Skipped to offset {}", lastRecordedOffset);
}
streamOffset = (lastRecordedOffset != null) ? (Long) lastRecordedOffset : 0L;
} else {
log.info("positionInfo is null!");
streamOffset = 0L;
}
reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8));
log.debug("Opened {} for reading", logFilename());
} catch (NoSuchFileException e) {
log.warn("Couldn't find file {} for FileStreamSourceTask, sleeping to wait for it to be created", logFilename());
synchronized (this) {
try {
this.wait(1000);
} catch (InterruptedException e1) {
log.error("Interrupt error .", e1);
}
}
return null;
} catch (IOException e) {
log.error("Error while trying to open file {}: ", fileConfig.getFilename(), e);
throw new ConnectException(-1, e);
}
}
try {
final BufferedReader readerCopy;
synchronized (this) {
readerCopy = reader;
}
if (readerCopy == null) {
return null;
}
Collection<SourceDataEntry> records = null;
int nread = 0;
while (readerCopy.ready()) {
nread = readerCopy.read(buffer, offset, buffer.length - offset);
log.trace("Read {} bytes from {}", nread, logFilename());
if (nread > 0) {
offset += nread;
if (offset == buffer.length) {
char[] newbuf = new char[buffer.length * 2];
System.arraycopy(buffer, 0, newbuf, 0, buffer.length);
buffer = newbuf;
}
String line;
do {
line = extractLine();
if (line != null) {
log.trace("Read a line from {}", logFilename());
if (records == null) {
records = new ArrayList<>();
}
Schema schema = new Schema();
schema.setDataSource(fileConfig.getFilename());
schema.setName(fileConfig.getFilename() + LINE);
final Field field = new Field(0, FileConstants.FILE_LINE_CONTENT, FieldType.STRING);
List<Field> fields = new ArrayList<Field>() {
{
add(field);
}
};
schema.setFields(fields);
DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema).entryType(EntryType.CREATE).queue(fileConfig.getTopic()).timestamp(System.currentTimeMillis()).putFiled(FileConstants.FILE_LINE_CONTENT, line);
final SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(offsetKey(FileConstants.getPartition(fileConfig.getFilename())), offsetValue(streamOffset));
records.add(sourceDataEntry);
if (records.size() >= batchSize) {
return records;
}
}
} while (line != null);
}
}
if (nread <= 0) {
synchronized (this) {
this.wait(1000);
}
}
return records;
} catch (IOException e) {
} catch (InterruptedException e) {
log.error("Interrupt error .", e);
}
return null;
}
use of io.openmessaging.connector.api.data.Schema in project rocketmq-externals by apache.
the class WorkerSinkTask method convertToSinkDataEntry.
private SinkDataEntry convertToSinkDataEntry(MessageExt message) {
Map<String, String> properties = message.getProperties();
String queueName;
EntryType entryType;
Schema schema;
Long timestamp;
Object[] datas = new Object[1];
if (null == recordConverter || recordConverter instanceof RocketMQConverter) {
queueName = properties.get(RuntimeConfigDefine.CONNECT_TOPICNAME);
String connectEntryType = properties.get(RuntimeConfigDefine.CONNECT_ENTRYTYPE);
entryType = StringUtils.isNotEmpty(connectEntryType) ? EntryType.valueOf(connectEntryType) : null;
String connectTimestamp = properties.get(RuntimeConfigDefine.CONNECT_TIMESTAMP);
timestamp = StringUtils.isNotEmpty(connectTimestamp) ? Long.valueOf(connectTimestamp) : null;
String connectSchema = properties.get(RuntimeConfigDefine.CONNECT_SCHEMA);
schema = StringUtils.isNotEmpty(connectSchema) ? JSON.parseObject(connectSchema, Schema.class) : null;
datas = new Object[1];
datas[0] = message.getBody();
} else {
final byte[] messageBody = message.getBody();
final SourceDataEntry sourceDataEntry = JSON.parseObject(new String(messageBody), SourceDataEntry.class);
final Object[] payload = sourceDataEntry.getPayload();
final byte[] decodeBytes = Base64.getDecoder().decode((String) payload[0]);
Object recodeObject;
if (recordConverter instanceof JsonConverter) {
JsonConverter jsonConverter = (JsonConverter) recordConverter;
jsonConverter.setClazz(Object[].class);
recodeObject = recordConverter.byteToObject(decodeBytes);
datas = (Object[]) recodeObject;
}
schema = sourceDataEntry.getSchema();
entryType = sourceDataEntry.getEntryType();
queueName = sourceDataEntry.getQueueName();
timestamp = sourceDataEntry.getTimestamp();
}
DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
dataEntryBuilder.entryType(entryType);
dataEntryBuilder.queue(queueName);
dataEntryBuilder.timestamp(timestamp);
List<Field> fields = schema.getFields();
if (null != fields && !fields.isEmpty()) {
for (Field field : fields) {
dataEntryBuilder.putFiled(field.getName(), datas[field.getIndex()]);
}
}
SinkDataEntry sinkDataEntry = dataEntryBuilder.buildSinkDataEntry(message.getQueueOffset());
return sinkDataEntry;
}
Aggregations