use of io.openmessaging.connector.api.data.Schema in project rocketmq-externals by apache.
the class WorkerDirectTask method sendRecord.
private void sendRecord(Collection<SourceDataEntry> sourceDataEntries) {
List<SinkDataEntry> sinkDataEntries = new ArrayList<>(sourceDataEntries.size());
ByteBuffer position = null;
ByteBuffer partition = null;
for (SourceDataEntry sourceDataEntry : sourceDataEntries) {
Schema schema = sourceDataEntry.getSchema();
Object[] payload = sourceDataEntry.getPayload();
DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema).entryType(sourceDataEntry.getEntryType()).queue(sourceDataEntry.getQueueName()).timestamp(sourceDataEntry.getTimestamp());
if (schema.getFields() != null) {
schema.getFields().forEach(field -> dataEntryBuilder.putFiled(field.getName(), payload[field.getIndex()]));
}
SinkDataEntry sinkDataEntry = dataEntryBuilder.buildSinkDataEntry(-1L);
sinkDataEntries.add(sinkDataEntry);
position = sourceDataEntry.getSourcePosition();
partition = sourceDataEntry.getSourcePartition();
}
try {
sinkTask.put(sinkDataEntries);
try {
if (null != position && null != partition) {
positionManagementService.putPosition(position, partition);
}
} catch (Exception e) {
log.error("Source task save position info failed.", e);
}
} catch (Exception e) {
log.error("Send message error, error info: {}.", e);
}
}
use of io.openmessaging.connector.api.data.Schema in project rocketmq-externals by apache.
the class FileSinkTask method put.
@Override
public void put(Collection<SinkDataEntry> sinkDataEntries) {
for (SinkDataEntry record : sinkDataEntries) {
Object[] payloads = record.getPayload();
log.trace("Writing line to {}: {}", logFilename(), payloads);
Schema schema = record.getSchema();
List<Field> fields = schema.getFields();
for (Field field : fields) {
FieldType type = field.getType();
if (type.equals(FieldType.STRING)) {
log.info("Writing line to {}: {}", logFilename(), payloads[field.getIndex()]);
outputStream.println(String.valueOf(payloads[field.getIndex()]));
}
}
}
}
use of io.openmessaging.connector.api.data.Schema in project rocketmq-externals by apache.
the class TestSourceTask method poll.
@Override
public Collection<SourceDataEntry> poll() {
Set<SourceDataEntry> sourceTasks = new HashSet<>();
Object[] newPayload = new Object[1];
newPayload[0] = Base64.getEncoder().encodeToString("test".getBytes());
sourceTasks.add(new SourceDataEntry(ByteBuffer.wrap("1".getBytes()), ByteBuffer.wrap("2".getBytes()), System.currentTimeMillis(), EntryType.CREATE, "test-queue", new Schema(), newPayload));
return sourceTasks;
}
Aggregations