Examples with SinkDataEntry io.openmessaging.connector.api.data.SinkDataEntry used on opensource projects

Example 1 with SinkDataEntry

use of io.openmessaging.connector.api.data.SinkDataEntry in project rocketmq-externals by apache.

the class CassandraSinkTask method put.

@Override
public void put(Collection<SinkDataEntry> sinkDataEntries) {
    try {
        if (tableQueue.size() > 1) {
            updater = tableQueue.poll(1000, TimeUnit.MILLISECONDS);
        } else {
            updater = tableQueue.peek();
        }
        log.info("Cassandra Sink Task trying to put()");
        for (SinkDataEntry record : sinkDataEntries) {
            Map<Field, Object[]> fieldMap = new HashMap<>();
            Object[] payloads = record.getPayload();
            Schema schema = record.getSchema();
            EntryType entryType = record.getEntryType();
            String cfName = schema.getName();
            String keyspaceName = schema.getDataSource();
            List<Field> fields = schema.getFields();
            Boolean parseError = false;
            if (!fields.isEmpty()) {
                for (Field field : fields) {
                    Object fieldValue = payloads[field.getIndex()];
                    Object[] value = JSONObject.parseArray((String) fieldValue).toArray();
                    if (value.length == 2) {
                        fieldMap.put(field, value);
                    } else {
                        log.error("parseArray error, fieldValue:{}", fieldValue);
                        parseError = true;
                    }
                }
            }
            if (!parseError) {
                log.info("Cassandra Sink Task trying to call updater.push()");
                Boolean isSuccess = updater.push(keyspaceName, cfName, fieldMap, entryType);
                if (!isSuccess) {
                    log.error("push data error, keyspaceName:{}, cfName:{}, entryType:{}, fieldMap:{}", keyspaceName, cfName, fieldMap, entryType);
                }
            }
        }
    } catch (Exception e) {
        log.error("put sinkDataEntries error, {}", e);
    }
}

Example 2 with SinkDataEntry

use of io.openmessaging.connector.api.data.SinkDataEntry in project rocketmq-externals by apache.

the class WorkerSinkTask method receiveMessages.

/**
 * receive message from MQ.
 *
 * @param messages
 */
private void receiveMessages(List<MessageExt> messages) {
    List<SinkDataEntry> sinkDataEntries = new ArrayList<>(32);
    for (MessageExt message : messages) {
        SinkDataEntry sinkDataEntry = convertToSinkDataEntry(message);
        sinkDataEntries.add(sinkDataEntry);
        String msgId = message.getMsgId();
        log.info("Received one message success : msgId {}", msgId);
    }
    sinkTask.put(sinkDataEntries);
}

Example 3 with SinkDataEntry

use of io.openmessaging.connector.api.data.SinkDataEntry in project rocketmq-externals by apache.

the class WorkerSinkTask method convertToSinkDataEntry.

private SinkDataEntry convertToSinkDataEntry(MessageExt message) {
    Map<String, String> properties = message.getProperties();
    String queueName;
    EntryType entryType;
    Schema schema;
    Long timestamp;
    Object[] datas = new Object[1];
    if (null == recordConverter || recordConverter instanceof RocketMQConverter) {
        queueName = properties.get(RuntimeConfigDefine.CONNECT_TOPICNAME);
        String connectEntryType = properties.get(RuntimeConfigDefine.CONNECT_ENTRYTYPE);
        entryType = StringUtils.isNotEmpty(connectEntryType) ? EntryType.valueOf(connectEntryType) : null;
        String connectTimestamp = properties.get(RuntimeConfigDefine.CONNECT_TIMESTAMP);
        timestamp = StringUtils.isNotEmpty(connectTimestamp) ? Long.valueOf(connectTimestamp) : null;
        String connectSchema = properties.get(RuntimeConfigDefine.CONNECT_SCHEMA);
        schema = StringUtils.isNotEmpty(connectSchema) ? JSON.parseObject(connectSchema, Schema.class) : null;
        datas = new Object[1];
        datas[0] = message.getBody();
    } else {
        final byte[] messageBody = message.getBody();
        final SourceDataEntry sourceDataEntry = JSON.parseObject(new String(messageBody), SourceDataEntry.class);
        final Object[] payload = sourceDataEntry.getPayload();
        final byte[] decodeBytes = Base64.getDecoder().decode((String) payload[0]);
        Object recodeObject;
        if (recordConverter instanceof JsonConverter) {
            JsonConverter jsonConverter = (JsonConverter) recordConverter;
            jsonConverter.setClazz(Object[].class);
            recodeObject = recordConverter.byteToObject(decodeBytes);
            datas = (Object[]) recodeObject;
        }
        schema = sourceDataEntry.getSchema();
        entryType = sourceDataEntry.getEntryType();
        queueName = sourceDataEntry.getQueueName();
        timestamp = sourceDataEntry.getTimestamp();
    }
    DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
    dataEntryBuilder.entryType(entryType);
    dataEntryBuilder.queue(queueName);
    dataEntryBuilder.timestamp(timestamp);
    List<Field> fields = schema.getFields();
    if (null != fields && !fields.isEmpty()) {
        for (Field field : fields) {
            dataEntryBuilder.putFiled(field.getName(), datas[field.getIndex()]);
        }
    }
    SinkDataEntry sinkDataEntry = dataEntryBuilder.buildSinkDataEntry(message.getQueueOffset());
    return sinkDataEntry;
}

Example 4 with SinkDataEntry

use of io.openmessaging.connector.api.data.SinkDataEntry in project rocketmq-externals by apache.

the class Updater method commit.

public void commit() {
    List<SinkDataEntry> commitList;
    if (inflightList.isEmpty()) {
        return;
    }
    synchronized (this.inflightList) {
        commitList = inflightList;
        inflightList = new ArrayList<>();
    }
    List<HoodieRecord> hoodieRecordsList = new ArrayList<>();
    for (SinkDataEntry record : commitList) {
        GenericRecord genericRecord = sinkDataEntry2GenericRecord(record);
        HoodieRecord<HoodieAvroPayload> hoodieRecord = new HoodieRecord(new HoodieKey(UUID.randomUUID().toString(), "shardingKey-" + record.getQueueName()), new HoodieAvroPayload(Option.of(genericRecord)));
        hoodieRecordsList.add(hoodieRecord);
    }
    try {
        List<WriteStatus> statuses = hudiWriteClient.upsert(hoodieRecordsList, hudiWriteClient.startCommit());
        log.info("Upserted data to hudi");
        long upserted = statuses.get(0).getStat().getNumInserts();
        if (upserted != commitList.size()) {
            log.warn("Upserted num not equals input");
        }
    } catch (Exception e) {
        log.error("Exception when upserting to Hudi", e);
    }
}

Example 5 with SinkDataEntry

use of io.openmessaging.connector.api.data.SinkDataEntry in project rocketmq-externals by apache.

the class HudiSinkTask method put.

@Override
public void put(Collection<SinkDataEntry> sinkDataEntries) {
    try {
        log.info("Hudi Sink Task trying to put()");
        for (SinkDataEntry record : sinkDataEntries) {
            log.info("Hudi Sink Task trying to call updater.push()");
            Boolean isSuccess = updater.push(record);
            if (!isSuccess) {
                log.error("Hudi sink push data error, record:{}", record);
            }
            log.debug("Hudi pushed data : " + record);
        }
    } catch (Exception e) {
        log.error("put sinkDataEntries error, {}", e);
    }
}