use of io.openmessaging.connector.api.data.Schema in project rocketmq-externals by apache.
the class JdbcSinkTask method put.
@Override
public void put(Collection<SinkDataEntry> sinkDataEntries) {
try {
if (tableQueue.size() > 1) {
updater = tableQueue.poll(1000, TimeUnit.MILLISECONDS);
} else {
updater = tableQueue.peek();
}
for (SinkDataEntry record : sinkDataEntries) {
Map<Field, Object[]> fieldMap = new HashMap<>();
Object[] payloads = record.getPayload();
Schema schema = record.getSchema();
EntryType entryType = record.getEntryType();
String tableName = schema.getName();
String dbName = schema.getDataSource();
List<Field> fields = schema.getFields();
Boolean parseError = false;
if (!fields.isEmpty()) {
for (Field field : fields) {
Object fieldValue = payloads[field.getIndex()];
Object[] value = JSONObject.parseArray((String) fieldValue).toArray();
if (value.length == 2) {
fieldMap.put(field, value);
} else {
log.error("parseArray error, fieldValue:{}", fieldValue);
parseError = true;
}
}
}
if (!parseError) {
Boolean isSuccess = updater.push(dbName, tableName, fieldMap, entryType);
if (!isSuccess) {
log.error("push data error, dbName:{}, tableName:{}, entryType:{}, fieldMap:{}", dbName, tableName, fieldMap, entryType);
}
}
}
} catch (Exception e) {
log.error("put sinkDataEntries error, {}", e);
}
}
use of io.openmessaging.connector.api.data.Schema in project rocketmq-externals by apache.
the class CassandraSourceTask method poll.
@Override
public Collection<SourceDataEntry> poll() {
List<SourceDataEntry> res = new ArrayList<>();
try {
if (tableQueue.size() > 1)
querier = tableQueue.poll(1000, TimeUnit.MILLISECONDS);
else
querier = tableQueue.peek();
Timer timer = new Timer();
try {
Thread.currentThread();
// 毫秒
Thread.sleep(1000);
} catch (Exception e) {
throw e;
}
querier.poll();
for (Table dataRow : querier.getList()) {
JSONObject jsonObject = new JSONObject();
jsonObject.put("nextQuery", "database");
jsonObject.put("nextPosition", "table");
Schema schema = new Schema();
schema.setDataSource(dataRow.getDatabase());
schema.setName(dataRow.getName());
schema.setFields(new ArrayList<>());
for (int i = 0; i < dataRow.getColList().size(); i++) {
String columnName = dataRow.getColList().get(i);
String rawDataType = dataRow.getRawDataTypeList().get(i);
Field field = new Field(i, columnName, ColumnParser.mapConnectorFieldType(rawDataType));
schema.getFields().add(field);
}
DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
dataEntryBuilder.timestamp(System.currentTimeMillis()).queue(dataRow.getName()).entryType(EntryType.UPDATE);
for (int i = 0; i < dataRow.getColList().size(); i++) {
Object[] value = new Object[2];
value[0] = value[1] = dataRow.getParserList().get(i).getValue(dataRow.getDataList().get(i));
dataEntryBuilder.putFiled(dataRow.getColList().get(i), JSONObject.toJSONString(value));
}
SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(ByteBuffer.wrap((ConstDefine.PREFIX + config.getDbUrl() + config.getDbPort()).getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(jsonObject.toJSONString().getBytes(StandardCharsets.UTF_8)));
res.add(sourceDataEntry);
log.debug("sourceDataEntry : {}", JSONObject.toJSONString(sourceDataEntry));
}
} catch (Exception e) {
log.error("Cassandra task poll error, current config:" + JSON.toJSONString(config), e);
}
log.debug("dataEntry poll successfully,{}", JSONObject.toJSONString(res));
return res;
}
use of io.openmessaging.connector.api.data.Schema in project rocketmq-externals by apache.
the class MongoSourceConnectorTest method testPoll.
@Test
public void testPoll() throws Exception {
LinkedBlockingQueue<SourceDataEntry> entries = new LinkedBlockingQueue<>();
ReplicaSetsContext context = new ReplicaSetsContext(sourceTaskConfig);
Field dataEntryQueue = ReplicaSetsContext.class.getDeclaredField("dataEntryQueue");
dataEntryQueue.setAccessible(true);
dataEntryQueue.set(context, entries);
ReplicationEvent event = new ReplicationEvent();
event.setOperationType(OperationType.INSERT);
event.setNamespace("test.person");
event.setTimestamp(new BsonTimestamp(1565609506, 1));
event.setDocument(new Document("testKey", "testValue"));
event.setH(324243242L);
event.setEventData(Optional.ofNullable(new Document("testEventKey", "testEventValue")));
event.setObjectId(Optional.empty());
context.publishEvent(event, new ReplicaSetConfig("", "testReplicaName", "localhost:27027"));
List<SourceDataEntry> sourceDataEntries = (List<SourceDataEntry>) context.poll();
Assert.assertTrue(sourceDataEntries.size() == 1);
SourceDataEntry sourceDataEntry = sourceDataEntries.get(0);
Assert.assertEquals("test-person", sourceDataEntry.getQueueName());
ByteBuffer sourcePartition = sourceDataEntry.getSourcePartition();
Assert.assertEquals("testReplicaName", new String(sourcePartition.array()));
ByteBuffer sourcePosition = sourceDataEntry.getSourcePosition();
Position position = JSONObject.parseObject(new String(sourcePosition.array()), Position.class);
Assert.assertEquals(position.getTimeStamp(), 1565609506);
Assert.assertEquals(position.getInc(), 1);
Assert.assertEquals(position.isInitSync(), false);
EntryType entryType = sourceDataEntry.getEntryType();
Assert.assertEquals(EntryType.CREATE, entryType);
String queueName = sourceDataEntry.getQueueName();
Assert.assertEquals("test-person", queueName);
Schema schema = sourceDataEntry.getSchema();
Assert.assertTrue(schema.getFields().size() == 6);
Object[] payload = sourceDataEntry.getPayload();
Assert.assertTrue(payload.length == 6);
}
use of io.openmessaging.connector.api.data.Schema in project rocketmq-externals by apache.
the class MongoDataEntry method createdSchema.
private static Schema createdSchema(String dataSourceName) {
Schema schema = new Schema();
schema.setDataSource(dataSourceName);
schema.setName(SCHEMA_CREATED_NAME);
schema.setFields(new ArrayList<>());
createdField(schema);
return schema;
}
use of io.openmessaging.connector.api.data.Schema in project rocketmq-externals by apache.
the class RedisEntryConverter method kVEntryToDataEntries.
@Override
public List<SourceDataEntry> kVEntryToDataEntries(KVEntry kvEntry) {
Schema schema = getRedisSchema(kvEntry.getValueType());
String partition = kvEntry.getPartition();
if (partition == null) {
throw new IllegalStateException("partition info error.");
}
List<SourceDataEntry> res = new ArrayList<>();
List<Object> values = splitValue(kvEntry.getValueType(), kvEntry.getValue(), this.maxValueSize);
for (int i = 0; i < values.size(); i++) {
DataEntryBuilder builder = newDataEntryBuilderWithoutValue(schema, kvEntry);
builder.putFiled(Options.REDIS_VALUE.name(), values.get(i));
builder.timestamp(System.currentTimeMillis());
SourceDataEntry entry = builder.buildSourceDataEntry(ByteBuffer.wrap(kvEntry.getPartition().getBytes()), ByteBuffer.wrap(RedisPositionConverter.longToJson(kvEntry.getOffset()).toJSONString().getBytes()));
res.add(entry);
}
return res;
}
Aggregations