use of io.openmessaging.connector.api.data.Field in project rocketmq-externals by apache.
the class CassandraSinkTask method put.
@Override
public void put(Collection<SinkDataEntry> sinkDataEntries) {
try {
if (tableQueue.size() > 1) {
updater = tableQueue.poll(1000, TimeUnit.MILLISECONDS);
} else {
updater = tableQueue.peek();
}
log.info("Cassandra Sink Task trying to put()");
for (SinkDataEntry record : sinkDataEntries) {
Map<Field, Object[]> fieldMap = new HashMap<>();
Object[] payloads = record.getPayload();
Schema schema = record.getSchema();
EntryType entryType = record.getEntryType();
String cfName = schema.getName();
String keyspaceName = schema.getDataSource();
List<Field> fields = schema.getFields();
Boolean parseError = false;
if (!fields.isEmpty()) {
for (Field field : fields) {
Object fieldValue = payloads[field.getIndex()];
Object[] value = JSONObject.parseArray((String) fieldValue).toArray();
if (value.length == 2) {
fieldMap.put(field, value);
} else {
log.error("parseArray error, fieldValue:{}", fieldValue);
parseError = true;
}
}
}
if (!parseError) {
log.info("Cassandra Sink Task trying to call updater.push()");
Boolean isSuccess = updater.push(keyspaceName, cfName, fieldMap, entryType);
if (!isSuccess) {
log.error("push data error, keyspaceName:{}, cfName:{}, entryType:{}, fieldMap:{}", keyspaceName, cfName, fieldMap, entryType);
}
}
}
} catch (Exception e) {
log.error("put sinkDataEntries error, {}", e);
}
}
use of io.openmessaging.connector.api.data.Field in project rocketmq-externals by apache.
the class JdbcSourceTask method poll.
@Override
public Collection<SourceDataEntry> poll() {
List<SourceDataEntry> res = new ArrayList<>();
try {
if (tableQueue.size() > 1)
querier = tableQueue.poll(1000, TimeUnit.MILLISECONDS);
else
querier = tableQueue.peek();
Timer timer = new Timer();
try {
Thread.currentThread();
// 毫秒
Thread.sleep(1000);
} catch (Exception e) {
throw e;
}
querier.poll();
for (Table dataRow : querier.getList()) {
JSONObject jsonObject = new JSONObject();
jsonObject.put("nextQuery", "database");
jsonObject.put("nextPosition", "table");
Schema schema = new Schema();
schema.setDataSource(dataRow.getDatabase());
schema.setName(dataRow.getName());
schema.setFields(new ArrayList<>());
for (int i = 0; i < dataRow.getColList().size(); i++) {
String columnName = dataRow.getColList().get(i);
String rawDataType = dataRow.getRawDataTypeList().get(i);
Field field = new Field(i, columnName, ColumnParser.mapConnectorFieldType(rawDataType));
schema.getFields().add(field);
}
DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
dataEntryBuilder.timestamp(System.currentTimeMillis()).queue(dataRow.getName()).entryType(EntryType.UPDATE);
for (int i = 0; i < dataRow.getColList().size(); i++) {
Object[] value = new Object[2];
value[0] = value[1] = dataRow.getParserList().get(i).getValue(dataRow.getDataList().get(i));
dataEntryBuilder.putFiled(dataRow.getColList().get(i), JSONObject.toJSONString(value));
}
SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(ByteBuffer.wrap((ConstDefine.PREFIX + config.getDbUrl() + config.getDbPort()).getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(jsonObject.toJSONString().getBytes(StandardCharsets.UTF_8)));
res.add(sourceDataEntry);
log.debug("sourceDataEntry : {}", JSONObject.toJSONString(sourceDataEntry));
}
} catch (Exception e) {
log.error("JDBC task poll error, current config:" + JSON.toJSONString(config), e);
}
log.debug("dataEntry poll successfully,{}", JSONObject.toJSONString(res));
return res;
}
use of io.openmessaging.connector.api.data.Field in project rocketmq-externals by apache.
the class Updater method updateRow.
private Boolean updateRow(String dbName, String tableName, Map<Field, Object[]> fieldMap, Integer id) {
int count = 0;
PreparedStatement stmt;
boolean finishUpdate = false;
String update = "replace into " + dbName + "." + tableName + " set ";
for (Map.Entry<Field, Object[]> entry : fieldMap.entrySet()) {
count++;
String fieldName = entry.getKey().getName();
FieldType fieldType = entry.getKey().getType();
Object fieldValue = entry.getValue()[1];
if ("id".equals(fieldName)) {
if (id == 0) {
if (count == fieldMap.size())
update = update.substring(0, update.length() - 1);
continue;
} else {
fieldValue = id;
}
}
if (fieldValue == null) {
update += fieldName + " = NULL";
} else {
update = typeParser(fieldType, fieldName, fieldValue, update);
}
if (count < fieldMap.size()) {
update += ",";
}
}
try {
while (!connection.isClosed() && !finishUpdate) {
stmt = connection.prepareStatement(update);
int result = stmt.executeUpdate();
if (result > 0) {
log.info("replace into table success");
return true;
}
finishUpdate = true;
stmt.close();
}
} catch (SQLException e) {
log.error("update table error,{}", e);
}
return false;
}
use of io.openmessaging.connector.api.data.Field in project rocketmq-externals by apache.
the class Updater method queryAfterUpdateRowId.
private Integer queryAfterUpdateRowId(String dbName, String tableName, Map<Field, Object[]> fieldMap) {
int count = 0, id = 0;
ResultSet rs;
PreparedStatement stmt;
Boolean finishQuery = false;
String query = "select id from " + dbName + "." + tableName + " where 1=1";
for (Map.Entry<Field, Object[]> entry : fieldMap.entrySet()) {
count++;
String fieldName = entry.getKey().getName();
FieldType fieldType = entry.getKey().getType();
Object fieldValue = entry.getValue()[1];
if ("id".equals(fieldName))
continue;
if (count <= fieldMap.size()) {
query += " and ";
}
if (fieldValue == null) {
query += fieldName + " is NULL";
} else {
query = typeParser(fieldType, fieldName, fieldValue, query);
}
}
try {
while (!connection.isClosed() && !finishQuery) {
stmt = connection.prepareStatement(query);
rs = stmt.executeQuery();
if (rs != null) {
while (rs.next()) {
id = rs.getInt("id");
}
finishQuery = true;
rs.close();
}
}
} catch (SQLException e) {
log.error("query table error,{}", e);
}
return id;
}
use of io.openmessaging.connector.api.data.Field in project rocketmq-externals by apache.
the class RedisEntryConverter method getRedisSchema.
private Schema getRedisSchema(FieldType valueType) {
Schema schema = new Schema();
schema.setDataSource(Options.REDIS_DATASOURCE.name());
List<Field> fields = new ArrayList<>();
fields.add(new Field(0, Options.REDIS_COMMAND.name(), FieldType.STRING));
fields.add(new Field(1, Options.REDIS_KEY.name(), FieldType.STRING));
fields.add(new Field(2, Options.REDIS_VALUE.name(), valueType));
fields.add(new Field(3, Options.REDIS_PARAMS.name(), FieldType.MAP));
schema.setFields(fields);
return schema;
}
Aggregations