use of org.apache.rocketmq.connect.cassandra.schema.Table in project rocketmq-externals by apache.
the class Querier method poll.
public void poll() {
try {
LinkedList<Table> tableLinkedList = new LinkedList<>();
for (Map.Entry<String, Database> entry : schema.getDbMap().entrySet()) {
String dbName = entry.getKey();
Iterator<Map.Entry<String, Table>> iterator = entry.getValue().getTableMap().entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, Table> tableEntry = iterator.next();
String tableName = tableEntry.getKey();
Table table = tableEntry.getValue();
Map<String, String> tableFilterMap = table.getFilterMap();
Select selectFrom = QueryBuilder.selectFrom(dbName, tableName).all();
if (tableFilterMap != null && !tableFilterMap.keySet().contains("NO-FILTER")) {
for (String key : tableFilterMap.keySet()) {
String value = tableFilterMap.get(key);
selectFrom.whereColumn(key).isEqualTo(QueryBuilder.literal(value));
}
}
SimpleStatement stmt;
boolean finishUpdate = false;
log.info("trying to execute sql query,{}", selectFrom.asCql());
ResultSet result = null;
while (!cqlSession.isClosed() && !finishUpdate) {
stmt = selectFrom.build();
result = cqlSession.execute(stmt);
if (result.wasApplied()) {
log.info("query columns success, executed cql query {}", selectFrom.asCql());
}
finishUpdate = true;
}
List<String> colList = tableEntry.getValue().getColList();
List<String> dataTypeList = tableEntry.getValue().getRawDataTypeList();
List<ColumnParser> parserList = tableEntry.getValue().getParserList();
for (Row row : result) {
Table tableWithData = new Table(dbName, tableName);
tableWithData.setColList(colList);
tableWithData.setRawDataTypeList(dataTypeList);
tableWithData.setParserList(parserList);
for (String col : colList) {
tableWithData.getDataList().add(row.getObject(col));
}
tableLinkedList.add(tableWithData);
}
}
}
list = tableLinkedList;
} catch (Exception e) {
log.error("fail to poll data, {}", e);
}
}
use of org.apache.rocketmq.connect.cassandra.schema.Table in project rocketmq-externals by apache.
the class CassandraSourceTask method poll.
@Override
public Collection<SourceDataEntry> poll() {
List<SourceDataEntry> res = new ArrayList<>();
try {
if (tableQueue.size() > 1)
querier = tableQueue.poll(1000, TimeUnit.MILLISECONDS);
else
querier = tableQueue.peek();
Timer timer = new Timer();
try {
Thread.currentThread();
// 毫秒
Thread.sleep(1000);
} catch (Exception e) {
throw e;
}
querier.poll();
for (Table dataRow : querier.getList()) {
JSONObject jsonObject = new JSONObject();
jsonObject.put("nextQuery", "database");
jsonObject.put("nextPosition", "table");
Schema schema = new Schema();
schema.setDataSource(dataRow.getDatabase());
schema.setName(dataRow.getName());
schema.setFields(new ArrayList<>());
for (int i = 0; i < dataRow.getColList().size(); i++) {
String columnName = dataRow.getColList().get(i);
String rawDataType = dataRow.getRawDataTypeList().get(i);
Field field = new Field(i, columnName, ColumnParser.mapConnectorFieldType(rawDataType));
schema.getFields().add(field);
}
DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
dataEntryBuilder.timestamp(System.currentTimeMillis()).queue(dataRow.getName()).entryType(EntryType.UPDATE);
for (int i = 0; i < dataRow.getColList().size(); i++) {
Object[] value = new Object[2];
value[0] = value[1] = dataRow.getParserList().get(i).getValue(dataRow.getDataList().get(i));
dataEntryBuilder.putFiled(dataRow.getColList().get(i), JSONObject.toJSONString(value));
}
SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(ByteBuffer.wrap((ConstDefine.PREFIX + config.getDbUrl() + config.getDbPort()).getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(jsonObject.toJSONString().getBytes(StandardCharsets.UTF_8)));
res.add(sourceDataEntry);
log.debug("sourceDataEntry : {}", JSONObject.toJSONString(sourceDataEntry));
}
} catch (Exception e) {
log.error("Cassandra task poll error, current config:" + JSON.toJSONString(config), e);
}
log.debug("dataEntry poll successfully,{}", JSONObject.toJSONString(res));
return res;
}
Aggregations