use of com.dtstack.taier.develop.utils.develop.sync.template.KuduWriter in project Taier by DTStack.
the class ImpalaSyncBuilder method syncWriterBuild.
@Override
public Writer syncWriterBuild(List<Long> targetIds, Map<String, Object> targetMap, Reader reader) {
TableLocationType tableLocationType = TableLocationType.getTableLocationType((String) targetMap.get(TableLocationType.key()));
if (tableLocationType == null) {
throw new RdosDefineException("不支持的表存储类型");
}
if (tableLocationType == TableLocationType.HIVE) {
Map<String, Object> clone = new HashMap<>(targetMap);
String writeMode = (String) clone.get("writeMode");
writeMode = writeMode != null && writeMode.trim().length() != 0 ? SyncWriteMode.tranferHiveMode(writeMode) : SyncWriteMode.HIVE_OVERWRITE.getMode();
clone.put("writeMode", writeMode);
// 设置hdfs index字段
List column = (List) clone.get("column");
List<Column> allColumns = (List<Column>) clone.get("allColumns");
List<Column> partitionColumns = (List<Column>) clone.get("partitionColumns");
Map<String, Column> allColumnsMap = allColumns.stream().collect(Collectors.toMap(Column::getName, item -> item));
for (Object col : column) {
String name = (String) ((Map<String, Object>) col).get("key");
((Map<String, Object>) col).put("index", allColumnsMap.get(name).getIndex());
}
// 设置 fullColumnNames 和 fullColumnTypes 脏数据记录的时候需要
// 需要去掉分区字段
Set<String> partitionColumnNameSet = CollectionUtils.isEmpty(partitionColumns) ? new HashSet<>() : partitionColumns.stream().map(pColumn -> pColumn.getName()).collect(Collectors.toSet());
List<String> fullColumnNames = new ArrayList<>();
List<String> fullColumnTypes = new ArrayList<>();
for (Column allColumn : allColumns) {
if (!partitionColumnNameSet.contains(allColumn.getName())) {
fullColumnNames.add(allColumn.getName());
fullColumnTypes.add(allColumn.getType());
}
}
clone.put("fullColumnNames", fullColumnNames);
clone.put("fullColumnTypes", fullColumnTypes);
String partition = (String) clone.get("partition");
// fileName 逻辑参考自HiveWriter
String fileName = StringUtils.isNotEmpty(partition) ? partition : "";
clone.put("fileName", fileName);
return objToObject(clone, ImpalaHdfsWriter.class);
} else if (tableLocationType == TableLocationType.KUDU) {
KuduWriter kuduWriter = objToObject(targetMap, KuduWriter.class);
String kuduTableName = (String) targetMap.get("kuduTableName");
LOGGER.info("syncWriterBuild format impala kuduTableName :{} ", kuduTableName);
kuduWriter.setTable(kuduTableName);
return kuduWriter;
}
return null;
}
Aggregations