Examples with TableLocationType com.dtstack.taier.develop.enums.develop.TableLocationType used on opensource projects

Example 1 with TableLocationType

use of com.dtstack.taier.develop.enums.develop.TableLocationType in project Taier by DTStack.

the class ImpalaSyncBuilder method syncWriterBuild.

@Override
public Writer syncWriterBuild(List<Long> targetIds, Map<String, Object> targetMap, Reader reader) {
    TableLocationType tableLocationType = TableLocationType.getTableLocationType((String) targetMap.get(TableLocationType.key()));
    if (tableLocationType == null) {
        throw new RdosDefineException("不支持的表存储类型");
    }
    if (tableLocationType == TableLocationType.HIVE) {
        Map<String, Object> clone = new HashMap<>(targetMap);
        String writeMode = (String) clone.get("writeMode");
        writeMode = writeMode != null && writeMode.trim().length() != 0 ? SyncWriteMode.tranferHiveMode(writeMode) : SyncWriteMode.HIVE_OVERWRITE.getMode();
        clone.put("writeMode", writeMode);
        // 设置hdfs index字段
        List column = (List) clone.get("column");
        List<Column> allColumns = (List<Column>) clone.get("allColumns");
        List<Column> partitionColumns = (List<Column>) clone.get("partitionColumns");
        Map<String, Column> allColumnsMap = allColumns.stream().collect(Collectors.toMap(Column::getName, item -> item));
        for (Object col : column) {
            String name = (String) ((Map<String, Object>) col).get("key");
            ((Map<String, Object>) col).put("index", allColumnsMap.get(name).getIndex());
        }
        // 设置 fullColumnNames 和 fullColumnTypes  脏数据记录的时候需要
        // 需要去掉分区字段
        Set<String> partitionColumnNameSet = CollectionUtils.isEmpty(partitionColumns) ? new HashSet<>() : partitionColumns.stream().map(pColumn -> pColumn.getName()).collect(Collectors.toSet());
        List<String> fullColumnNames = new ArrayList<>();
        List<String> fullColumnTypes = new ArrayList<>();
        for (Column allColumn : allColumns) {
            if (!partitionColumnNameSet.contains(allColumn.getName())) {
                fullColumnNames.add(allColumn.getName());
                fullColumnTypes.add(allColumn.getType());
            }
        }
        clone.put("fullColumnNames", fullColumnNames);
        clone.put("fullColumnTypes", fullColumnTypes);
        String partition = (String) clone.get("partition");
        // fileName 逻辑参考自HiveWriter
        String fileName = StringUtils.isNotEmpty(partition) ? partition : "";
        clone.put("fileName", fileName);
        return objToObject(clone, ImpalaHdfsWriter.class);
    } else if (tableLocationType == TableLocationType.KUDU) {
        KuduWriter kuduWriter = objToObject(targetMap, KuduWriter.class);
        String kuduTableName = (String) targetMap.get("kuduTableName");
        LOGGER.info("syncWriterBuild format impala  kuduTableName :{} ", kuduTableName);
        kuduWriter.setTable(kuduTableName);
        return kuduWriter;
    }
    return null;
}

Example 2 with TableLocationType

use of com.dtstack.taier.develop.enums.develop.TableLocationType in project Taier by DTStack.

the class ImpalaSyncBuilder method setWriterJson.

@Override
public void setWriterJson(Map<String, Object> map, Map<String, Object> dataSource, Map<String, Object> kerberos) {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("setWriterJson DataSourceType: Impala \nsourceMap :{} \n datasourceJson :{}", JSON.toJSONString(map), JSON.toJSONString(dataSource));
    }
    String tableName = (String) map.get("table");
    // 优化点 前端可以回传tableLocation数据，减少链接的创建
    JSONObject tableLocation = tableLocation(dataSource, tableName, kerberos);
    LOGGER.info("get read tableLocation :{}", tableLocation.toJSONString());
    TableLocationType tableLocationType = TableLocationType.getTableLocationType(tableLocation.getString(TableLocationType.key()));
    if (tableLocationType == null) {
        throw new RdosDefineException("不支持的表存储类型");
    }
    if (tableLocationType == TableLocationType.HIVE) {
        // map.put("type", DataSourceType.HIVE);
        // map.put("partition", map.get(HIVE_PARTITION));
        // map.put("defaultFS", dataSource.get(HDFS_DEFAULTFS));
        // map.put("password", dataSource.get(JDBC_PASSWORD));
        // map.put("username", dataSource.get(JDBC_USERNAME));
        // map.put("jdbcUrl", dataSource.get(JDBC_URL));
        // String hadoopConfig = (String) dataSource.get(HADOOP_CONFIG);
        // if (StringUtils.isNotBlank(hadoopConfig)) {
        // map.put("hadoopConfig", JSONObject.parse(hadoopConfig));
        // }
        map.putAll(tableLocation);
    } else if (tableLocationType == TableLocationType.KUDU) {
        List column = (List) map.get("column");
        for (Object col : column) {
            String name = (String) ((Map<String, Object>) col).get("key");
            ((Map<String, Object>) col).put("name", name);
        }
        map.putAll(tableLocation);
    }
}

Example 3 with TableLocationType

use of com.dtstack.taier.develop.enums.develop.TableLocationType in project Taier by DTStack.

the class ImpalaSyncBuilder method setReaderJson.

@Override
public void setReaderJson(Map<String, Object> map, Map<String, Object> dataSource, Map<String, Object> kerberos) {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("set read json DataSourceType: Impala \nsourceMap :{} \n datasourceJson :{}", JSON.toJSONString(map), JSON.toJSONString(dataSource));
    }
    String tableName = (String) map.get("table");
    // 优化点 前端可以回传tableLocation数据，减少链接的创建
    JSONObject tableLocation = tableLocation(dataSource, tableName, kerberos);
    LOGGER.info("get read tableLocation :{}", tableLocation.toJSONString());
    TableLocationType tableLocationType = TableLocationType.getTableLocationType(tableLocation.getString(TableLocationType.key()));
    if (tableLocationType == null) {
        throw new RdosDefineException("不支持的表存储类型");
    }
    if (tableLocationType == TableLocationType.HIVE) {
        // map.put("type", DataSourceType.HIVE);
        // map.put("partition", map.get(HIVE_PARTITION));
        // map.put("defaultFS", dataSource.get(HDFS_DEFAULTFS));
        // map.put("password", dataSource.get(JDBC_PASSWORD));
        // map.put("username", dataSource.get(JDBC_USERNAME));
        // map.put("jdbcUrl", dataSource.get(JDBC_URL));
        // String hadoopConfig = (String) dataSource.get(HADOOP_CONFIG);
        // if (StringUtils.isNotBlank(hadoopConfig)) {
        // map.put("hadoopConfig", JSONObject.parse(hadoopConfig));
        // }
        map.putAll(tableLocation);
    } else if (tableLocationType == TableLocationType.KUDU) {
        List column = (List) map.get("column");
        for (Object col : column) {
            String name = (String) ((Map<String, Object>) col).get("key");
            ((Map<String, Object>) col).put("name", name);
        }
        map.putAll(tableLocation);
    }
}