Examples with Reader com.dtstack.taier.develop.common.template.Reader used on opensource projects

Example 1 with Reader

use of com.dtstack.taier.develop.common.template.Reader in project Taier by DTStack.

the class ImpalaSyncBuilder method syncWriterBuild.

@Override
public Writer syncWriterBuild(List<Long> targetIds, Map<String, Object> targetMap, Reader reader) {
    TableLocationType tableLocationType = TableLocationType.getTableLocationType((String) targetMap.get(TableLocationType.key()));
    if (tableLocationType == null) {
        throw new RdosDefineException("不支持的表存储类型");
    }
    if (tableLocationType == TableLocationType.HIVE) {
        Map<String, Object> clone = new HashMap<>(targetMap);
        String writeMode = (String) clone.get("writeMode");
        writeMode = writeMode != null && writeMode.trim().length() != 0 ? SyncWriteMode.tranferHiveMode(writeMode) : SyncWriteMode.HIVE_OVERWRITE.getMode();
        clone.put("writeMode", writeMode);
        // 设置hdfs index字段
        List column = (List) clone.get("column");
        List<Column> allColumns = (List<Column>) clone.get("allColumns");
        List<Column> partitionColumns = (List<Column>) clone.get("partitionColumns");
        Map<String, Column> allColumnsMap = allColumns.stream().collect(Collectors.toMap(Column::getName, item -> item));
        for (Object col : column) {
            String name = (String) ((Map<String, Object>) col).get("key");
            ((Map<String, Object>) col).put("index", allColumnsMap.get(name).getIndex());
        }
        // 设置 fullColumnNames 和 fullColumnTypes  脏数据记录的时候需要
        // 需要去掉分区字段
        Set<String> partitionColumnNameSet = CollectionUtils.isEmpty(partitionColumns) ? new HashSet<>() : partitionColumns.stream().map(pColumn -> pColumn.getName()).collect(Collectors.toSet());
        List<String> fullColumnNames = new ArrayList<>();
        List<String> fullColumnTypes = new ArrayList<>();
        for (Column allColumn : allColumns) {
            if (!partitionColumnNameSet.contains(allColumn.getName())) {
                fullColumnNames.add(allColumn.getName());
                fullColumnTypes.add(allColumn.getType());
            }
        }
        clone.put("fullColumnNames", fullColumnNames);
        clone.put("fullColumnTypes", fullColumnTypes);
        String partition = (String) clone.get("partition");
        // fileName 逻辑参考自HiveWriter
        String fileName = StringUtils.isNotEmpty(partition) ? partition : "";
        clone.put("fileName", fileName);
        return objToObject(clone, ImpalaHdfsWriter.class);
    } else if (tableLocationType == TableLocationType.KUDU) {
        KuduWriter kuduWriter = objToObject(targetMap, KuduWriter.class);
        String kuduTableName = (String) targetMap.get("kuduTableName");
        LOGGER.info("syncWriterBuild format impala  kuduTableName :{} ", kuduTableName);
        kuduWriter.setTable(kuduTableName);
        return kuduWriter;
    }
    return null;
}

Example 2 with Reader

use of com.dtstack.taier.develop.common.template.Reader in project Taier by DTStack.

the class DatasourceService method syncReaderBuild.

private Reader syncReaderBuild(final Integer sourceType, final Map<String, Object> sourceMap, final List<Long> sourceIds) throws IOException {
    Reader reader = null;
    if (Objects.nonNull(RDBMSSourceType.getByDataSourceType(sourceType)) && !DataSourceType.HIVE.getVal().equals(sourceType) && !DataSourceType.HIVE1X.getVal().equals(sourceType) && !DataSourceType.HIVE3X.getVal().equals(sourceType) && !DataSourceType.CarbonData.getVal().equals(sourceType) && !DataSourceType.IMPALA.getVal().equals(sourceType) && !DataSourceType.SparkThrift2_1.getVal().equals(sourceType)) {
        reader = PublicUtil.objectToObject(sourceMap, RDBReader.class);
        ((RDBBase) reader).setSourceIds(sourceIds);
        return reader;
    }
    if (DataSourceType.HDFS.getVal().equals(sourceType)) {
        return PublicUtil.objectToObject(sourceMap, HDFSReader.class);
    }
    if (DataSourceType.HIVE.getVal().equals(sourceType) || DataSourceType.HIVE3X.getVal().equals(sourceType) || DataSourceType.HIVE1X.getVal().equals(sourceType) || DataSourceType.SparkThrift2_1.getVal().equals(sourceType)) {
        return PublicUtil.objectToObject(sourceMap, HiveReader.class);
    }
    if (DataSourceType.HBASE.getVal().equals(sourceType)) {
        return PublicUtil.objectToObject(sourceMap, HBaseReader.class);
    }
    if (DataSourceType.FTP.getVal().equals(sourceType)) {
        reader = PublicUtil.objectToObject(sourceMap, FtpReader.class);
        if (sourceMap.containsKey("isFirstLineHeader") && (Boolean) sourceMap.get("isFirstLineHeader")) {
            ((FtpReader) reader).setFirstLineHeader(true);
        } else {
            ((FtpReader) reader).setFirstLineHeader(false);
        }
        return reader;
    }
    if (DataSourceType.MAXCOMPUTE.getVal().equals(sourceType)) {
        reader = PublicUtil.objectToObject(sourceMap, OdpsReader.class);
        ((OdpsBase) reader).setSourceId(sourceIds.get(0));
        return reader;
    }
    if (DataSourceType.ES.getVal().equals(sourceType)) {
        return PublicUtil.objectToObject(sourceMap, EsReader.class);
    }
    if (DataSourceType.MONGODB.getVal().equals(sourceType)) {
        return PublicUtil.objectToObject(sourceMap, MongoDbReader.class);
    }
    if (DataSourceType.CarbonData.getVal().equals(sourceType)) {
        return PublicUtil.objectToObject(sourceMap, CarbonDataReader.class);
    }
    if (DataSourceType.Kudu.getVal().equals(sourceType)) {
        return syncBuilderFactory.getSyncBuilder(DataSourceType.Kudu.getVal()).syncReaderBuild(sourceMap, sourceIds);
    }
    if (DataSourceType.INFLUXDB.getVal().equals(sourceType)) {
        return PublicUtil.objectToObject(sourceMap, InfluxDBReader.class);
    }
    if (DataSourceType.IMPALA.getVal().equals(sourceType)) {
        // setSftpConf时，设置的hdfsConfig和sftpConf
        if (sourceMap.containsKey(HADOOP_CONFIG)) {
            Object impalaConfig = sourceMap.get(HADOOP_CONFIG);
            if (impalaConfig instanceof Map) {
                sourceMap.put(HADOOP_CONFIG, impalaConfig);
                sourceMap.put("sftpConf", ((Map) impalaConfig).get("sftpConf"));
            }
        }
        return syncBuilderFactory.getSyncBuilder(DataSourceType.IMPALA.getVal()).syncReaderBuild(sourceMap, sourceIds);
    }
    if (DataSourceType.AWS_S3.getVal().equals(sourceType)) {
        return PublicUtil.objectToObject(sourceMap, AwsS3Reader.class);
    }
    throw new RdosDefineException("暂不支持" + DataSourceType.getSourceType(sourceType).name() + "作为数据同步的源");
}

Example 3 with Reader

use of com.dtstack.taier.develop.common.template.Reader in project Taier by DTStack.

the class DatasourceService method getSyncSql.

/**
 * 配置或修改离线任务
 *
 * @param isFilter 获取数据同步脚本时候是否进行过滤用户名密码操作
 * @return
 * @throws IOException
 */
public String getSyncSql(final TaskResourceParam param, boolean isFilter) {
    // 来源集合
    final Map<String, Object> sourceMap = param.getSourceMap();
    // 目标集合
    final Map<String, Object> targetMap = param.getTargetMap();
    // 流控、错误集合
    final Map<String, Object> settingMap = param.getSettingMap();
    try {
        this.setReaderJson(sourceMap, param.getId(), param.getTenantId(), isFilter);
        this.setWriterJson(targetMap, param.getId(), param.getTenantId(), isFilter);
        Reader reader = null;
        Writer writer = null;
        Setting setting = null;
        final Integer sourceType = Integer.parseInt(sourceMap.get("dataSourceType").toString());
        final Integer targetType = Integer.parseInt(targetMap.get("dataSourceType").toString());
        if (!this.checkDataSourcePermission(sourceType, EDataSourcePermission.READ.getType())) {
            throw new RdosDefineException(ErrorCode.SOURCE_CAN_NOT_AS_INPUT);
        }
        if (!this.checkDataSourcePermission(targetType, EDataSourcePermission.WRITE.getType())) {
            throw new RdosDefineException(ErrorCode.SOURCE_CAN_NOT_AS_OUTPUT);
        }
        final List<Long> sourceIds = (List<Long>) sourceMap.get("sourceIds");
        final List<Long> targetIds = (List<Long>) targetMap.get("sourceIds");
        reader = this.syncReaderBuild(sourceType, sourceMap, sourceIds);
        writer = this.syncWriterBuild(targetType, targetIds, targetMap, reader);
        setting = PublicUtil.objectToObject(settingMap, DefaultSetting.class);
        // 检查有效性
        if (writer instanceof HiveWriter) {
            final HiveWriter hiveWriter = (HiveWriter) writer;
            if (!hiveWriter.isValid()) {
                throw new RdosDefineException(hiveWriter.getErrMsg());
            }
        }
        if (param.getCreateModel() == TaskCreateModelType.TEMPLATE.getType()) {
            // 脚本模式直接返回
            return this.getJobText(this.putDefaultEmptyValueForReader(sourceType, reader), this.putDefaultEmptyValueForWriter(targetType, writer), this.putDefaultEmptyValueForSetting(setting));
        }
        // 获得数据同步job.xml的配置
        final String jobXml = this.getJobText(reader, writer, setting);
        final String parserXml = this.getParserText(sourceMap, targetMap, settingMap);
        final JSONObject sql = new JSONObject(3);
        sql.put("job", jobXml);
        sql.put("parser", parserXml);
        sql.put("createModel", TaskCreateModelType.GUIDE.getType());
        this.batchTaskParamService.checkParams(this.batchTaskParamService.checkSyncJobParams(sql.toJSONString()), param.getTaskVariables());
        return sql.toJSONString();
    } catch (final Exception e) {
        LOGGER.error("", e);
        throw new RdosDefineException("解析同步任务失败: " + e.getMessage(), ErrorCode.SERVER_EXCEPTION);
    }
}

Example 4 with Reader

use of com.dtstack.taier.develop.common.template.Reader in project Taier by DTStack.

the class JobTemplate method toJobJsonString.

public String toJobJsonString() {
    Reader reader = newReader();
    Writer writer = newWrite();
    Setting setting = newSetting();
    JSONObject content = new JSONObject(2);
    content.put("reader", reader.toReaderJson());
    content.put("writer", writer.toWriterJson());
    JSONObject jobJson = new JSONObject(2);
    jobJson.put("content", Lists.newArrayList(content));
    jobJson.put("setting", setting.toSettingJson());
    StringBuilder job = new StringBuilder();
    job.append("{ \"job\":");
    job.append(jobJson.toJSONString());
    job.append(" }");
    return job.toString();
}