Examples with HiveColumnQueryBean org.apache.inlong.manager.common.pojo.query.hive.HiveColumnQueryBean used on opensource projects

Example 1 with HiveColumnQueryBean

use of org.apache.inlong.manager.common.pojo.query.hive.HiveColumnQueryBean in project incubator-inlong by apache.

the class HiveAddColumnSqlBuilder method buildColumns.

private String buildColumns(List<HiveColumnQueryBean> columns) {
    List<String> columnInfoList = new ArrayList<>();
    for (HiveColumnQueryBean columnBean : columns) {
        if (columnBean.isPartition()) {
            continue;
        }
        // Support _ beginning with underscore
        String columnName = "`" + columnBean.getColumnName() + "`";
        StringBuilder columnInfo = new StringBuilder().append(columnName).append(" ").append(columnBean.getColumnType());
        if (StringUtils.isNotEmpty(columnBean.getColumnDesc())) {
            // comment is not empty
            columnInfo.append(" COMMENT ").append("'").append(columnBean.getColumnDesc()).append("'");
        }
        columnInfoList.add(columnInfo.toString());
    }
    return " ADD COLUMNS (" + StringUtils.join(columnInfoList, ",") + ") ";
}

Example 2 with HiveColumnQueryBean

use of org.apache.inlong.manager.common.pojo.query.hive.HiveColumnQueryBean in project incubator-inlong by apache.

the class HiveReplaceColumnSqlBuilder method buildColumns.

private String buildColumns(List<HiveColumnQueryBean> columns) {
    List<String> columnInfoList = new ArrayList<>();
    for (HiveColumnQueryBean columnBean : columns) {
        if (columnBean.isPartition()) {
            continue;
        }
        // Support _ beginning with underscore
        String columnName = "`" + columnBean.getColumnName() + "`";
        StringBuilder columnInfo = new StringBuilder().append(columnName).append(" ").append(columnBean.getColumnType());
        if (StringUtils.isNotEmpty(columnBean.getColumnDesc())) {
            // comment is not empty
            columnInfo.append(" COMMENT ").append("'").append(columnBean.getColumnDesc()).append("'");
        }
        columnInfoList.add(columnInfo.toString());
    }
    return " REPLACE COLUMNS (" + StringUtils.join(columnInfoList, ",") + ") ";
}

Example 3 with HiveColumnQueryBean

use of org.apache.inlong.manager.common.pojo.query.hive.HiveColumnQueryBean in project incubator-inlong by apache.

the class HiveTableCreateSqlBuilder method buildColumnsAndComments.

// (col_name data_type [COMMENT col_comment], col_name data_type [COMMENT col_comment]....)
private String buildColumnsAndComments(List<HiveColumnQueryBean> columns, String tableComment) {
    List<String> columnInfoList = new ArrayList<>();
    List<String> partitionList = new ArrayList<>();
    for (HiveColumnQueryBean columnBean : columns) {
        // Construct columns and partition columns
        String columnName = "`" + columnBean.getColumnName() + "`";
        StringBuilder columnInfo = new StringBuilder().append(columnName).append(" ").append(columnBean.getColumnType());
        if (StringUtils.isNotEmpty(columnBean.getColumnDesc())) {
            // comment is not empty
            columnInfo.append(" COMMENT ").append("'").append(columnBean.getColumnDesc()).append("'");
        }
        if (columnBean.isPartition()) {
            // partition field
            partitionList.add(columnInfo.toString());
        } else {
            // hive field
            columnInfoList.add(columnInfo.toString());
        }
    }
    StringBuilder result = new StringBuilder().append(" (").append(StringUtils.join(columnInfoList, ",")).append(") ");
    if (StringUtils.isNotEmpty(tableComment)) {
        // set table comment
        result.append("COMMENT ").append("'").append(tableComment).append("' ");
    }
    if (partitionList.size() > 0) {
        // set partitions
        result.append("PARTITIONED BY (").append(StringUtils.join(partitionList, ",")).append(") ");
    }
    return result.toString();
}

Example 4 with HiveColumnQueryBean

use of org.apache.inlong.manager.common.pojo.query.hive.HiveColumnQueryBean in project incubator-inlong by apache.

the class DefaultHiveTableOperator method getTableQueryBean.

protected HiveTableQueryBean getTableQueryBean(SinkForSortDTO config, HiveSinkDTO hiveInfo) {
    String groupId = config.getInlongGroupId();
    String streamId = config.getInlongStreamId();
    LOGGER.info("begin to get table query bean for groupId={}, streamId={}", groupId, streamId);
    List<StreamSinkFieldEntity> fieldEntities = hiveFieldMapper.selectFields(groupId, streamId);
    List<HiveColumnQueryBean> columnQueryBeans = new ArrayList<>();
    for (StreamSinkFieldEntity field : fieldEntities) {
        HiveColumnQueryBean columnBean = new HiveColumnQueryBean();
        columnBean.setColumnName(field.getFieldName());
        columnBean.setColumnType(field.getFieldType());
        columnBean.setColumnDesc(field.getFieldComment());
        columnQueryBeans.add(columnBean);
    }
    // set partition field and type
    String partitionField = hiveInfo.getPrimaryPartition();
    if (partitionField != null) {
        HiveColumnQueryBean partColumn = new HiveColumnQueryBean();
        partColumn.setPartition(true);
        partColumn.setColumnName(partitionField);
        // currently, only supports 'string' type
        partColumn.setColumnType("string");
        columnQueryBeans.add(partColumn);
    }
    HiveTableQueryBean queryBean = new HiveTableQueryBean();
    queryBean.setColumns(columnQueryBeans);
    // set terminated symbol
    if (hiveInfo.getDataSeparator() != null) {
        char ch = (char) Integer.parseInt(hiveInfo.getDataSeparator());
        queryBean.setFieldTerSymbol(String.valueOf(ch));
    }
    queryBean.setUsername(hiveInfo.getUsername());
    queryBean.setPassword(hiveInfo.getPassword());
    queryBean.setTableName(hiveInfo.getTableName());
    queryBean.setDbName(hiveInfo.getDbName());
    queryBean.setJdbcUrl(hiveInfo.getJdbcUrl());
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("success to get table query bean={}", queryBean);
    }
    return queryBean;
}

Example 5 with HiveColumnQueryBean

use of org.apache.inlong.manager.common.pojo.query.hive.HiveColumnQueryBean in project incubator-inlong by apache.

the class DefaultHiveTableOperator method createTable.

private void createTable(String groupId, SinkForSortDTO config) {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("begin create hive table for inlong group={}, config={}", groupId, config);
    }
    // Get all info from config
    HiveSinkDTO hiveInfo = HiveSinkDTO.getFromJson(config.getExtParams());
    HiveTableQueryBean tableBean = getTableQueryBean(config, hiveInfo);
    try {
        // create database if not exists
        dataSourceService.createDb(tableBean);
        // check if the table exists
        List<ColumnInfoBean> columns = dataSourceService.queryColumns(tableBean);
        if (columns.size() == 0) {
            // no such table, create one
            dataSourceService.createTable(tableBean);
        } else {
            // set columns, skip the first columns already exist in hive
            List<HiveColumnQueryBean> columnsSkipHistory = tableBean.getColumns().stream().skip(columns.size()).collect(toList());
            if (columnsSkipHistory.size() != 0) {
                tableBean.setColumns(columnsSkipHistory);
                dataSourceService.createColumn(tableBean);
            }
        }
        sinkService.updateStatus(config.getId(), EntityStatus.SINK_CONFIG_SUCCESSFUL.getCode(), "create hive table success");
    } catch (Throwable e) {
        LOGGER.error("create hive table error, ", e);
        sinkService.updateStatus(config.getId(), EntityStatus.SINK_CONFIG_FAILED.getCode(), e.getMessage());
        throw new WorkflowException("create hive table failed, reason: " + e.getMessage());
    }
    LOGGER.info("success create hive table for data group [" + groupId + "]");
}