Examples with HiveTableQueryBean org.apache.inlong.manager.common.pojo.query.hive.HiveTableQueryBean used on opensource projects

Example 1 with HiveTableQueryBean

use of org.apache.inlong.manager.common.pojo.query.hive.HiveTableQueryBean in project incubator-inlong by apache.

the class DefaultHiveTableOperator method getTableQueryBean.

protected HiveTableQueryBean getTableQueryBean(SinkForSortDTO config, HiveSinkDTO hiveInfo) {
    String groupId = config.getInlongGroupId();
    String streamId = config.getInlongStreamId();
    LOGGER.info("begin to get table query bean for groupId={}, streamId={}", groupId, streamId);
    List<StreamSinkFieldEntity> fieldEntities = hiveFieldMapper.selectFields(groupId, streamId);
    List<HiveColumnQueryBean> columnQueryBeans = new ArrayList<>();
    for (StreamSinkFieldEntity field : fieldEntities) {
        HiveColumnQueryBean columnBean = new HiveColumnQueryBean();
        columnBean.setColumnName(field.getFieldName());
        columnBean.setColumnType(field.getFieldType());
        columnBean.setColumnDesc(field.getFieldComment());
        columnQueryBeans.add(columnBean);
    }
    // set partition field and type
    String partitionField = hiveInfo.getPrimaryPartition();
    if (partitionField != null) {
        HiveColumnQueryBean partColumn = new HiveColumnQueryBean();
        partColumn.setPartition(true);
        partColumn.setColumnName(partitionField);
        // currently, only supports 'string' type
        partColumn.setColumnType("string");
        columnQueryBeans.add(partColumn);
    }
    HiveTableQueryBean queryBean = new HiveTableQueryBean();
    queryBean.setColumns(columnQueryBeans);
    // set terminated symbol
    if (hiveInfo.getDataSeparator() != null) {
        char ch = (char) Integer.parseInt(hiveInfo.getDataSeparator());
        queryBean.setFieldTerSymbol(String.valueOf(ch));
    }
    queryBean.setUsername(hiveInfo.getUsername());
    queryBean.setPassword(hiveInfo.getPassword());
    queryBean.setTableName(hiveInfo.getTableName());
    queryBean.setDbName(hiveInfo.getDbName());
    queryBean.setJdbcUrl(hiveInfo.getJdbcUrl());
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("success to get table query bean={}", queryBean);
    }
    return queryBean;
}

Example 2 with HiveTableQueryBean

use of org.apache.inlong.manager.common.pojo.query.hive.HiveTableQueryBean in project incubator-inlong by apache.

the class DefaultHiveTableOperator method createTable.

private void createTable(String groupId, SinkForSortDTO config) {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("begin create hive table for inlong group={}, config={}", groupId, config);
    }
    // Get all info from config
    HiveSinkDTO hiveInfo = HiveSinkDTO.getFromJson(config.getExtParams());
    HiveTableQueryBean tableBean = getTableQueryBean(config, hiveInfo);
    try {
        // create database if not exists
        dataSourceService.createDb(tableBean);
        // check if the table exists
        List<ColumnInfoBean> columns = dataSourceService.queryColumns(tableBean);
        if (columns.size() == 0) {
            // no such table, create one
            dataSourceService.createTable(tableBean);
        } else {
            // set columns, skip the first columns already exist in hive
            List<HiveColumnQueryBean> columnsSkipHistory = tableBean.getColumns().stream().skip(columns.size()).collect(toList());
            if (columnsSkipHistory.size() != 0) {
                tableBean.setColumns(columnsSkipHistory);
                dataSourceService.createColumn(tableBean);
            }
        }
        sinkService.updateStatus(config.getId(), EntityStatus.SINK_CONFIG_SUCCESSFUL.getCode(), "create hive table success");
    } catch (Throwable e) {
        LOGGER.error("create hive table error, ", e);
        sinkService.updateStatus(config.getId(), EntityStatus.SINK_CONFIG_FAILED.getCode(), e.getMessage());
        throw new WorkflowException("create hive table failed, reason: " + e.getMessage());
    }
    LOGGER.info("success create hive table for data group [" + groupId + "]");
}