Examples with HiveUserToken com.qlangtech.tis.config.hive.HiveUserToken used on opensource projects

Example 1 with HiveUserToken

use of com.qlangtech.tis.config.hive.HiveUserToken in project plugins by qlangtech.

the class HudiDumpPostTask method writeSourceProps.

private void writeSourceProps(ITISFileSystem fs, IPath dumpDir, IPath fsSourcePropsPath) {
    IPath fsSourceSchemaPath = HudiTableMeta.createFsSourceSchema(fs, this.hudiTab.getName(), dumpDir, this.hudiTab);
    IPath tabDumpParentPath = TisDataXHudiWriter.createTabDumpParentPath(fs, dumpDir);
    try (OutputStream write = fs.create(fsSourcePropsPath, true)) {
        // TypedProperties props = new TypedProperties();
        TypedPropertiesBuilder props = new TypedPropertiesBuilder();
        String shuffleParallelism = String.valueOf(this.hudiWriter.shuffleParallelism);
        props.setProperty("hoodie.upsert.shuffle.parallelism", shuffleParallelism);
        props.setProperty("hoodie.insert.shuffle.parallelism", (shuffleParallelism));
        props.setProperty("hoodie.delete.shuffle.parallelism", (shuffleParallelism));
        props.setProperty("hoodie.bulkinsert.shuffle.parallelism", (shuffleParallelism));
        props.setProperty("hoodie.embed.timeline.server", "true");
        props.setProperty("hoodie.filesystem.view.type", "EMBEDDED_KV_STORE");
        // @see HoodieCompactionConfig.INLINE_COMPACT
        // props.setProperty("hoodie.compact.inline", (hudiTabType == HudiWriteTabType.MOR) ? "true" : "false");
        // BasicFSWriter writerPlugin = this.getWriterPlugin();
        // https://spark.apache.org/docs/3.2.1/sql-data-sources-csv.html
        props.setProperty("hoodie.deltastreamer.source.dfs.root", String.valueOf(tabDumpParentPath));
        props.setProperty("hoodie.deltastreamer.csv.header", Boolean.toString(TisDataXHudiWriter.CSV_FILE_USE_HEADER));
        props.setProperty("hoodie.deltastreamer.csv.sep", String.valueOf(TisDataXHudiWriter.CSV_Column_Separator));
        props.setProperty("hoodie.deltastreamer.csv.nullValue", TisDataXHudiWriter.CSV_NULL_VALUE);
        props.setProperty("hoodie.deltastreamer.csv.escape", String.valueOf(TisDataXHudiWriter.CSV_ESCAPE_CHAR));
        // props.setProperty("hoodie.deltastreamer.csv.escapeQuotes", "false");
        props.setProperty("hoodie.deltastreamer.schemaprovider.source.schema.file", String.valueOf(fsSourceSchemaPath));
        props.setProperty("hoodie.deltastreamer.schemaprovider.target.schema.file", String.valueOf(fsSourceSchemaPath));
        // please reference: DataSourceWriteOptions , HiveSyncConfig
        final IHiveConnGetter hiveMeta = this.hudiWriter.getHiveConnMeta();
        props.setProperty("hoodie.datasource.hive_sync.database", hiveMeta.getDbName());
        props.setProperty("hoodie.datasource.hive_sync.table", this.hudiTab.getName());
        if (this.hudiTab.partition == null) {
            throw new IllegalStateException("hudiPlugin.partitionedBy can not be empty");
        }
        this.hudiTab.partition.setProps(props, this.hudiWriter);
        // props.setProperty("hoodie.datasource.hive_sync.partition_fields", hudiPlugin.partitionedBy);
        // // "org.apache.hudi.hive.MultiPartKeysValueExtractor";
        // // partition 分区值抽取类
        // props.setProperty("hoodie.datasource.hive_sync.partition_extractor_class"
        // , "org.apache.hudi.hive.MultiPartKeysValueExtractor");
        Optional<HiveUserToken> hiveUserToken = hiveMeta.getUserToken();
        if (hiveUserToken.isPresent()) {
            HiveUserToken token = hiveUserToken.get();
            props.setProperty("hoodie.datasource.hive_sync.username", token.userName);
            props.setProperty("hoodie.datasource.hive_sync.password", token.password);
        }
        props.setProperty("hoodie.datasource.hive_sync.jdbcurl", hiveMeta.getJdbcUrl());
        props.setProperty("hoodie.datasource.hive_sync.mode", "jdbc");
        props.setProperty("hoodie.datasource.write.recordkey.field", this.hudiTab.recordField);
        // props.setProperty("hoodie.datasource.write.partitionpath.field", hudiWriter.partitionedBy);
        props.store(write);
    } catch (IOException e) {
        throw new RuntimeException("faild to write " + tabDumpParentPath + " CSV file metaData", e);
    }
}

Example 2 with HiveUserToken

use of com.qlangtech.tis.config.hive.HiveUserToken in project plugins by qlangtech.

the class HiveFlatTableBuilder method validateHiveAvailable.

public static boolean validateHiveAvailable(IControlMsgHandler msgHandler, Context context, Descriptor.PostFormVals postFormVals) {
    String hiveAddress = postFormVals.getField(KEY_HIVE_ADDRESS);
    String dbName = postFormVals.getField(KEY_DB_NAME);
    boolean useUserToken = Boolean.parseBoolean(postFormVals.getField(DefaultHiveConnGetter.KEY_USE_USERTOKEN));
    HiveUserToken userToken = null;
    if (useUserToken) {
        userToken = new HiveUserToken(postFormVals.getField(DefaultHiveConnGetter.KEY_USER_NAME), postFormVals.getField(DefaultHiveConnGetter.KEY_PASSWORD));
        if (StringUtils.isBlank(userToken.userName)) {
            msgHandler.addFieldError(context, DefaultHiveConnGetter.KEY_USER_NAME, ValidatorCommons.MSG_EMPTY_INPUT_ERROR);
            return false;
        }
    }
    Connection conn = null;
    try {
        conn = HiveDBUtils.getInstance(hiveAddress, dbName, Optional.ofNullable(userToken)).createConnection();
    } catch (Throwable e) {
        Throwable[] throwables = ExceptionUtils.getThrowables(e);
        for (Throwable t : throwables) {
            if (StringUtils.indexOf(t.getMessage(), "refused") > -1) {
                msgHandler.addFieldError(context, KEY_HIVE_ADDRESS, "连接地址不可用，请确保连接Hive服务地址可用");
                return false;
            }
            if (StringUtils.indexOf(t.getMessage(), "NoSuchDatabaseException") > -1) {
                msgHandler.addFieldError(context, KEY_DB_NAME, "dbName:" + dbName + " is not exist ,please create");
                return false;
            }
        }
        throw e;
    } finally {
        try {
            conn.close();
        } catch (Throwable e) {
        }
    }
    return true;
}

Example 3 with HiveUserToken

use of com.qlangtech.tis.config.hive.HiveUserToken in project plugins by qlangtech.

the class HiveDBUtils method createDatasource.

// private static final String hiveHost;
private BasicDataSource createDatasource(String hiveHost, String defaultDbName, Optional<HiveUserToken> userToken) {
    if (StringUtils.isEmpty(hiveHost)) {
        throw new IllegalArgumentException("param 'hiveHost' can not be null");
    }
    if (StringUtils.isEmpty(defaultDbName)) {
        throw new IllegalArgumentException("param 'defaultDbName' can not be null");
    }
    BasicDataSource hiveDatasource = new BasicDataSource();
    hiveDatasource.setDriverClassName("org.apache.hive.jdbc.HiveDriver");
    hiveDatasource.setDriverClassLoader(this.getClass().getClassLoader());
    Assert.assertNotNull("driverClassLoader can not be null", hiveDatasource.getDriverClassLoader());
    // hiveDatasource.setUsername("hive");
    // 这个配置是在每次操作之后连接没有有效关闭时候，定时会执行清理操作，把没有及时归还的，將2.5小時還沒有歸還pool的連接直接關閉掉
    hiveDatasource.setMaxActive(-1);
    hiveDatasource.setRemoveAbandoned(true);
    hiveDatasource.setLogAbandoned(true);
    hiveDatasource.setRemoveAbandonedTimeout(300 * 30);
    if (userToken.isPresent()) {
        HiveUserToken ut = userToken.get();
        hiveDatasource.setUsername(ut.userName);
        hiveDatasource.setPassword(ut.password);
    }
    // 测试空闲的连接是否有效
    hiveDatasource.setTestWhileIdle(true);
    if (StringUtils.isBlank(hiveHost)) {
        throw new IllegalStateException("hivehost can not be null");
    }
    // String hiveJdbcUrl = "jdbc:hive2://" + hiveHost + "/tis";
    hiveJdbcUrl = IHiveConnGetter.HIVE2_JDBC_SCHEMA + hiveHost + "/" + defaultDbName;
    hiveDatasource.setUrl(hiveJdbcUrl);
    log.info("hiveJdbcUrl:" + hiveJdbcUrl);
    return hiveDatasource;
}