Examples with HiveSyncConfig org.apache.hudi.hive.HiveSyncConfig used on opensource projects

Example 1 with HiveSyncConfig

use of org.apache.hudi.hive.HiveSyncConfig in project hudi by apache.

the class BootstrapExecutor method syncHive.

/**
 * Sync to Hive.
 */
private void syncHive() {
    if (cfg.enableHiveSync || cfg.enableMetaSync) {
        HiveSyncConfig hiveSyncConfig = DataSourceUtils.buildHiveSyncConfig(props, cfg.targetBasePath, cfg.baseFileFormat);
        HiveConf hiveConf = new HiveConf(fs.getConf(), HiveConf.class);
        hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, hiveSyncConfig.metastoreUris);
        LOG.info("Hive Conf => " + hiveConf.getAllProperties().toString());
        LOG.info("Hive Sync Conf => " + hiveSyncConfig);
        new HiveSyncTool(hiveSyncConfig, new HiveConf(configuration, HiveConf.class), fs).syncHoodieTable();
    }
}

Example 2 with HiveSyncConfig

use of org.apache.hudi.hive.HiveSyncConfig in project hudi by apache.

the class DeltaSync method syncHive.

public void syncHive() {
    HiveSyncConfig hiveSyncConfig = DataSourceUtils.buildHiveSyncConfig(props, cfg.targetBasePath, cfg.baseFileFormat);
    HiveConf hiveConf = new HiveConf(conf, HiveConf.class);
    if (StringUtils.isNullOrEmpty(hiveConf.get(HiveConf.ConfVars.METASTOREURIS.varname))) {
        hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, hiveSyncConfig.metastoreUris);
    }
    LOG.info("Hive Conf => " + hiveConf.getAllProperties().toString());
    LOG.info("Hive Sync Conf => " + hiveSyncConfig.toString());
    new HiveSyncTool(hiveSyncConfig, hiveConf, fs).syncHoodieTable();
}

Example 3 with HiveSyncConfig

use of org.apache.hudi.hive.HiveSyncConfig in project hudi by apache.

the class HiveSyncContext method buildSyncConfig.

private static HiveSyncConfig buildSyncConfig(Configuration conf) {
    HiveSyncConfig hiveSyncConfig = new HiveSyncConfig();
    hiveSyncConfig.basePath = conf.getString(FlinkOptions.PATH);
    hiveSyncConfig.baseFileFormat = conf.getString(FlinkOptions.HIVE_SYNC_FILE_FORMAT);
    hiveSyncConfig.usePreApacheInputFormat = false;
    hiveSyncConfig.databaseName = conf.getString(FlinkOptions.HIVE_SYNC_DB);
    hiveSyncConfig.tableName = conf.getString(FlinkOptions.HIVE_SYNC_TABLE);
    hiveSyncConfig.syncMode = conf.getString(FlinkOptions.HIVE_SYNC_MODE);
    hiveSyncConfig.hiveUser = conf.getString(FlinkOptions.HIVE_SYNC_USERNAME);
    hiveSyncConfig.hivePass = conf.getString(FlinkOptions.HIVE_SYNC_PASSWORD);
    hiveSyncConfig.tableProperties = conf.getString(FlinkOptions.HIVE_SYNC_TABLE_PROPERTIES);
    hiveSyncConfig.serdeProperties = conf.getString(FlinkOptions.HIVE_SYNC_TABLE_SERDE_PROPERTIES);
    hiveSyncConfig.jdbcUrl = conf.getString(FlinkOptions.HIVE_SYNC_JDBC_URL);
    hiveSyncConfig.partitionFields = Arrays.asList(FilePathUtils.extractPartitionKeys(conf));
    hiveSyncConfig.partitionValueExtractorClass = conf.getString(FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS_NAME);
    hiveSyncConfig.useJdbc = conf.getBoolean(FlinkOptions.HIVE_SYNC_USE_JDBC);
    hiveSyncConfig.useFileListingFromMetadata = conf.getBoolean(FlinkOptions.METADATA_ENABLED);
    hiveSyncConfig.ignoreExceptions = conf.getBoolean(FlinkOptions.HIVE_SYNC_IGNORE_EXCEPTIONS);
    hiveSyncConfig.supportTimestamp = conf.getBoolean(FlinkOptions.HIVE_SYNC_SUPPORT_TIMESTAMP);
    hiveSyncConfig.autoCreateDatabase = conf.getBoolean(FlinkOptions.HIVE_SYNC_AUTO_CREATE_DB);
    hiveSyncConfig.decodePartition = conf.getBoolean(FlinkOptions.URL_ENCODE_PARTITIONING);
    hiveSyncConfig.skipROSuffix = conf.getBoolean(FlinkOptions.HIVE_SYNC_SKIP_RO_SUFFIX);
    hiveSyncConfig.assumeDatePartitioning = conf.getBoolean(FlinkOptions.HIVE_SYNC_ASSUME_DATE_PARTITION);
    hiveSyncConfig.withOperationField = conf.getBoolean(FlinkOptions.CHANGELOG_ENABLED);
    return hiveSyncConfig;
}

Example 4 with HiveSyncConfig

use of org.apache.hudi.hive.HiveSyncConfig in project hudi by apache.

the class HiveSyncFunctionalTestHarness method dropDatabases.

public void dropDatabases(String... databases) throws IOException, HiveException, MetaException {
    HiveSyncConfig hiveSyncConfig = hiveSyncConf();
    for (String database : databases) {
        hiveSyncConfig.databaseName = database;
        new HiveQueryDDLExecutor(hiveSyncConfig, fs(), hiveConf()).runSQL("drop database if exists " + database);
    }
}

Example 5 with HiveSyncConfig

use of org.apache.hudi.hive.HiveSyncConfig in project hudi by apache.

the class HiveSyncFunctionalTestHarness method hiveSyncConf.

public HiveSyncConfig hiveSyncConf() throws IOException {
    HiveSyncConfig conf = new HiveSyncConfig();
    conf.jdbcUrl = hiveTestService.getJdbcHive2Url();
    conf.hiveUser = "";
    conf.hivePass = "";
    conf.databaseName = "hivesynctestdb";
    conf.tableName = "hivesynctesttable";
    conf.basePath = Files.createDirectories(tempDir.resolve("hivesynctestcase-" + Instant.now().toEpochMilli())).toUri().toString();
    conf.assumeDatePartitioning = true;
    conf.usePreApacheInputFormat = false;
    conf.partitionFields = Collections.singletonList("datestr");
    return conf;
}