Examples with HiveSyncConfig org.apache.hudi.hive.HiveSyncConfig used on opensource projects

Example 16 with HiveSyncConfig

use of org.apache.hudi.hive.HiveSyncConfig in project hudi by apache.

the class TestDataSourceUtils method testBuildHiveSyncConfig.

@ParameterizedTest
@ValueSource(booleans = { true, false })
public void testBuildHiveSyncConfig(boolean useSyncMode) {
    TypedProperties props = new TypedProperties();
    if (useSyncMode) {
        props.setProperty(DataSourceWriteOptions.HIVE_SYNC_MODE().key(), HMS.name());
        props.setProperty(DataSourceWriteOptions.HIVE_USE_JDBC().key(), String.valueOf(false));
    }
    props.setProperty(DataSourceWriteOptions.HIVE_DATABASE().key(), HIVE_DATABASE);
    props.setProperty(DataSourceWriteOptions.HIVE_TABLE().key(), HIVE_TABLE);
    HiveSyncConfig hiveSyncConfig = DataSourceUtils.buildHiveSyncConfig(props, config.getBasePath(), PARQUET.name());
    if (useSyncMode) {
        assertFalse(hiveSyncConfig.useJdbc);
        assertEquals(HMS.name(), hiveSyncConfig.syncMode);
    } else {
        assertTrue(hiveSyncConfig.useJdbc);
        assertNull(hiveSyncConfig.syncMode);
    }
    assertEquals(HIVE_DATABASE, hiveSyncConfig.databaseName);
    assertEquals(HIVE_TABLE, hiveSyncConfig.tableName);
}

Example 17 with HiveSyncConfig

use of org.apache.hudi.hive.HiveSyncConfig in project hudi by apache.

the class DataSourceUtils method buildHiveSyncConfig.

public static HiveSyncConfig buildHiveSyncConfig(TypedProperties props, String basePath, String baseFileFormat) {
    checkRequiredProperties(props, Collections.singletonList(DataSourceWriteOptions.HIVE_TABLE().key()));
    HiveSyncConfig hiveSyncConfig = new HiveSyncConfig();
    hiveSyncConfig.basePath = basePath;
    hiveSyncConfig.usePreApacheInputFormat = props.getBoolean(DataSourceWriteOptions.HIVE_USE_PRE_APACHE_INPUT_FORMAT().key(), Boolean.parseBoolean(DataSourceWriteOptions.HIVE_USE_PRE_APACHE_INPUT_FORMAT().defaultValue()));
    hiveSyncConfig.databaseName = props.getString(DataSourceWriteOptions.HIVE_DATABASE().key(), DataSourceWriteOptions.HIVE_DATABASE().defaultValue());
    hiveSyncConfig.tableName = props.getString(DataSourceWriteOptions.HIVE_TABLE().key());
    hiveSyncConfig.baseFileFormat = baseFileFormat;
    hiveSyncConfig.hiveUser = props.getString(DataSourceWriteOptions.HIVE_USER().key(), DataSourceWriteOptions.HIVE_USER().defaultValue());
    hiveSyncConfig.hivePass = props.getString(DataSourceWriteOptions.HIVE_PASS().key(), DataSourceWriteOptions.HIVE_PASS().defaultValue());
    hiveSyncConfig.jdbcUrl = props.getString(DataSourceWriteOptions.HIVE_URL().key(), DataSourceWriteOptions.HIVE_URL().defaultValue());
    hiveSyncConfig.metastoreUris = props.getString(DataSourceWriteOptions.METASTORE_URIS().key(), DataSourceWriteOptions.METASTORE_URIS().defaultValue());
    hiveSyncConfig.partitionFields = props.getStringList(DataSourceWriteOptions.HIVE_PARTITION_FIELDS().key(), ",", new ArrayList<>());
    hiveSyncConfig.partitionValueExtractorClass = props.getString(DataSourceWriteOptions.HIVE_PARTITION_EXTRACTOR_CLASS().key(), SlashEncodedDayPartitionValueExtractor.class.getName());
    hiveSyncConfig.useJdbc = Boolean.valueOf(props.getString(DataSourceWriteOptions.HIVE_USE_JDBC().key(), DataSourceWriteOptions.HIVE_USE_JDBC().defaultValue()));
    if (props.containsKey(DataSourceWriteOptions.HIVE_SYNC_MODE().key())) {
        hiveSyncConfig.syncMode = props.getString(DataSourceWriteOptions.HIVE_SYNC_MODE().key());
    }
    hiveSyncConfig.autoCreateDatabase = Boolean.valueOf(props.getString(DataSourceWriteOptions.HIVE_AUTO_CREATE_DATABASE().key(), DataSourceWriteOptions.HIVE_AUTO_CREATE_DATABASE().defaultValue()));
    hiveSyncConfig.ignoreExceptions = Boolean.valueOf(props.getString(DataSourceWriteOptions.HIVE_IGNORE_EXCEPTIONS().key(), DataSourceWriteOptions.HIVE_IGNORE_EXCEPTIONS().defaultValue()));
    hiveSyncConfig.skipROSuffix = Boolean.valueOf(props.getString(DataSourceWriteOptions.HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE().key(), DataSourceWriteOptions.HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE().defaultValue()));
    hiveSyncConfig.supportTimestamp = Boolean.valueOf(props.getString(DataSourceWriteOptions.HIVE_SUPPORT_TIMESTAMP_TYPE().key(), DataSourceWriteOptions.HIVE_SUPPORT_TIMESTAMP_TYPE().defaultValue()));
    hiveSyncConfig.isConditionalSync = Boolean.valueOf(props.getString(DataSourceWriteOptions.HIVE_CONDITIONAL_SYNC().key(), DataSourceWriteOptions.HIVE_CONDITIONAL_SYNC().defaultValue()));
    hiveSyncConfig.bucketSpec = props.getBoolean(DataSourceWriteOptions.HIVE_SYNC_BUCKET_SYNC().key(), (boolean) DataSourceWriteOptions.HIVE_SYNC_BUCKET_SYNC().defaultValue()) ? HiveSyncConfig.getBucketSpec(props.getString(HoodieIndexConfig.BUCKET_INDEX_HASH_FIELD.key()), props.getInteger(HoodieIndexConfig.BUCKET_INDEX_NUM_BUCKETS.key())) : null;
    if (props.containsKey(HiveExternalCatalog.CREATED_SPARK_VERSION())) {
        hiveSyncConfig.sparkVersion = props.getString(HiveExternalCatalog.CREATED_SPARK_VERSION());
    }
    hiveSyncConfig.syncComment = Boolean.valueOf(props.getString(DataSourceWriteOptions.HIVE_SYNC_COMMENT().key(), DataSourceWriteOptions.HIVE_SYNC_COMMENT().defaultValue()));
    return hiveSyncConfig;
}

Example 18 with HiveSyncConfig

use of org.apache.hudi.hive.HiveSyncConfig in project hudi by apache.

the class KafkaConnectTransactionServices method syncHive.

private void syncHive() {
    HiveSyncConfig hiveSyncConfig = KafkaConnectUtils.buildSyncConfig(new TypedProperties(connectConfigs.getProps()), tableBasePath);
    String url;
    if (!StringUtils.isNullOrEmpty(hiveSyncConfig.syncMode) && HiveSyncMode.of(hiveSyncConfig.syncMode) == HiveSyncMode.HMS) {
        url = hadoopConf.get(KafkaConnectConfigs.HIVE_METASTORE_URIS);
    } else {
        url = hiveSyncConfig.jdbcUrl;
    }
    LOG.info("Syncing target hoodie table with hive table(" + hiveSyncConfig.tableName + "). Hive URL :" + url + ", basePath :" + tableBasePath);
    LOG.info("Hive Sync Conf => " + hiveSyncConfig);
    FileSystem fs = FSUtils.getFs(tableBasePath, hadoopConf);
    HiveConf hiveConf = new HiveConf();
    hiveConf.addResource(fs.getConf());
    LOG.info("Hive Conf => " + hiveConf.getAllProperties().toString());
    new HiveSyncTool(hiveSyncConfig, hiveConf, fs).syncHoodieTable();
}

Example 19 with HiveSyncConfig

use of org.apache.hudi.hive.HiveSyncConfig in project hudi by apache.

the class KafkaConnectUtils method buildSyncConfig.

/**
 * Build Hive Sync Config
 * Note: This method is a temporary solution.
 * Future solutions can be referred to: https://issues.apache.org/jira/browse/HUDI-3199
 */
public static HiveSyncConfig buildSyncConfig(TypedProperties props, String tableBasePath) {
    HiveSyncConfig hiveSyncConfig = new HiveSyncConfig();
    hiveSyncConfig.basePath = tableBasePath;
    hiveSyncConfig.usePreApacheInputFormat = props.getBoolean(KafkaConnectConfigs.HIVE_USE_PRE_APACHE_INPUT_FORMAT, false);
    hiveSyncConfig.databaseName = props.getString(KafkaConnectConfigs.HIVE_DATABASE, "default");
    hiveSyncConfig.tableName = props.getString(KafkaConnectConfigs.HIVE_TABLE, "");
    hiveSyncConfig.hiveUser = props.getString(KafkaConnectConfigs.HIVE_USER, "");
    hiveSyncConfig.hivePass = props.getString(KafkaConnectConfigs.HIVE_PASS, "");
    hiveSyncConfig.jdbcUrl = props.getString(KafkaConnectConfigs.HIVE_URL, "");
    hiveSyncConfig.partitionFields = props.getStringList(KafkaConnectConfigs.HIVE_PARTITION_FIELDS, ",", Collections.emptyList());
    hiveSyncConfig.partitionValueExtractorClass = props.getString(KafkaConnectConfigs.HIVE_PARTITION_EXTRACTOR_CLASS, SlashEncodedDayPartitionValueExtractor.class.getName());
    hiveSyncConfig.useJdbc = props.getBoolean(KafkaConnectConfigs.HIVE_USE_JDBC, true);
    if (props.containsKey(KafkaConnectConfigs.HIVE_SYNC_MODE)) {
        hiveSyncConfig.syncMode = props.getString(KafkaConnectConfigs.HIVE_SYNC_MODE);
    }
    hiveSyncConfig.autoCreateDatabase = props.getBoolean(KafkaConnectConfigs.HIVE_AUTO_CREATE_DATABASE, true);
    hiveSyncConfig.ignoreExceptions = props.getBoolean(KafkaConnectConfigs.HIVE_IGNORE_EXCEPTIONS, false);
    hiveSyncConfig.skipROSuffix = props.getBoolean(KafkaConnectConfigs.HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE, false);
    hiveSyncConfig.supportTimestamp = props.getBoolean(KafkaConnectConfigs.HIVE_SUPPORT_TIMESTAMP_TYPE, false);
    return hiveSyncConfig;
}

Example 20 with HiveSyncConfig

use of org.apache.hudi.hive.HiveSyncConfig in project hudi by apache.

the class HiveSyncContext method create.

public static HiveSyncContext create(Configuration conf) {
    HiveSyncConfig syncConfig = buildSyncConfig(conf);
    org.apache.hadoop.conf.Configuration hadoopConf = StreamerUtil.getHadoopConf();
    String path = conf.getString(FlinkOptions.PATH);
    FileSystem fs = FSUtils.getFs(path, hadoopConf);
    HiveConf hiveConf = new HiveConf();
    if (!FlinkOptions.isDefaultValueDefined(conf, FlinkOptions.HIVE_SYNC_METASTORE_URIS)) {
        hadoopConf.set(HiveConf.ConfVars.METASTOREURIS.varname, conf.getString(FlinkOptions.HIVE_SYNC_METASTORE_URIS));
    }
    hiveConf.addResource(hadoopConf);
    return new HiveSyncContext(syncConfig, hiveConf, fs);
}