Examples with KafkaConnectConfigs org.apache.hudi.connect.writers.KafkaConnectConfigs used on opensource projects

Example 1 with KafkaConnectConfigs

use of org.apache.hudi.connect.writers.KafkaConnectConfigs in project hudi by apache.

the class KafkaConnectUtils method getDefaultHadoopConf.

/**
 * Returns the default Hadoop Configuration.
 *
 * @return
 */
public static Configuration getDefaultHadoopConf(KafkaConnectConfigs connectConfigs) {
    Configuration hadoopConf = new Configuration();
    // add hadoop config files
    if (!StringUtils.isNullOrEmpty(connectConfigs.getHadoopConfDir()) || !StringUtils.isNullOrEmpty(connectConfigs.getHadoopConfHome())) {
        try {
            List<Path> configFiles = getHadoopConfigFiles(connectConfigs.getHadoopConfDir(), connectConfigs.getHadoopConfHome());
            configFiles.forEach(f -> hadoopConf.addResource(new org.apache.hadoop.fs.Path(f.toAbsolutePath().toUri())));
        } catch (Exception e) {
            throw new HoodieException("Failed to read hadoop configuration!", e);
        }
    } else {
        DEFAULT_HADOOP_CONF_FILES.forEach(f -> hadoopConf.addResource(new org.apache.hadoop.fs.Path(f.toAbsolutePath().toUri())));
    }
    connectConfigs.getProps().keySet().stream().filter(prop -> {
        // configuration items before passing to hadoop/hive configs
        return !prop.toString().startsWith(HOODIE_CONF_PREFIX);
    }).forEach(prop -> {
        hadoopConf.set(prop.toString(), connectConfigs.getProps().get(prop.toString()).toString());
    });
    return hadoopConf;
}

Example 2 with KafkaConnectConfigs

use of org.apache.hudi.connect.writers.KafkaConnectConfigs in project hudi by apache.

the class TestHdfsConfiguration method testKafkaConfig.

@Test
public void testKafkaConfig() throws Exception {
    KafkaConnectConfigs connectConfigs = KafkaConnectConfigs.newBuilder().withHadoopHome("src/test/resources/hadoop_home").build();
    List<Path> paths = KafkaConnectUtils.getHadoopConfigFiles(connectConfigs.getHadoopConfDir(), connectConfigs.getHadoopConfHome());
    assertEquals(paths.size(), 2);
    assertTrue(checkFiles(paths));
}