Examples with TypedProperties org.apache.hudi.common.config.TypedProperties used on opensource projects

Example 26 with TypedProperties

use of org.apache.hudi.common.config.TypedProperties in project hudi by apache.

the class TestTypedProperties method testGetInteger.

@Test
public void testGetInteger() {
    Properties properties = new Properties();
    properties.put("key1", "123");
    TypedProperties typedProperties = new TypedProperties(properties);
    assertEquals(123, typedProperties.getInteger("key1"));
    assertEquals(123, typedProperties.getInteger("key1", 456));
    assertEquals(456, typedProperties.getInteger("key2", 456));
}

Example 27 with TypedProperties

use of org.apache.hudi.common.config.TypedProperties in project hudi by apache.

the class TestTypedProperties method testGetDouble.

@Test
public void testGetDouble() {
    Properties properties = new Properties();
    properties.put("key1", "123.4");
    TypedProperties typedProperties = new TypedProperties(properties);
    assertEquals(123.4, typedProperties.getDouble("key1"));
    assertEquals(123.4, typedProperties.getDouble("key1", 0.001D));
    assertEquals(0.001D, typedProperties.getDouble("key2", 0.001D));
}

Example 28 with TypedProperties

use of org.apache.hudi.common.config.TypedProperties in project hudi by apache.

the class TestTypedProperties method testGetBoolean.

@Test
public void testGetBoolean() {
    Properties properties = new Properties();
    properties.put("key1", "true");
    TypedProperties typedProperties = new TypedProperties(properties);
    assertTrue(typedProperties.getBoolean("key1"));
    assertTrue(typedProperties.getBoolean("key1", false));
    assertFalse(typedProperties.getBoolean("key2", false));
    // test getBoolean with non-string value for key2
    properties.put("key2", true);
    typedProperties = new TypedProperties(properties);
    assertTrue(typedProperties.getBoolean("key1", false));
    assertTrue(typedProperties.getBoolean("key2", false));
    // put non-string value in TypedProperties
    typedProperties.put("key3", true);
    assertTrue(typedProperties.getBoolean("key3", false));
}

Example 29 with TypedProperties

use of org.apache.hudi.common.config.TypedProperties in project hudi by apache.

the class TestTypedProperties method testGetString.

@Test
public void testGetString() {
    Properties properties = new Properties();
    properties.put("key1", "value1");
    TypedProperties typedProperties = new TypedProperties(properties);
    assertEquals("value1", typedProperties.getString("key1"));
    assertEquals("value1", typedProperties.getString("key1", "default"));
    assertEquals("default", typedProperties.getString("key2", "default"));
}

Example 30 with TypedProperties

use of org.apache.hudi.common.config.TypedProperties in project hudi by apache.

the class SparkMain method doBootstrap.

private static int doBootstrap(JavaSparkContext jsc, String tableName, String tableType, String basePath, String sourcePath, String recordKeyCols, String partitionFields, String parallelism, String schemaProviderClass, String bootstrapIndexClass, String selectorClass, String keyGenerator, String fullBootstrapInputProvider, String payloadClassName, String enableHiveSync, String propsFilePath, List<String> configs) throws IOException {
    TypedProperties properties = propsFilePath == null ? UtilHelpers.buildProperties(configs) : UtilHelpers.readConfig(jsc.hadoopConfiguration(), new Path(propsFilePath), configs).getProps(true);
    properties.setProperty(HoodieBootstrapConfig.BASE_PATH.key(), sourcePath);
    if (!StringUtils.isNullOrEmpty(keyGenerator) && KeyGeneratorType.getNames().contains(keyGenerator.toUpperCase(Locale.ROOT))) {
        properties.setProperty(HoodieBootstrapConfig.KEYGEN_TYPE.key(), keyGenerator.toUpperCase(Locale.ROOT));
    } else {
        properties.setProperty(HoodieBootstrapConfig.KEYGEN_CLASS_NAME.key(), keyGenerator);
    }
    properties.setProperty(HoodieBootstrapConfig.FULL_BOOTSTRAP_INPUT_PROVIDER_CLASS_NAME.key(), fullBootstrapInputProvider);
    properties.setProperty(HoodieBootstrapConfig.PARALLELISM_VALUE.key(), parallelism);
    properties.setProperty(HoodieBootstrapConfig.MODE_SELECTOR_CLASS_NAME.key(), selectorClass);
    properties.setProperty(DataSourceWriteOptions.RECORDKEY_FIELD().key(), recordKeyCols);
    properties.setProperty(DataSourceWriteOptions.PARTITIONPATH_FIELD().key(), partitionFields);
    HoodieDeltaStreamer.Config cfg = new HoodieDeltaStreamer.Config();
    cfg.targetTableName = tableName;
    cfg.targetBasePath = basePath;
    cfg.tableType = tableType;
    cfg.schemaProviderClassName = schemaProviderClass;
    cfg.bootstrapIndexClass = bootstrapIndexClass;
    cfg.payloadClassName = payloadClassName;
    cfg.enableHiveSync = Boolean.valueOf(enableHiveSync);
    new BootstrapExecutor(cfg, jsc, FSUtils.getFs(basePath, jsc.hadoopConfiguration()), jsc.hadoopConfiguration(), properties).execute();
    return 0;
}