Examples with TypedProperties org.apache.hudi.common.config.TypedProperties used on opensource projects

Example 66 with TypedProperties

use of org.apache.hudi.common.config.TypedProperties in project hudi by apache.

the class BulkInsertDataInternalWriterHelper method getKeyGenerator.

/**
 * Instantiate {@link BuiltinKeyGenerator}.
 *
 * @param properties properties map.
 * @return the key generator thus instantiated.
 */
private Option<BuiltinKeyGenerator> getKeyGenerator(Properties properties) {
    TypedProperties typedProperties = new TypedProperties();
    typedProperties.putAll(properties);
    if (properties.get(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key()).equals(NonpartitionedKeyGenerator.class.getName())) {
        // Do not instantiate NonPartitionKeyGen
        return Option.empty();
    } else {
        try {
            return Option.of((BuiltinKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(typedProperties));
        } catch (ClassCastException cce) {
            throw new HoodieIOException("Only those key generators implementing BuiltInKeyGenerator interface is supported with virtual keys");
        } catch (IOException e) {
            throw new HoodieIOException("Key generator instantiation failed ", e);
        }
    }
}

Example 67 with TypedProperties

use of org.apache.hudi.common.config.TypedProperties in project hudi by apache.

the class TestHoodieWriteConfig method testAutoConcurrencyConfigAdjustmentWithUserConfigs.

@ParameterizedTest
@EnumSource(HoodieTableType.class)
public void testAutoConcurrencyConfigAdjustmentWithUserConfigs(HoodieTableType tableType) {
    // 1. User override for the lock provider should always take the precedence
    TypedProperties properties = new TypedProperties();
    properties.setProperty(HoodieTableConfig.TYPE.key(), tableType.name());
    HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath("/tmp").withLockConfig(HoodieLockConfig.newBuilder().withLockProvider(FileSystemBasedLockProviderTestClass.class).build()).withProperties(properties).build();
    verifyConcurrencyControlRelatedConfigs(writeConfig, true, tableType == HoodieTableType.MERGE_ON_READ, WriteConcurrencyMode.valueOf(WRITE_CONCURRENCY_MODE.defaultValue()), HoodieFailedWritesCleaningPolicy.valueOf(FAILED_WRITES_CLEANER_POLICY.defaultValue()), FileSystemBasedLockProviderTestClass.class.getName());
    // 2. User can set the lock provider via properties
    verifyConcurrencyControlRelatedConfigs(createWriteConfig(new HashMap<String, String>() {

        {
            put(HoodieTableConfig.TYPE.key(), tableType.name());
            put(ASYNC_CLUSTERING_ENABLE.key(), "false");
            put(INLINE_COMPACT.key(), "true");
            put(AUTO_CLEAN.key(), "true");
            put(ASYNC_CLEAN.key(), "true");
            put(HoodieLockConfig.LOCK_PROVIDER_CLASS_NAME.key(), ZookeeperBasedLockProvider.class.getName());
        }
    }), true, true, WriteConcurrencyMode.valueOf(WRITE_CONCURRENCY_MODE.defaultValue()), HoodieFailedWritesCleaningPolicy.valueOf(FAILED_WRITES_CLEANER_POLICY.defaultValue()), ZookeeperBasedLockProvider.class.getName());
    // 3. Default config should have default lock provider
    writeConfig = createWriteConfig(new HashMap<String, String>() {

        {
            put(HoodieTableConfig.TYPE.key(), tableType.name());
        }
    });
    if (writeConfig.areAnyTableServicesAsync()) {
        verifyConcurrencyControlRelatedConfigs(writeConfig, true, true, WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL, HoodieFailedWritesCleaningPolicy.LAZY, InProcessLockProvider.class.getName());
    } else {
        verifyConcurrencyControlRelatedConfigs(writeConfig, true, false, WriteConcurrencyMode.valueOf(WRITE_CONCURRENCY_MODE.defaultValue()), HoodieFailedWritesCleaningPolicy.valueOf(FAILED_WRITES_CLEANER_POLICY.defaultValue()), HoodieLockConfig.LOCK_PROVIDER_CLASS_NAME.defaultValue());
    }
}

Example 68 with TypedProperties

use of org.apache.hudi.common.config.TypedProperties in project hudi by apache.

the class TestCreateAvroKeyGeneratorByTypeWithFactory method init.

@BeforeEach
public void init() {
    props = new TypedProperties();
    props.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
    props.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
    props.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
    // for timestamp based key generator
    props.put("hoodie.deltastreamer.keygen.timebased.timestamp.type", "DATE_STRING");
    props.put("hoodie.deltastreamer.keygen.timebased.input.dateformat", "yyyy-MM-dd");
    props.put("hoodie.deltastreamer.keygen.timebased.output.dateformat", "yyyyMMdd");
}

Example 69 with TypedProperties

use of org.apache.hudi.common.config.TypedProperties in project hudi by apache.

the class TestHoodieAvroKeyGeneratorFactory method testKeyGeneratorFactory.

@Test
public void testKeyGeneratorFactory() throws IOException {
    TypedProperties props = getCommonProps();
    // set KeyGenerator type only
    props.put(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), KeyGeneratorType.SIMPLE.name());
    KeyGenerator keyGenerator = HoodieAvroKeyGeneratorFactory.createKeyGenerator(props);
    Assertions.assertEquals(SimpleAvroKeyGenerator.class.getName(), keyGenerator.getClass().getName());
    // set KeyGenerator class only
    props = getCommonProps();
    props.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), SimpleAvroKeyGenerator.class.getName());
    KeyGenerator keyGenerator2 = HoodieAvroKeyGeneratorFactory.createKeyGenerator(props);
    Assertions.assertEquals(SimpleAvroKeyGenerator.class.getName(), keyGenerator2.getClass().getName());
    // set both class name and keyGenerator type
    props.put(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), KeyGeneratorType.CUSTOM.name());
    KeyGenerator keyGenerator3 = HoodieAvroKeyGeneratorFactory.createKeyGenerator(props);
    // KEYGENERATOR_TYPE_PROP was overitten by KEYGENERATOR_CLASS_PROP
    Assertions.assertEquals(SimpleAvroKeyGenerator.class.getName(), keyGenerator3.getClass().getName());
    // set wrong class name
    final TypedProperties props2 = getCommonProps();
    props2.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), TestHoodieAvroKeyGeneratorFactory.class.getName());
    assertThrows(IOException.class, () -> HoodieAvroKeyGeneratorFactory.createKeyGenerator(props2));
    // set wrong keyGenerator type
    final TypedProperties props3 = getCommonProps();
    props3.put(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), "wrong_type");
    assertThrows(HoodieKeyGeneratorException.class, () -> HoodieAvroKeyGeneratorFactory.createKeyGenerator(props3));
}

Example 70 with TypedProperties

use of org.apache.hudi.common.config.TypedProperties in project hudi by apache.

the class TestMetricsReporterFactory method metricsReporterFactoryShouldThrowExceptionWhenMetricsReporterClassIsIllegal.

@Test
public void metricsReporterFactoryShouldThrowExceptionWhenMetricsReporterClassIsIllegal() {
    when(config.getMetricReporterClassName()).thenReturn(IllegalTestMetricsReporter.class.getName());
    when(config.getProps()).thenReturn(new TypedProperties());
    assertThrows(HoodieException.class, () -> MetricsReporterFactory.createReporter(config, registry));
}