Search in sources :

Example 6 with KeyGenerator

use of org.apache.hudi.keygen.KeyGenerator in project hudi by apache.

the class TestHoodieAvroKeyGeneratorFactory method testKeyGeneratorFactory.

@Test
public void testKeyGeneratorFactory() throws IOException {
    TypedProperties props = getCommonProps();
    // set KeyGenerator type only
    props.put(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), KeyGeneratorType.SIMPLE.name());
    KeyGenerator keyGenerator = HoodieAvroKeyGeneratorFactory.createKeyGenerator(props);
    Assertions.assertEquals(SimpleAvroKeyGenerator.class.getName(), keyGenerator.getClass().getName());
    // set KeyGenerator class only
    props = getCommonProps();
    props.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), SimpleAvroKeyGenerator.class.getName());
    KeyGenerator keyGenerator2 = HoodieAvroKeyGeneratorFactory.createKeyGenerator(props);
    Assertions.assertEquals(SimpleAvroKeyGenerator.class.getName(), keyGenerator2.getClass().getName());
    // set both class name and keyGenerator type
    props.put(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), KeyGeneratorType.CUSTOM.name());
    KeyGenerator keyGenerator3 = HoodieAvroKeyGeneratorFactory.createKeyGenerator(props);
    // KEYGENERATOR_TYPE_PROP was overitten by KEYGENERATOR_CLASS_PROP
    Assertions.assertEquals(SimpleAvroKeyGenerator.class.getName(), keyGenerator3.getClass().getName());
    // set wrong class name
    final TypedProperties props2 = getCommonProps();
    props2.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), TestHoodieAvroKeyGeneratorFactory.class.getName());
    assertThrows(IOException.class, () -> HoodieAvroKeyGeneratorFactory.createKeyGenerator(props2));
    // set wrong keyGenerator type
    final TypedProperties props3 = getCommonProps();
    props3.put(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), "wrong_type");
    assertThrows(HoodieKeyGeneratorException.class, () -> HoodieAvroKeyGeneratorFactory.createKeyGenerator(props3));
}
Also used : TypedProperties(org.apache.hudi.common.config.TypedProperties) SimpleAvroKeyGenerator(org.apache.hudi.keygen.SimpleAvroKeyGenerator) KeyGenerator(org.apache.hudi.keygen.KeyGenerator) SimpleAvroKeyGenerator(org.apache.hudi.keygen.SimpleAvroKeyGenerator) Test(org.junit.jupiter.api.Test)

Example 7 with KeyGenerator

use of org.apache.hudi.keygen.KeyGenerator in project hudi by apache.

the class TestCreateKeyGeneratorByTypeWithFactory method testKeyGeneratorTypes.

@ParameterizedTest
@MethodSource("configParams")
public void testKeyGeneratorTypes(String keyGenType) throws IOException {
    props.put(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), keyGenType);
    KeyGeneratorType keyType = KeyGeneratorType.valueOf(keyGenType);
    KeyGenerator keyGenerator = HoodieSparkKeyGeneratorFactory.createKeyGenerator(props);
    switch(keyType) {
        case SIMPLE:
            Assertions.assertEquals(SimpleKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        case COMPLEX:
            Assertions.assertEquals(ComplexKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        case TIMESTAMP:
            Assertions.assertEquals(TimestampBasedKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        case CUSTOM:
            Assertions.assertEquals(CustomKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        case NON_PARTITION:
            Assertions.assertEquals(NonpartitionedKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        case GLOBAL_DELETE:
            Assertions.assertEquals(GlobalDeleteKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        default:
            throw new HoodieKeyGeneratorException("Unsupported keyGenerator Type " + keyGenType);
    }
}
Also used : HoodieKeyGeneratorException(org.apache.hudi.exception.HoodieKeyGeneratorException) GlobalDeleteKeyGenerator(org.apache.hudi.keygen.GlobalDeleteKeyGenerator) CustomKeyGenerator(org.apache.hudi.keygen.CustomKeyGenerator) ComplexKeyGenerator(org.apache.hudi.keygen.ComplexKeyGenerator) TimestampBasedKeyGenerator(org.apache.hudi.keygen.TimestampBasedKeyGenerator) SimpleKeyGenerator(org.apache.hudi.keygen.SimpleKeyGenerator) KeyGeneratorType(org.apache.hudi.keygen.constant.KeyGeneratorType) CustomKeyGenerator(org.apache.hudi.keygen.CustomKeyGenerator) TimestampBasedKeyGenerator(org.apache.hudi.keygen.TimestampBasedKeyGenerator) ComplexKeyGenerator(org.apache.hudi.keygen.ComplexKeyGenerator) SimpleKeyGenerator(org.apache.hudi.keygen.SimpleKeyGenerator) GlobalDeleteKeyGenerator(org.apache.hudi.keygen.GlobalDeleteKeyGenerator) KeyGenerator(org.apache.hudi.keygen.KeyGenerator) NonpartitionedKeyGenerator(org.apache.hudi.keygen.NonpartitionedKeyGenerator) NonpartitionedKeyGenerator(org.apache.hudi.keygen.NonpartitionedKeyGenerator) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) MethodSource(org.junit.jupiter.params.provider.MethodSource)

Aggregations

KeyGenerator (org.apache.hudi.keygen.KeyGenerator)7 TypedProperties (org.apache.hudi.common.config.TypedProperties)5 GenericRecord (org.apache.avro.generic.GenericRecord)3 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)3 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 List (java.util.List)2 HoodieAvroUtils (org.apache.hudi.avro.HoodieAvroUtils)2 HoodieRecord (org.apache.hudi.common.model.HoodieRecord)2 Option (org.apache.hudi.common.util.Option)2 HoodieKeyGeneratorException (org.apache.hudi.exception.HoodieKeyGeneratorException)2 Collections (java.util.Collections)1 Comparator (java.util.Comparator)1 Iterator (java.util.Iterator)1 Collectors (java.util.stream.Collectors)1 Schema (org.apache.avro.Schema)1 GenericDatumReader (org.apache.avro.generic.GenericDatumReader)1 IndexedRecord (org.apache.avro.generic.IndexedRecord)1 DecoderFactory (org.apache.avro.io.DecoderFactory)1 DataSourceUtils (org.apache.hudi.DataSourceUtils)1