Examples with TypedProperties org.apache.hudi.common.config.TypedProperties used on opensource projects

Example 11 with TypedProperties

use of org.apache.hudi.common.config.TypedProperties in project hudi by apache.

the class TestComplexKeyGenerator method getProps.

private TypedProperties getProps() {
    TypedProperties properties = getCommonProps(true);
    properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp,ts_ms");
    return properties;
}

Example 12 with TypedProperties

use of org.apache.hudi.common.config.TypedProperties in project hudi by apache.

the class TestComplexKeyGenerator method testMultipleValueKeyGeneratorNonPartitioned.

@Test
public void testMultipleValueKeyGeneratorNonPartitioned() {
    TypedProperties properties = new TypedProperties();
    properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key,timestamp");
    properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "");
    ComplexKeyGenerator compositeKeyGenerator = new ComplexKeyGenerator(properties);
    assertEquals(compositeKeyGenerator.getRecordKeyFields().size(), 2);
    assertEquals(compositeKeyGenerator.getPartitionPathFields().size(), 0);
    HoodieTestDataGenerator dataGenerator = new HoodieTestDataGenerator();
    GenericRecord record = dataGenerator.generateGenericRecords(1).get(0);
    String rowKey = "_row_key" + ComplexAvroKeyGenerator.DEFAULT_RECORD_KEY_SEPARATOR + record.get("_row_key").toString() + "," + "timestamp" + ComplexAvroKeyGenerator.DEFAULT_RECORD_KEY_SEPARATOR + record.get("timestamp").toString();
    String partitionPath = "";
    HoodieKey hoodieKey = compositeKeyGenerator.getKey(record);
    assertEquals(rowKey, hoodieKey.getRecordKey());
    assertEquals(partitionPath, hoodieKey.getPartitionPath());
    Row row = KeyGeneratorTestUtilities.getRow(record, HoodieTestDataGenerator.AVRO_SCHEMA, AvroConversionUtils.convertAvroSchemaToStructType(HoodieTestDataGenerator.AVRO_SCHEMA));
    Assertions.assertEquals(compositeKeyGenerator.getPartitionPath(row), partitionPath);
    InternalRow internalRow = KeyGeneratorTestUtilities.getInternalRow(row);
    Assertions.assertEquals(compositeKeyGenerator.getPartitionPath(internalRow, row.schema()), partitionPath);
}

Example 13 with TypedProperties

use of org.apache.hudi.common.config.TypedProperties in project hudi by apache.

the class TestCustomKeyGenerator method getComplexRecordKeyWithSimplePartitionProps.

private TypedProperties getComplexRecordKeyWithSimplePartitionProps(boolean useKeyGeneratorClassName) {
    TypedProperties properties = getCommonProps(true, useKeyGeneratorClassName);
    properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp:simple");
    return properties;
}

Example 14 with TypedProperties

use of org.apache.hudi.common.config.TypedProperties in project hudi by apache.

the class TestCustomKeyGenerator method getComplexRecordKeyAndPartitionPathProps.

private TypedProperties getComplexRecordKeyAndPartitionPathProps(boolean useKeyGeneratorClassName) {
    TypedProperties properties = getCommonProps(true, useKeyGeneratorClassName);
    properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp:simple,ts_ms:timestamp");
    populateNecessaryPropsForTimestampBasedKeyGen(properties);
    return properties;
}

Example 15 with TypedProperties

use of org.apache.hudi.common.config.TypedProperties in project hudi by apache.

the class TestCustomKeyGenerator method getCommonProps.

/**
 * Method to create props used for common cases.
 *
 * @param getComplexRecordKey      Use complex record key or not
 * @param useKeyGeneratorClassName Use KeyGenerator class name initialize KeyGenerator or not.
 *                                 true use {@code HoodieWriteConfig.KEYGENERATOR_CLASS_PROP},
 *                                 false use {@code HoodieWriteConfig.KEYGENERATOR_TYPE_PROP}
 * @return TypedProperties used to initialize KeyGenerator.
 */
private TypedProperties getCommonProps(boolean getComplexRecordKey, boolean useKeyGeneratorClassName) {
    TypedProperties properties = new TypedProperties();
    if (getComplexRecordKey) {
        properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key, pii_col");
    } else {
        properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
    }
    if (useKeyGeneratorClassName) {
        properties.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), CustomKeyGenerator.class.getName());
    } else {
        properties.put(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), KeyGeneratorType.CUSTOM.name());
    }
    properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
    return properties;
}