Search in sources :

Example 6 with BaseKeyGenerator

use of org.apache.hudi.keygen.BaseKeyGenerator in project hudi by apache.

the class SingleSparkJobExecutionStrategy method transform.

/**
 * Transform IndexedRecord into HoodieRecord.
 */
private HoodieRecord<T> transform(IndexedRecord indexedRecord) {
    GenericRecord record = (GenericRecord) indexedRecord;
    Option<BaseKeyGenerator> keyGeneratorOpt = Option.empty();
    if (!getWriteConfig().populateMetaFields()) {
        try {
            keyGeneratorOpt = Option.of((BaseKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(new TypedProperties(getWriteConfig().getProps())));
        } catch (IOException e) {
            throw new HoodieIOException("Only BaseKeyGenerators are supported when meta columns are disabled ", e);
        }
    }
    String key = KeyGenUtils.getRecordKeyFromGenericRecord(record, keyGeneratorOpt);
    String partition = KeyGenUtils.getPartitionPathFromGenericRecord(record, keyGeneratorOpt);
    HoodieKey hoodieKey = new HoodieKey(key, partition);
    HoodieRecordPayload avroPayload = new RewriteAvroPayload(record);
    HoodieRecord hoodieRecord = new HoodieAvroRecord(hoodieKey, avroPayload);
    return hoodieRecord;
}
Also used : HoodieIOException(org.apache.hudi.exception.HoodieIOException) HoodieAvroRecord(org.apache.hudi.common.model.HoodieAvroRecord) HoodieRecord(org.apache.hudi.common.model.HoodieRecord) HoodieKey(org.apache.hudi.common.model.HoodieKey) IOException(java.io.IOException) HoodieIOException(org.apache.hudi.exception.HoodieIOException) RewriteAvroPayload(org.apache.hudi.common.model.RewriteAvroPayload) GenericRecord(org.apache.avro.generic.GenericRecord) TypedProperties(org.apache.hudi.common.config.TypedProperties) HoodieRecordPayload(org.apache.hudi.common.model.HoodieRecordPayload) BaseKeyGenerator(org.apache.hudi.keygen.BaseKeyGenerator)

Aggregations

HoodieKey (org.apache.hudi.common.model.HoodieKey)6 HoodieRecord (org.apache.hudi.common.model.HoodieRecord)6 BaseKeyGenerator (org.apache.hudi.keygen.BaseKeyGenerator)6 GenericRecord (org.apache.avro.generic.GenericRecord)5 IOException (java.io.IOException)4 HoodieAvroRecord (org.apache.hudi.common.model.HoodieAvroRecord)4 HoodieRecordPayload (org.apache.hudi.common.model.HoodieRecordPayload)4 ArrayList (java.util.ArrayList)3 List (java.util.List)3 TypedProperties (org.apache.hudi.common.config.TypedProperties)3 HoodieIOException (org.apache.hudi.exception.HoodieIOException)3 HashMap (java.util.HashMap)2 HashSet (java.util.HashSet)2 Map (java.util.Map)2 Properties (java.util.Properties)2 Set (java.util.Set)2 Collectors (java.util.stream.Collectors)2 Stream (java.util.stream.Stream)2 Path (org.apache.hadoop.fs.Path)2 HoodieBaseFile (org.apache.hudi.common.model.HoodieBaseFile)2