Examples with HoodieKeyGeneratorException org.apache.hudi.exception.HoodieKeyGeneratorException used on opensource projects

Example 1 with HoodieKeyGeneratorException

use of org.apache.hudi.exception.HoodieKeyGeneratorException in project hudi by apache.

the class TestCreateAvroKeyGeneratorByTypeWithFactory method testKeyGeneratorTypes.

@ParameterizedTest
@MethodSource("configParams")
public void testKeyGeneratorTypes(String keyGenType) throws IOException {
    props.put(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), keyGenType);
    KeyGeneratorType keyType = KeyGeneratorType.valueOf(keyGenType);
    KeyGenerator keyGenerator = HoodieAvroKeyGeneratorFactory.createKeyGenerator(props);
    switch(keyType) {
        case SIMPLE:
            Assertions.assertEquals(SimpleAvroKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        case COMPLEX:
            Assertions.assertEquals(ComplexAvroKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        case TIMESTAMP:
            Assertions.assertEquals(TimestampBasedAvroKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        case CUSTOM:
            Assertions.assertEquals(CustomAvroKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        case NON_PARTITION:
            Assertions.assertEquals(NonpartitionedAvroKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        case GLOBAL_DELETE:
            Assertions.assertEquals(GlobalAvroDeleteKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        default:
            throw new HoodieKeyGeneratorException("Unsupported keyGenerator Type " + keyGenType);
    }
}

Example 2 with HoodieKeyGeneratorException

use of org.apache.hudi.exception.HoodieKeyGeneratorException in project hudi by apache.

the class HoodieAvroKeyGeneratorFactory method createAvroKeyGeneratorByType.

private static KeyGenerator createAvroKeyGeneratorByType(TypedProperties props) throws IOException {
    // Use KeyGeneratorType.SIMPLE as default keyGeneratorType
    String keyGeneratorType = props.getString(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), null);
    if (StringUtils.isNullOrEmpty(keyGeneratorType)) {
        LOG.info("The value of {} is empty, using SIMPLE", HoodieWriteConfig.KEYGENERATOR_TYPE.key());
        keyGeneratorType = KeyGeneratorType.SIMPLE.name();
    }
    KeyGeneratorType keyGeneratorTypeEnum;
    try {
        keyGeneratorTypeEnum = KeyGeneratorType.valueOf(keyGeneratorType.toUpperCase(Locale.ROOT));
    } catch (IllegalArgumentException e) {
        throw new HoodieKeyGeneratorException("Unsupported keyGenerator Type " + keyGeneratorType);
    }
    switch(keyGeneratorTypeEnum) {
        case SIMPLE:
            return new SimpleAvroKeyGenerator(props);
        case COMPLEX:
            return new ComplexAvroKeyGenerator(props);
        case TIMESTAMP:
            return new TimestampBasedAvroKeyGenerator(props);
        case CUSTOM:
            return new CustomAvroKeyGenerator(props);
        case NON_PARTITION:
            return new NonpartitionedAvroKeyGenerator(props);
        case GLOBAL_DELETE:
            return new GlobalAvroDeleteKeyGenerator(props);
        default:
            throw new HoodieKeyGeneratorException("Unsupported keyGenerator Type " + keyGeneratorType);
    }
}

Example 3 with HoodieKeyGeneratorException

use of org.apache.hudi.exception.HoodieKeyGeneratorException in project hudi by apache.

the class CustomAvroKeyGenerator method getPartitionPath.

@Override
public String getPartitionPath(GenericRecord record) {
    if (getPartitionPathFields() == null) {
        throw new HoodieKeyException("Unable to find field names for partition path in cfg");
    }
    String partitionPathField;
    StringBuilder partitionPath = new StringBuilder();
    // Corresponds to no partition case
    if (getPartitionPathFields().size() == 1 && getPartitionPathFields().get(0).isEmpty()) {
        return "";
    }
    for (String field : getPartitionPathFields()) {
        String[] fieldWithType = field.split(SPLIT_REGEX);
        if (fieldWithType.length != 2) {
            throw new HoodieKeyException("Unable to find field names for partition path in proper format");
        }
        partitionPathField = fieldWithType[0];
        PartitionKeyType keyType = PartitionKeyType.valueOf(fieldWithType[1].toUpperCase());
        switch(keyType) {
            case SIMPLE:
                partitionPath.append(new SimpleAvroKeyGenerator(config, partitionPathField).getPartitionPath(record));
                break;
            case TIMESTAMP:
                try {
                    partitionPath.append(new TimestampBasedAvroKeyGenerator(config, partitionPathField).getPartitionPath(record));
                } catch (IOException e) {
                    throw new HoodieKeyGeneratorException("Unable to initialise TimestampBasedKeyGenerator class", e);
                }
                break;
            default:
                throw new HoodieKeyGeneratorException("Please provide valid PartitionKeyType with fields! You provided: " + keyType);
        }
        partitionPath.append(DEFAULT_PARTITION_PATH_SEPARATOR);
    }
    partitionPath.deleteCharAt(partitionPath.length() - 1);
    return partitionPath.toString();
}

Example 4 with HoodieKeyGeneratorException

use of org.apache.hudi.exception.HoodieKeyGeneratorException in project hudi by apache.

the class TestCreateKeyGeneratorByTypeWithFactory method testKeyGeneratorTypes.

@ParameterizedTest
@MethodSource("configParams")
public void testKeyGeneratorTypes(String keyGenType) throws IOException {
    props.put(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), keyGenType);
    KeyGeneratorType keyType = KeyGeneratorType.valueOf(keyGenType);
    KeyGenerator keyGenerator = HoodieSparkKeyGeneratorFactory.createKeyGenerator(props);
    switch(keyType) {
        case SIMPLE:
            Assertions.assertEquals(SimpleKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        case COMPLEX:
            Assertions.assertEquals(ComplexKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        case TIMESTAMP:
            Assertions.assertEquals(TimestampBasedKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        case CUSTOM:
            Assertions.assertEquals(CustomKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        case NON_PARTITION:
            Assertions.assertEquals(NonpartitionedKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        case GLOBAL_DELETE:
            Assertions.assertEquals(GlobalDeleteKeyGenerator.class.getName(), keyGenerator.getClass().getName());
            return;
        default:
            throw new HoodieKeyGeneratorException("Unsupported keyGenerator Type " + keyGenType);
    }
}

Example 5 with HoodieKeyGeneratorException

use of org.apache.hudi.exception.HoodieKeyGeneratorException in project hudi by apache.

the class SparkBootstrapCommitActionExecutor method runMetadataBootstrap.

private HoodieData<BootstrapWriteStatus> runMetadataBootstrap(List<Pair<String, List<HoodieFileStatus>>> partitions) {
    if (null == partitions || partitions.isEmpty()) {
        return context.emptyHoodieData();
    }
    TypedProperties properties = new TypedProperties();
    properties.putAll(config.getProps());
    KeyGeneratorInterface keyGenerator;
    try {
        keyGenerator = HoodieSparkKeyGeneratorFactory.createKeyGenerator(properties);
    } catch (IOException e) {
        throw new HoodieKeyGeneratorException("Init keyGenerator failed ", e);
    }
    BootstrapPartitionPathTranslator translator = (BootstrapPartitionPathTranslator) ReflectionUtils.loadClass(config.getBootstrapPartitionPathTranslatorClass(), properties);
    List<Pair<String, Pair<String, HoodieFileStatus>>> bootstrapPaths = partitions.stream().flatMap(p -> {
        String translatedPartitionPath = translator.getBootstrapTranslatedPath(p.getKey());
        return p.getValue().stream().map(f -> Pair.of(p.getKey(), Pair.of(translatedPartitionPath, f)));
    }).collect(Collectors.toList());
    context.setJobStatus(this.getClass().getSimpleName(), "Bootstrap metadata table.");
    return context.parallelize(bootstrapPaths, config.getBootstrapParallelism()).map(partitionFsPair -> getMetadataHandler(config, table, partitionFsPair.getRight().getRight()).runMetadataBootstrap(partitionFsPair.getLeft(), partitionFsPair.getRight().getLeft(), keyGenerator));
}