Examples with BootstrapBaseFileSplit org.apache.hudi.hadoop.BootstrapBaseFileSplit used on opensource projects

Example 1 with BootstrapBaseFileSplit

use of org.apache.hudi.hadoop.BootstrapBaseFileSplit in project presto by prestodb.

the class HudiBootstrapBaseFileSplitConverter method extractCustomSplitInfo.

@Override
public Optional<Map<String, String>> extractCustomSplitInfo(FileSplit split) {
    if (split instanceof BootstrapBaseFileSplit) {
        ImmutableMap.Builder<String, String> customSplitInfo = new ImmutableMap.Builder<>();
        BootstrapBaseFileSplit hudiSplit = (BootstrapBaseFileSplit) split;
        customSplitInfo.put(CUSTOM_FILE_SPLIT_CLASS_KEY, BootstrapBaseFileSplit.class.getName());
        customSplitInfo.put(BOOTSTRAP_FILE_SPLIT_PATH_KEY, hudiSplit.getBootstrapFileSplit().getPath().toString());
        customSplitInfo.put(BOOTSTRAP_FILE_SPLIT_START_KEY, String.valueOf(hudiSplit.getBootstrapFileSplit().getStart()));
        customSplitInfo.put(BOOTSTRAP_FILE_SPLIT_LEN_KEY, String.valueOf(hudiSplit.getBootstrapFileSplit().getLength()));
        return Optional.of(customSplitInfo.build());
    }
    return Optional.empty();
}

Example 2 with BootstrapBaseFileSplit

use of org.apache.hudi.hadoop.BootstrapBaseFileSplit in project presto by prestodb.

the class TestCustomSplitConversionUtils method testHudiBootstrapBaseFileSplitConverter.

@Test
public void testHudiBootstrapBaseFileSplitConverter() throws IOException {
    Path bootstrapSourceFilePath = new Path("/test/source/test.parquet");
    long bootstrapSourceSplitStartPos = 0L;
    long bootstrapSourceSplitLength = 200L;
    FileSplit baseSplit = new FileSplit(FILE_PATH, SPLIT_START_POS, SPLIT_LENGTH, SPLIT_HOSTS);
    FileSplit bootstrapSourceSplit = new FileSplit(bootstrapSourceFilePath, bootstrapSourceSplitStartPos, bootstrapSourceSplitLength, new String[0]);
    FileSplit hudiSplit = new BootstrapBaseFileSplit(baseSplit, bootstrapSourceSplit);
    // Test conversion of HudiSplit -> customSplitInfo
    Map<String, String> customSplitInfo = CustomSplitConversionUtils.extractCustomSplitInfo(hudiSplit);
    // Test conversion of (customSplitInfo + baseSplit) -> HudiSplit
    BootstrapBaseFileSplit recreatedSplit = (BootstrapBaseFileSplit) CustomSplitConversionUtils.recreateSplitWithCustomInfo(baseSplit, customSplitInfo);
    assertEquals(FILE_PATH, recreatedSplit.getPath());
    assertEquals(SPLIT_START_POS, recreatedSplit.getStart());
    assertEquals(SPLIT_LENGTH, recreatedSplit.getLength());
    assertEquals(SPLIT_HOSTS, recreatedSplit.getLocations());
    assertEquals(bootstrapSourceFilePath, recreatedSplit.getBootstrapFileSplit().getPath());
    assertEquals(bootstrapSourceSplitStartPos, recreatedSplit.getBootstrapFileSplit().getStart());
    assertEquals(bootstrapSourceSplitLength, recreatedSplit.getBootstrapFileSplit().getLength());
}

Example 3 with BootstrapBaseFileSplit

use of org.apache.hudi.hadoop.BootstrapBaseFileSplit in project presto by prestodb.

the class HudiBootstrapBaseFileSplitConverter method recreateFileSplitWithCustomInfo.

@Override
public Optional<FileSplit> recreateFileSplitWithCustomInfo(FileSplit split, Map<String, String> customSplitInfo) throws IOException {
    requireNonNull(customSplitInfo);
    String customFileSplitClass = customSplitInfo.get(CUSTOM_FILE_SPLIT_CLASS_KEY);
    if (!isNullOrEmpty(customFileSplitClass) && BootstrapBaseFileSplit.class.getName().equals(customFileSplitClass)) {
        FileSplit bootstrapFileSplit = new FileSplit(new Path(customSplitInfo.get(BOOTSTRAP_FILE_SPLIT_PATH_KEY)), parseLong(customSplitInfo.get(BOOTSTRAP_FILE_SPLIT_START_KEY)), parseLong(customSplitInfo.get(BOOTSTRAP_FILE_SPLIT_LEN_KEY)), (String[]) null);
        split = new BootstrapBaseFileSplit(split, bootstrapFileSplit);
        return Optional.of(split);
    }
    return Optional.empty();
}