Examples with BootstrapIndex org.apache.hudi.common.bootstrap.index.BootstrapIndex used on opensource projects

Example 6 with BootstrapIndex

use of org.apache.hudi.common.bootstrap.index.BootstrapIndex in project hudi by apache.

the class TestBootstrapIndex method validateBootstrapIndex.

private void validateBootstrapIndex(Map<String, List<BootstrapFileMapping>> bootstrapMapping) {
    BootstrapIndex index = new HFileBootstrapIndex(metaClient);
    try (BootstrapIndex.IndexReader reader = index.createReader()) {
        List<String> indexedPartitions = reader.getIndexedPartitionPaths();
        assertEquals(bootstrapMapping.size(), indexedPartitions.size());
        indexedPartitions.forEach(partition -> assertTrue(PARTITION_SET.contains(partition)));
        long expNumFileGroupKeys = bootstrapMapping.values().stream().flatMap(Collection::stream).count();
        List<HoodieFileGroupId> fileGroupIds = reader.getIndexedFileGroupIds();
        long gotNumFileGroupKeys = fileGroupIds.size();
        assertEquals(expNumFileGroupKeys, gotNumFileGroupKeys);
        fileGroupIds.forEach(fgId -> assertTrue(PARTITION_SET.contains(fgId.getPartitionPath())));
        bootstrapMapping.entrySet().stream().forEach(e -> {
            List<BootstrapFileMapping> gotMapping = reader.getSourceFileMappingForPartition(e.getKey());
            List<BootstrapFileMapping> expected = new ArrayList<>(e.getValue());
            Collections.sort(gotMapping);
            Collections.sort(expected);
            assertEquals(expected, gotMapping, "Check for bootstrap index entries for partition " + e.getKey());
            List<HoodieFileGroupId> fileIds = e.getValue().stream().map(BootstrapFileMapping::getFileGroupId).collect(Collectors.toList());
            Map<HoodieFileGroupId, BootstrapFileMapping> lookupResult = reader.getSourceFileMappingForFileIds(fileIds);
            assertEquals(fileIds.size(), lookupResult.size());
            e.getValue().forEach(x -> {
                BootstrapFileMapping res = lookupResult.get(x.getFileGroupId());
                assertNotNull(res);
                assertEquals(x.getFileId(), res.getFileId());
                assertEquals(x.getPartitionPath(), res.getPartitionPath());
                assertEquals(BOOTSTRAP_BASE_PATH, res.getBootstrapBasePath());
                assertEquals(x.getBootstrapFileStatus(), res.getBootstrapFileStatus());
                assertEquals(x.getBootstrapPartitionPath(), res.getBootstrapPartitionPath());
            });
        });
    }
}