Examples with FileSlice org.apache.hudi.common.model.FileSlice used on opensource projects

Example 81 with FileSlice

use of org.apache.hudi.common.model.FileSlice in project hudi by apache.

the class TestHoodieTableFileSystemView method testStreamLatestVersionInPartition.

private void testStreamLatestVersionInPartition(boolean isLatestFileSliceOnly, String fullPartitionPath, String commitTime1, String commitTime2, String commitTime3, String commitTime4, String fileId1, String fileId2, String fileId3, String fileId4) throws IOException {
    // Now we list the entire partition
    FileStatus[] statuses = metaClient.getFs().listStatus(new Path(fullPartitionPath));
    assertEquals(11, statuses.length);
    refreshFsView();
    // Check files as of lastest commit.
    List<FileSlice> allSlices = rtView.getAllFileSlices("2016/05/01").collect(Collectors.toList());
    assertEquals(isLatestFileSliceOnly ? 4 : 8, allSlices.size());
    Map<String, Long> fileSliceMap = allSlices.stream().collect(Collectors.groupingBy(FileSlice::getFileId, Collectors.counting()));
    assertEquals(isLatestFileSliceOnly ? 1 : 2, fileSliceMap.get(fileId1).longValue());
    assertEquals(isLatestFileSliceOnly ? 1 : 3, fileSliceMap.get(fileId2).longValue());
    assertEquals(isLatestFileSliceOnly ? 1 : 2, fileSliceMap.get(fileId3).longValue());
    assertEquals(1, fileSliceMap.get(fileId4).longValue());
    List<HoodieBaseFile> dataFileList = roView.getLatestBaseFilesBeforeOrOn("2016/05/01", commitTime4).collect(Collectors.toList());
    assertEquals(3, dataFileList.size());
    Set<String> filenames = new HashSet<>();
    for (HoodieBaseFile status : dataFileList) {
        filenames.add(status.getFileName());
    }
    assertTrue(filenames.contains(FSUtils.makeDataFileName(commitTime4, TEST_WRITE_TOKEN, fileId1)));
    assertTrue(filenames.contains(FSUtils.makeDataFileName(commitTime3, TEST_WRITE_TOKEN, fileId2)));
    assertTrue(filenames.contains(FSUtils.makeDataFileName(commitTime4, TEST_WRITE_TOKEN, fileId3)));
    filenames = new HashSet<>();
    List<HoodieLogFile> logFilesList = rtView.getLatestFileSlicesBeforeOrOn("2016/05/01", commitTime4, true).map(FileSlice::getLogFiles).flatMap(logFileList -> logFileList).collect(Collectors.toList());
    assertEquals(logFilesList.size(), 4);
    for (HoodieLogFile logFile : logFilesList) {
        filenames.add(logFile.getFileName());
    }
    assertTrue(filenames.contains(FSUtils.makeLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, commitTime4, 0, TEST_WRITE_TOKEN)));
    assertTrue(filenames.contains(FSUtils.makeLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, commitTime4, 1, TEST_WRITE_TOKEN)));
    assertTrue(filenames.contains(FSUtils.makeLogFileName(fileId2, HoodieLogFile.DELTA_EXTENSION, commitTime3, 0, TEST_WRITE_TOKEN)));
    assertTrue(filenames.contains(FSUtils.makeLogFileName(fileId4, HoodieLogFile.DELTA_EXTENSION, commitTime4, 0, TEST_WRITE_TOKEN)));
    // Reset the max commit time
    List<HoodieBaseFile> dataFiles = roView.getLatestBaseFilesBeforeOrOn("2016/05/01", commitTime3).collect(Collectors.toList());
    filenames = new HashSet<>();
    for (HoodieBaseFile status : dataFiles) {
        filenames.add(status.getFileName());
    }
    if (!isLatestFileSliceOnly) {
        assertEquals(3, dataFiles.size());
        assertTrue(filenames.contains(FSUtils.makeDataFileName(commitTime1, TEST_WRITE_TOKEN, fileId1)));
        assertTrue(filenames.contains(FSUtils.makeDataFileName(commitTime3, TEST_WRITE_TOKEN, fileId2)));
        assertTrue(filenames.contains(FSUtils.makeDataFileName(commitTime3, TEST_WRITE_TOKEN, fileId3)));
    } else {
        assertEquals(1, dataFiles.size());
        assertTrue(filenames.contains(FSUtils.makeDataFileName(commitTime3, TEST_WRITE_TOKEN, fileId2)));
    }
    logFilesList = rtView.getLatestFileSlicesBeforeOrOn("2016/05/01", commitTime3, true).map(FileSlice::getLogFiles).flatMap(logFileList -> logFileList).collect(Collectors.toList());
    assertEquals(logFilesList.size(), 1);
    assertEquals(logFilesList.get(0).getFileName(), FSUtils.makeLogFileName(fileId2, HoodieLogFile.DELTA_EXTENSION, commitTime3, 0, TEST_WRITE_TOKEN));
}

Example 82 with FileSlice

use of org.apache.hudi.common.model.FileSlice in project hudi by apache.

the class TestIncrementalFSViewSync method areViewsConsistent.

/**
 * Check for equality of views.
 *
 * @param view1 View1
 * @param view2 View2
 */
private void areViewsConsistent(SyncableFileSystemView view1, SyncableFileSystemView view2, long expectedTotalFileSlices) {
    // Timeline check
    assertEquals(view1.getLastInstant(), view2.getLastInstant());
    // View Checks
    Map<HoodieFileGroupId, HoodieFileGroup> fileGroupsMap1 = partitions.stream().flatMap(view1::getAllFileGroups).collect(Collectors.toMap(HoodieFileGroup::getFileGroupId, fg -> fg));
    Map<HoodieFileGroupId, HoodieFileGroup> fileGroupsMap2 = partitions.stream().flatMap(view2::getAllFileGroups).collect(Collectors.toMap(HoodieFileGroup::getFileGroupId, fg -> fg));
    assertEquals(fileGroupsMap1.keySet(), fileGroupsMap2.keySet());
    long gotSlicesCount = fileGroupsMap1.keySet().stream().map(k -> Pair.of(fileGroupsMap1.get(k), fileGroupsMap2.get(k))).mapToLong(e -> {
        HoodieFileGroup fg1 = e.getKey();
        HoodieFileGroup fg2 = e.getValue();
        assertEquals(fg1.getFileGroupId(), fg2.getFileGroupId());
        List<FileSlice> slices1 = fg1.getAllRawFileSlices().collect(Collectors.toList());
        List<FileSlice> slices2 = fg2.getAllRawFileSlices().collect(Collectors.toList());
        assertEquals(slices1.size(), slices2.size());
        IntStream.range(0, slices1.size()).mapToObj(idx -> Pair.of(slices1.get(idx), slices2.get(idx))).forEach(e2 -> {
            FileSlice slice1 = e2.getKey();
            FileSlice slice2 = e2.getValue();
            assertEquals(slice1.getBaseInstantTime(), slice2.getBaseInstantTime());
            assertEquals(slice1.getFileId(), slice2.getFileId());
            assertEquals(slice1.getBaseFile().isPresent(), slice2.getBaseFile().isPresent());
            if (slice1.getBaseFile().isPresent()) {
                HoodieBaseFile df1 = slice1.getBaseFile().get();
                HoodieBaseFile df2 = slice2.getBaseFile().get();
                assertEquals(df1.getCommitTime(), df2.getCommitTime());
                assertEquals(df1.getFileId(), df2.getFileId());
                assertEquals(df1.getFileName(), df2.getFileName());
                assertEquals(Path.getPathWithoutSchemeAndAuthority(new Path(df1.getPath())), Path.getPathWithoutSchemeAndAuthority(new Path(df2.getPath())));
            }
            List<Path> logPaths1 = slice1.getLogFiles().map(lf -> Path.getPathWithoutSchemeAndAuthority(lf.getPath())).collect(Collectors.toList());
            List<Path> logPaths2 = slice2.getLogFiles().map(lf -> Path.getPathWithoutSchemeAndAuthority(lf.getPath())).collect(Collectors.toList());
            assertEquals(logPaths1, logPaths2);
        });
        return slices1.size();
    }).sum();
    assertEquals(expectedTotalFileSlices, gotSlicesCount);
    // Pending Compaction Operations Check
    Set<Pair<String, CompactionOperation>> ops1 = view1.getPendingCompactionOperations().collect(Collectors.toSet());
    Set<Pair<String, CompactionOperation>> ops2 = view2.getPendingCompactionOperations().collect(Collectors.toSet());
    assertEquals(ops1, ops2);
}

Example 83 with FileSlice

use of org.apache.hudi.common.model.FileSlice in project hudi by apache.

the class TestIncrementalFSViewSync method scheduleCompaction.

/**
 * Schedule a pending compaction and validate.
 *
 * @param view Hoodie View
 * @param instantTime COmpaction Instant Time
 */
private void scheduleCompaction(SyncableFileSystemView view, String instantTime) throws IOException {
    List<Pair<String, FileSlice>> slices = partitions.stream().flatMap(p -> view.getLatestFileSlices(p).map(s -> Pair.of(p, s))).collect(Collectors.toList());
    long initialExpTotalFileSlices = partitions.stream().mapToLong(p -> view.getAllFileSlices(p).count()).sum();
    HoodieCompactionPlan plan = CompactionUtils.buildFromFileSlices(slices, Option.empty(), Option.empty());
    HoodieInstant compactionInstant = new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, instantTime);
    metaClient.getActiveTimeline().saveToCompactionRequested(compactionInstant, TimelineMetadataUtils.serializeCompactionPlan(plan));
    view.sync();
    partitions.forEach(p -> {
        view.getLatestFileSlices(p).forEach(fs -> {
            assertEquals(instantTime, fs.getBaseInstantTime());
            assertEquals(p, fs.getPartitionPath());
            assertFalse(fs.getBaseFile().isPresent());
        });
        view.getLatestMergedFileSlicesBeforeOrOn(p, instantTime).forEach(fs -> {
            assertTrue(HoodieTimeline.compareTimestamps(instantTime, HoodieTimeline.GREATER_THAN, fs.getBaseInstantTime()));
            assertEquals(p, fs.getPartitionPath());
        });
    });
    metaClient.reloadActiveTimeline();
    SyncableFileSystemView newView = getFileSystemView(metaClient);
    areViewsConsistent(view, newView, initialExpTotalFileSlices + partitions.size() * fileIdsPerPartition.size());
}

Example 84 with FileSlice

use of org.apache.hudi.common.model.FileSlice in project hudi by apache.

the class HoodieBackedTableMetadata method getPartitionFileSliceToKeysMapping.

/**
 * Get the latest file slices for the interested keys in a given partition.
 *
 * @param partitionName - Partition to get the file slices from
 * @param keys          - Interested keys
 * @return FileSlices for the keys
 */
private Map<Pair<String, FileSlice>, List<String>> getPartitionFileSliceToKeysMapping(final String partitionName, final List<String> keys) {
    // Metadata is in sync till the latest completed instant on the dataset
    List<FileSlice> latestFileSlices = HoodieTableMetadataUtil.getPartitionLatestMergedFileSlices(metadataMetaClient, partitionName);
    Map<Pair<String, FileSlice>, List<String>> partitionFileSliceToKeysMap = new HashMap<>();
    for (String key : keys) {
        final FileSlice slice = latestFileSlices.get(HoodieTableMetadataUtil.mapRecordKeyToFileGroupIndex(key, latestFileSlices.size()));
        final Pair<String, FileSlice> partitionNameFileSlicePair = Pair.of(partitionName, slice);
        partitionFileSliceToKeysMap.computeIfAbsent(partitionNameFileSlicePair, k -> new ArrayList<>()).add(key);
    }
    return partitionFileSliceToKeysMap;
}

Example 85 with FileSlice

use of org.apache.hudi.common.model.FileSlice in project hudi by apache.

the class HoodieMetadataMetrics method getStats.

private Map<String, String> getStats(HoodieTableFileSystemView fsView, boolean detailed, HoodieTableMetadata tableMetadata) throws IOException {
    Map<String, String> stats = new HashMap<>();
    // Total size of the metadata and count of base/log files
    for (String metadataPartition : MetadataPartitionType.allPaths()) {
        List<FileSlice> latestSlices = fsView.getLatestFileSlices(metadataPartition).collect(Collectors.toList());
        // Total size of the metadata and count of base/log files
        long totalBaseFileSizeInBytes = 0;
        long totalLogFileSizeInBytes = 0;
        int baseFileCount = 0;
        int logFileCount = 0;
        for (FileSlice slice : latestSlices) {
            if (slice.getBaseFile().isPresent()) {
                totalBaseFileSizeInBytes += slice.getBaseFile().get().getFileStatus().getLen();
                ++baseFileCount;
            }
            Iterator<HoodieLogFile> it = slice.getLogFiles().iterator();
            while (it.hasNext()) {
                totalLogFileSizeInBytes += it.next().getFileSize();
                ++logFileCount;
            }
        }
        stats.put(metadataPartition + "." + STAT_TOTAL_BASE_FILE_SIZE, String.valueOf(totalBaseFileSizeInBytes));
        stats.put(metadataPartition + "." + STAT_TOTAL_LOG_FILE_SIZE, String.valueOf(totalLogFileSizeInBytes));
        stats.put(metadataPartition + "." + STAT_COUNT_BASE_FILES, String.valueOf(baseFileCount));
        stats.put(metadataPartition + "." + STAT_COUNT_LOG_FILES, String.valueOf(logFileCount));
    }
    if (detailed) {
        stats.put(HoodieMetadataMetrics.STAT_COUNT_PARTITION, String.valueOf(tableMetadata.getAllPartitionPaths().size()));
    }
    return stats;
}