Examples with DirectWriteMarkers org.apache.hudi.table.marker.DirectWriteMarkers used on opensource projects

Example 1 with DirectWriteMarkers

use of org.apache.hudi.table.marker.DirectWriteMarkers in project hudi by apache.

the class TwoToOneDowngradeHandler method convertToDirectMarkers.

/**
 * Converts the markers in new format(timeline server based) to old format of direct markers,
 * i.e., one marker file per data file, without MARKERS.type file.
 * This needs to be idempotent.
 * 1. read all markers from timeline server based marker files
 * 2. create direct style markers
 * 3. delete marker type file
 * 4. delete timeline server based marker files
 *
 * @param commitInstantTime instant of interest for marker conversion.
 * @param table             instance of {@link HoodieTable} to use
 * @param context           instance of {@link HoodieEngineContext} to use
 * @param parallelism       parallelism to use
 */
private void convertToDirectMarkers(final String commitInstantTime, HoodieTable table, HoodieEngineContext context, int parallelism) throws IOException {
    String markerDir = table.getMetaClient().getMarkerFolderPath(commitInstantTime);
    FileSystem fileSystem = FSUtils.getFs(markerDir, context.getHadoopConf().newCopy());
    Option<MarkerType> markerTypeOption = MarkerUtils.readMarkerType(fileSystem, markerDir);
    if (markerTypeOption.isPresent()) {
        switch(markerTypeOption.get()) {
            case TIMELINE_SERVER_BASED:
                // Reads all markers written by the timeline server
                Map<String, Set<String>> markersMap = MarkerUtils.readTimelineServerBasedMarkersFromFileSystem(markerDir, fileSystem, context, parallelism);
                DirectWriteMarkers directWriteMarkers = new DirectWriteMarkers(table, commitInstantTime);
                // Recreates the markers in the direct format
                markersMap.values().stream().flatMap(Collection::stream).forEach(directWriteMarkers::create);
                // Deletes marker type file
                MarkerUtils.deleteMarkerTypeFile(fileSystem, markerDir);
                // Deletes timeline server based markers
                deleteTimelineBasedMarkerFiles(context, markerDir, fileSystem, parallelism);
                break;
            default:
                throw new HoodieException("The marker type \"" + markerTypeOption.get().name() + "\" is not supported for rollback.");
        }
    } else {
        if (fileSystem.exists(new Path(markerDir))) {
            // In case of partial failures during downgrade, there is a chance that marker type file was deleted,
            // but timeline server based marker files are left.  So deletes them if any
            deleteTimelineBasedMarkerFiles(context, markerDir, fileSystem, parallelism);
        }
    }
}