Examples with HoodieIOException org.apache.hudi.exception.HoodieIOException used on opensource projects

Example 16 with HoodieIOException

use of org.apache.hudi.exception.HoodieIOException in project hudi by apache.

the class HoodieActiveTimeline method createRequestedReplaceCommit.

public void createRequestedReplaceCommit(String instantTime, String actionType) {
    try {
        HoodieInstant instant = new HoodieInstant(State.REQUESTED, actionType, instantTime);
        LOG.info("Creating a new instant " + instant);
        // Create the request replace file
        createFileInMetaPath(instant.getFileName(), TimelineMetadataUtils.serializeRequestedReplaceMetadata(new HoodieRequestedReplaceMetadata()), false);
    } catch (IOException e) {
        throw new HoodieIOException("Error create requested replace commit ", e);
    }
}

Example 17 with HoodieIOException

use of org.apache.hudi.exception.HoodieIOException in project hudi by apache.

the class HoodieActiveTimeline method transitionState.

private void transitionState(HoodieInstant fromInstant, HoodieInstant toInstant, Option<byte[]> data, boolean allowRedundantTransitions) {
    ValidationUtils.checkArgument(fromInstant.getTimestamp().equals(toInstant.getTimestamp()));
    try {
        if (metaClient.getTimelineLayoutVersion().isNullVersion()) {
            // Re-create the .inflight file by opening a new file and write the commit metadata in
            createFileInMetaPath(fromInstant.getFileName(), data, allowRedundantTransitions);
            Path fromInstantPath = new Path(metaClient.getMetaPath(), fromInstant.getFileName());
            Path toInstantPath = new Path(metaClient.getMetaPath(), toInstant.getFileName());
            boolean success = metaClient.getFs().rename(fromInstantPath, toInstantPath);
            if (!success) {
                throw new HoodieIOException("Could not rename " + fromInstantPath + " to " + toInstantPath);
            }
        } else {
            // Ensures old state exists in timeline
            LOG.info("Checking for file exists ?" + new Path(metaClient.getMetaPath(), fromInstant.getFileName()));
            ValidationUtils.checkArgument(metaClient.getFs().exists(new Path(metaClient.getMetaPath(), fromInstant.getFileName())));
            // Use Write Once to create Target File
            if (allowRedundantTransitions) {
                FileIOUtils.createFileInPath(metaClient.getFs(), new Path(metaClient.getMetaPath(), toInstant.getFileName()), data);
            } else {
                createImmutableFileInPath(new Path(metaClient.getMetaPath(), toInstant.getFileName()), data);
            }
            LOG.info("Create new file for toInstant ?" + new Path(metaClient.getMetaPath(), toInstant.getFileName()));
        }
    } catch (IOException e) {
        throw new HoodieIOException("Could not complete " + fromInstant, e);
    }
}

Example 18 with HoodieIOException

use of org.apache.hudi.exception.HoodieIOException in project hudi by apache.

the class HoodieArchivedTimeline method loadInstants.

/**
 * This is method to read selected instants. Do NOT use this directly use one of the helper methods above
 * If loadInstantDetails is set to true, this would also update 'readCommits' map with commit details
 * If filter is specified, only the filtered instants are loaded
 * If commitsFilter is specified, only the filtered records are loaded
 */
private List<HoodieInstant> loadInstants(TimeRangeFilter filter, boolean loadInstantDetails, Function<GenericRecord, Boolean> commitsFilter) {
    try {
        // List all files
        FileStatus[] fsStatuses = metaClient.getFs().globStatus(new Path(metaClient.getArchivePath() + "/.commits_.archive*"));
        // Sort files by version suffix in reverse (implies reverse chronological order)
        Arrays.sort(fsStatuses, new ArchiveFileVersionComparator());
        Set<HoodieInstant> instantsInRange = new HashSet<>();
        for (FileStatus fs : fsStatuses) {
            // Read the archived file
            try (HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(metaClient.getFs(), new HoodieLogFile(fs.getPath()), HoodieArchivedMetaEntry.getClassSchema())) {
                int instantsInPreviousFile = instantsInRange.size();
                // Read the avro blocks
                while (reader.hasNext()) {
                    HoodieAvroDataBlock blk = (HoodieAvroDataBlock) reader.next();
                    // (such as startTime, endTime of records in the block)
                    try (ClosableIterator<IndexedRecord> itr = blk.getRecordItr()) {
                        StreamSupport.stream(Spliterators.spliteratorUnknownSize(itr, Spliterator.IMMUTABLE), true).filter(r -> commitsFilter.apply((GenericRecord) r)).map(r -> readCommit((GenericRecord) r, loadInstantDetails)).filter(c -> filter == null || filter.isInRange(c)).forEach(instantsInRange::add);
                    }
                }
                if (filter != null) {
                    int instantsInCurrentFile = instantsInRange.size() - instantsInPreviousFile;
                    if (instantsInPreviousFile > 0 && instantsInCurrentFile == 0) {
                        // This signals we crossed lower bound of desired time window.
                        break;
                    }
                }
            } catch (Exception originalException) {
                // need to ignore this kind of exception here.
                try {
                    Path planPath = new Path(metaClient.getArchivePath(), MERGE_ARCHIVE_PLAN_NAME);
                    HoodieWrapperFileSystem fileSystem = metaClient.getFs();
                    if (fileSystem.exists(planPath)) {
                        HoodieMergeArchiveFilePlan plan = TimelineMetadataUtils.deserializeAvroMetadata(FileIOUtils.readDataFromPath(fileSystem, planPath).get(), HoodieMergeArchiveFilePlan.class);
                        String mergedArchiveFileName = plan.getMergedArchiveFileName();
                        if (!StringUtils.isNullOrEmpty(mergedArchiveFileName) && fs.getPath().getName().equalsIgnoreCase(mergedArchiveFileName)) {
                            LOG.warn("Catch exception because of reading uncompleted merging archive file " + mergedArchiveFileName + ". Ignore it here.");
                            continue;
                        }
                    }
                    throw originalException;
                } catch (Exception e) {
                    // For example corrupted archive file and corrupted plan are both existed.
                    throw originalException;
                }
            }
        }
        ArrayList<HoodieInstant> result = new ArrayList<>(instantsInRange);
        Collections.sort(result);
        return result;
    } catch (IOException e) {
        throw new HoodieIOException("Could not load archived commit timeline from path " + metaClient.getArchivePath(), e);
    }
}

Example 19 with HoodieIOException

use of org.apache.hudi.exception.HoodieIOException in project hudi by apache.

the class MarkerUtils method readTimelineServerBasedMarkersFromFileSystem.

/**
 * Reads files containing the markers written by timeline-server-based marker mechanism.
 *
 * @param markerDir   marker directory.
 * @param fileSystem  file system to use.
 * @param context     instance of {@link HoodieEngineContext} to use
 * @param parallelism parallelism to use
 * @return A {@code Map} of file name to the set of markers stored in the file.
 */
public static Map<String, Set<String>> readTimelineServerBasedMarkersFromFileSystem(String markerDir, FileSystem fileSystem, HoodieEngineContext context, int parallelism) {
    Path dirPath = new Path(markerDir);
    try {
        if (fileSystem.exists(dirPath)) {
            Predicate<FileStatus> prefixFilter = fileStatus -> fileStatus.getPath().getName().startsWith(MARKERS_FILENAME_PREFIX);
            Predicate<FileStatus> markerTypeFilter = fileStatus -> !fileStatus.getPath().getName().equals(MARKER_TYPE_FILENAME);
            return FSUtils.parallelizeSubPathProcess(context, fileSystem, dirPath, parallelism, prefixFilter.and(markerTypeFilter), pairOfSubPathAndConf -> {
                String markersFilePathStr = pairOfSubPathAndConf.getKey();
                SerializableConfiguration conf = pairOfSubPathAndConf.getValue();
                return readMarkersFromFile(new Path(markersFilePathStr), conf);
            });
        }
        return new HashMap<>();
    } catch (IOException ioe) {
        throw new HoodieIOException(ioe.getMessage(), ioe);
    }
}

Example 20 with HoodieIOException

use of org.apache.hudi.exception.HoodieIOException in project hudi by apache.

the class MarkerUtils method readMarkerType.

/**
 * Reads the marker type from `MARKERS.type` file.
 *
 * @param fileSystem file system to use.
 * @param markerDir  marker directory.
 * @return the marker type, or empty if the marker type file does not exist.
 */
public static Option<MarkerType> readMarkerType(FileSystem fileSystem, String markerDir) {
    Path markerTypeFilePath = new Path(markerDir, MARKER_TYPE_FILENAME);
    FSDataInputStream fsDataInputStream = null;
    Option<MarkerType> content = Option.empty();
    try {
        if (!doesMarkerTypeFileExist(fileSystem, markerDir)) {
            return Option.empty();
        }
        fsDataInputStream = fileSystem.open(markerTypeFilePath);
        content = Option.of(MarkerType.valueOf(FileIOUtils.readAsUTFString(fsDataInputStream)));
    } catch (IOException e) {
        throw new HoodieIOException("Cannot read marker type file " + markerTypeFilePath.toString() + "; " + e.getMessage(), e);
    } finally {
        closeQuietly(fsDataInputStream);
    }
    return content;
}