Examples with HoodieIOException org.apache.hudi.exception.HoodieIOException used on opensource projects

Example 41 with HoodieIOException

use of org.apache.hudi.exception.HoodieIOException in project hudi by apache.

the class FSUtils method getFs.

public static FileSystem getFs(String path, Configuration conf) {
    FileSystem fs;
    prepareHadoopConf(conf);
    try {
        fs = new Path(path).getFileSystem(conf);
    } catch (IOException e) {
        throw new HoodieIOException("Failed to get instance of " + FileSystem.class.getName(), e);
    }
    return fs;
}

Example 42 with HoodieIOException

use of org.apache.hudi.exception.HoodieIOException in project hudi by apache.

the class FSUtils method getFileStatusAtLevel.

/**
 * Lists file status at a certain level in the directory hierarchy.
 * <p>
 * E.g., given "/tmp/hoodie_table" as the rootPath, and 3 as the expected level,
 * this method gives back the {@link FileStatus} of all files under
 * "/tmp/hoodie_table/[*]/[*]/[*]/" folders.
 *
 * @param hoodieEngineContext {@link HoodieEngineContext} instance.
 * @param fs                  {@link FileSystem} instance.
 * @param rootPath            Root path for the file listing.
 * @param expectLevel         Expected level of directory hierarchy for files to be added.
 * @param parallelism         Parallelism for the file listing.
 * @return A list of file status of files at the level.
 */
public static List<FileStatus> getFileStatusAtLevel(HoodieEngineContext hoodieEngineContext, FileSystem fs, Path rootPath, int expectLevel, int parallelism) {
    List<String> levelPaths = new ArrayList<>();
    List<FileStatus> result = new ArrayList<>();
    levelPaths.add(rootPath.toString());
    for (int i = 0; i <= expectLevel; i++) {
        result = FSUtils.parallelizeFilesProcess(hoodieEngineContext, fs, parallelism, pairOfSubPathAndConf -> {
            Path path = new Path(pairOfSubPathAndConf.getKey());
            try {
                FileSystem fileSystem = path.getFileSystem(pairOfSubPathAndConf.getValue().get());
                return Arrays.stream(fileSystem.listStatus(path)).collect(Collectors.toList());
            } catch (IOException e) {
                throw new HoodieIOException("Failed to list " + path, e);
            }
        }, levelPaths).values().stream().flatMap(list -> list.stream()).collect(Collectors.toList());
        if (i < expectLevel) {
            levelPaths = result.stream().filter(FileStatus::isDirectory).map(fileStatus -> fileStatus.getPath().toString()).collect(Collectors.toList());
        }
    }
    return result;
}

Example 43 with HoodieIOException

use of org.apache.hudi.exception.HoodieIOException in project hudi by apache.

the class HoodieMetadataMetrics method getStats.

public Map<String, String> getStats(boolean detailed, HoodieTableMetaClient metaClient, HoodieTableMetadata metadata) {
    try {
        metaClient.reloadActiveTimeline();
        HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metaClient, metaClient.getActiveTimeline());
        return getStats(fsView, detailed, metadata);
    } catch (IOException ioe) {
        throw new HoodieIOException("Unable to get metadata stats.", ioe);
    }
}

Example 44 with HoodieIOException

use of org.apache.hudi.exception.HoodieIOException in project hudi by apache.

the class HoodieHFileReader method close.

@Override
public synchronized void close() {
    try {
        reader.close();
        reader = null;
        if (fsDataInputStream != null) {
            fsDataInputStream.close();
        }
        keyScanner = null;
    } catch (IOException e) {
        throw new HoodieIOException("Error closing the hfile reader", e);
    }
}

Example 45 with HoodieIOException

use of org.apache.hudi.exception.HoodieIOException in project hudi by apache.

the class HoodieHFileReader method getRecordIterator.

@Override
public Iterator getRecordIterator(Schema readerSchema) throws IOException {
    final HFileScanner scanner = reader.getScanner(false, false);
    final Option<Schema.Field> keyFieldSchema = Option.ofNullable(readerSchema.getField(KEY_FIELD_NAME));
    ValidationUtils.checkState(keyFieldSchema != null, "Missing key field '" + KEY_FIELD_NAME + "' in the schema!");
    return new Iterator<R>() {

        private R next = null;

        private boolean eof = false;

        @Override
        public boolean hasNext() {
            try {
                // To handle when hasNext() is called multiple times for idempotency and/or the first time
                if (this.next == null && !this.eof) {
                    if (!scanner.isSeeked() && scanner.seekTo()) {
                        final Pair<String, R> keyAndRecordPair = getRecordFromCell(scanner.getKeyValue(), getSchema(), readerSchema, keyFieldSchema);
                        this.next = keyAndRecordPair.getSecond();
                    }
                }
                return this.next != null;
            } catch (IOException io) {
                throw new HoodieIOException("unable to read next record from hfile ", io);
            }
        }

        @Override
        public R next() {
            try {
                // To handle case when next() is called before hasNext()
                if (this.next == null) {
                    if (!hasNext()) {
                        throw new HoodieIOException("No more records left to read from hfile");
                    }
                }
                R retVal = this.next;
                if (scanner.next()) {
                    final Pair<String, R> keyAndRecordPair = getRecordFromCell(scanner.getKeyValue(), getSchema(), readerSchema, keyFieldSchema);
                    this.next = keyAndRecordPair.getSecond();
                } else {
                    this.next = null;
                    this.eof = true;
                }
                return retVal;
            } catch (IOException io) {
                throw new HoodieIOException("unable to read next record from parquet file ", io);
            }
        }
    };
}