Examples with HoodieIOException org.apache.hudi.exception.HoodieIOException used on opensource projects

Example 66 with HoodieIOException

use of org.apache.hudi.exception.HoodieIOException in project hudi by apache.

the class BufferedConnectWriter method flushRecords.

@Override
public List<WriteStatus> flushRecords() {
    try {
        LOG.info("Number of entries in MemoryBasedMap => " + bufferedRecords.getInMemoryMapNumEntries() + "Total size in bytes of MemoryBasedMap => " + bufferedRecords.getCurrentInMemoryMapSize() + "Number of entries in BitCaskDiskMap => " + bufferedRecords.getDiskBasedMapNumEntries() + "Size of file spilled to disk => " + bufferedRecords.getSizeOfFileOnDiskInBytes());
        List<WriteStatus> writeStatuses = new ArrayList<>();
        boolean isMorTable = Option.ofNullable(connectConfigs.getString(HoodieTableConfig.TYPE)).map(t -> t.equals(HoodieTableType.MERGE_ON_READ.name())).orElse(false);
        // Write out all records if non-empty
        if (!bufferedRecords.isEmpty()) {
            if (isMorTable) {
                writeStatuses = writeClient.upsertPreppedRecords(new LinkedList<>(bufferedRecords.values()), instantTime);
            } else {
                writeStatuses = writeClient.bulkInsertPreppedRecords(new LinkedList<>(bufferedRecords.values()), instantTime, Option.empty());
            }
        }
        bufferedRecords.close();
        LOG.info("Flushed hudi records and got writeStatuses: " + writeStatuses);
        return writeStatuses;
    } catch (Exception e) {
        throw new HoodieIOException("Write records failed", new IOException(e));
    }
}

Example 67 with HoodieIOException

use of org.apache.hudi.exception.HoodieIOException in project hudi by apache.

the class BulkInsertDataInternalWriterHelper method getKeyGenerator.

/**
 * Instantiate {@link BuiltinKeyGenerator}.
 *
 * @param properties properties map.
 * @return the key generator thus instantiated.
 */
private Option<BuiltinKeyGenerator> getKeyGenerator(Properties properties) {
    TypedProperties typedProperties = new TypedProperties();
    typedProperties.putAll(properties);
    if (properties.get(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key()).equals(NonpartitionedKeyGenerator.class.getName())) {
        // Do not instantiate NonPartitionKeyGen
        return Option.empty();
    } else {
        try {
            return Option.of((BuiltinKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(typedProperties));
        } catch (ClassCastException cce) {
            throw new HoodieIOException("Only those key generators implementing BuiltInKeyGenerator interface is supported with virtual keys");
        } catch (IOException e) {
            throw new HoodieIOException("Key generator instantiation failed ", e);
        }
    }
}

Example 68 with HoodieIOException

use of org.apache.hudi.exception.HoodieIOException in project hudi by apache.

the class SparkFullBootstrapDataProviderBase method generateInputRecords.

@Override
public JavaRDD<HoodieRecord> generateInputRecords(String tableName, String sourceBasePath, List<Pair<String, List<HoodieFileStatus>>> partitionPathsWithFiles) {
    String[] filePaths = partitionPathsWithFiles.stream().map(Pair::getValue).flatMap(f -> f.stream().map(fs -> FileStatusUtils.toPath(fs.getPath()).toString())).toArray(String[]::new);
    Dataset inputDataset = sparkSession.read().format(getFormat()).load(filePaths);
    try {
        KeyGenerator keyGenerator = HoodieSparkKeyGeneratorFactory.createKeyGenerator(props);
        String structName = tableName + "_record";
        String namespace = "hoodie." + tableName;
        RDD<GenericRecord> genericRecords = HoodieSparkUtils.createRdd(inputDataset, structName, namespace, false, Option.empty());
        return genericRecords.toJavaRDD().map(gr -> {
            String orderingVal = HoodieAvroUtils.getNestedFieldValAsString(gr, props.getString("hoodie.datasource.write.precombine.field"), false, props.getBoolean(KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.key(), Boolean.parseBoolean(KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.defaultValue())));
            try {
                return DataSourceUtils.createHoodieRecord(gr, orderingVal, keyGenerator.getKey(gr), props.getString("hoodie.datasource.write.payload.class"));
            } catch (IOException ioe) {
                throw new HoodieIOException(ioe.getMessage(), ioe);
            }
        });
    } catch (IOException ioe) {
        throw new HoodieIOException(ioe.getMessage(), ioe);
    }
}

Example 69 with HoodieIOException

use of org.apache.hudi.exception.HoodieIOException in project hudi by apache.

the class RestorePlanActionExecutor method execute.

@Override
public Option<HoodieRestorePlan> execute() {
    final HoodieInstant restoreInstant = new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.RESTORE_ACTION, instantTime);
    try {
        // Get all the commits on the timeline after the provided commit time
        // rollback pending clustering instants first before other instants (See HUDI-3362)
        List<HoodieInstant> pendingClusteringInstantsToRollback = table.getActiveTimeline().filterPendingReplaceTimeline().filter(instant -> ClusteringUtils.isPendingClusteringInstant(table.getMetaClient(), instant)).getReverseOrderedInstants().filter(instant -> HoodieActiveTimeline.GREATER_THAN.test(instant.getTimestamp(), restoreInstantTime)).collect(Collectors.toList());
        // Get all the commits on the timeline after the provided commit time
        List<HoodieInstant> commitInstantsToRollback = table.getActiveTimeline().getWriteTimeline().getReverseOrderedInstants().filter(instant -> HoodieActiveTimeline.GREATER_THAN.test(instant.getTimestamp(), restoreInstantTime)).filter(instant -> !pendingClusteringInstantsToRollback.contains(instant)).collect(Collectors.toList());
        // Combine both lists - first rollback pending clustering and then rollback all other commits
        List<HoodieInstantInfo> instantsToRollback = Stream.concat(pendingClusteringInstantsToRollback.stream(), commitInstantsToRollback.stream()).map(entry -> new HoodieInstantInfo(entry.getTimestamp(), entry.getAction())).collect(Collectors.toList());
        HoodieRestorePlan restorePlan = new HoodieRestorePlan(instantsToRollback, LATEST_RESTORE_PLAN_VERSION);
        table.getActiveTimeline().saveToRestoreRequested(restoreInstant, TimelineMetadataUtils.serializeRestorePlan(restorePlan));
        table.getMetaClient().reloadActiveTimeline();
        LOG.info("Requesting Restore with instant time " + restoreInstant);
        return Option.of(restorePlan);
    } catch (IOException e) {
        LOG.error("Got exception when saving restore requested file", e);
        throw new HoodieIOException(e.getMessage(), e);
    }
}

Example 70 with HoodieIOException

use of org.apache.hudi.exception.HoodieIOException in project hudi by apache.

the class BaseRollbackActionExecutor method doRollbackAndGetStats.

public List<HoodieRollbackStat> doRollbackAndGetStats(HoodieRollbackPlan hoodieRollbackPlan) {
    final String instantTimeToRollback = instantToRollback.getTimestamp();
    final boolean isPendingCompaction = Objects.equals(HoodieTimeline.COMPACTION_ACTION, instantToRollback.getAction()) && !instantToRollback.isCompleted();
    final boolean isPendingClustering = Objects.equals(HoodieTimeline.REPLACE_COMMIT_ACTION, instantToRollback.getAction()) && !instantToRollback.isCompleted() && ClusteringUtils.getClusteringPlan(table.getMetaClient(), instantToRollback).isPresent();
    validateSavepointRollbacks();
    if (!isPendingCompaction && !isPendingClustering) {
        validateRollbackCommitSequence();
    }
    try {
        List<HoodieRollbackStat> stats = executeRollback(hoodieRollbackPlan);
        LOG.info("Rolled back inflight instant " + instantTimeToRollback);
        if (!isPendingCompaction) {
            rollBackIndex();
        }
        return stats;
    } catch (IOException e) {
        throw new HoodieIOException("Unable to execute rollback ", e);
    }
}