Examples with HoodieException org.apache.hudi.exception.HoodieException used on opensource projects

Example 26 with HoodieException

use of org.apache.hudi.exception.HoodieException in project hudi by apache.

the class StreamWriteOperatorCoordinator method start.

@Override
public void start() throws Exception {
    // setup classloader for APIs that use reflection without taking ClassLoader param
    // reference: https://stackoverflow.com/questions/1771679/difference-between-threads-context-class-loader-and-normal-classloader
    Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
    // initialize event buffer
    reset();
    this.gateways = new SubtaskGateway[this.parallelism];
    // init table, create if not exists.
    this.metaClient = initTableIfNotExists(this.conf);
    // the write client must create after the table creation
    this.writeClient = StreamerUtil.createWriteClient(conf);
    this.tableState = TableState.create(conf);
    // start the executor
    this.executor = NonThrownExecutor.builder(LOG).exceptionHook((errMsg, t) -> this.context.failJob(new HoodieException(errMsg, t))).waitForTasksFinish(true).build();
    // start the executor if required
    if (tableState.syncHive) {
        initHiveSync();
    }
    if (tableState.syncMetadata) {
        initMetadataSync();
    }
    this.ckpMetadata = CkpMetadata.getInstance(this.metaClient.getFs(), metaClient.getBasePath());
    this.ckpMetadata.bootstrap(this.metaClient);
}

Example 27 with HoodieException

use of org.apache.hudi.exception.HoodieException in project hudi by apache.

the class AppendWriteFunction method initWriterHelper.

// -------------------------------------------------------------------------
// Utilities
// -------------------------------------------------------------------------
private void initWriterHelper() {
    this.currentInstant = instantToWrite(true);
    if (this.currentInstant == null) {
        // in case there are empty checkpoints that has no input data
        throw new HoodieException("No inflight instant when flushing data!");
    }
    this.writerHelper = new BulkInsertWriterHelper(this.config, this.writeClient.getHoodieTable(), this.writeClient.getConfig(), this.currentInstant, this.taskID, getRuntimeContext().getNumberOfParallelSubtasks(), getRuntimeContext().getAttemptNumber(), this.rowType);
}

Example 28 with HoodieException

use of org.apache.hudi.exception.HoodieException in project hudi by apache.

the class BootstrapOperator method loadRecords.

/**
 * Loads all the indices of give partition path into the backup state.
 *
 * @param partitionPath The partition path
 */
@SuppressWarnings("unchecked")
protected void loadRecords(String partitionPath) throws Exception {
    long start = System.currentTimeMillis();
    final int parallelism = getRuntimeContext().getNumberOfParallelSubtasks();
    final int maxParallelism = getRuntimeContext().getMaxNumberOfParallelSubtasks();
    final int taskID = getRuntimeContext().getIndexOfThisSubtask();
    HoodieTimeline commitsTimeline = this.hoodieTable.getMetaClient().getCommitsTimeline();
    if (!StringUtils.isNullOrEmpty(lastInstantTime)) {
        commitsTimeline = commitsTimeline.findInstantsAfter(lastInstantTime);
    }
    Option<HoodieInstant> latestCommitTime = commitsTimeline.filterCompletedInstants().lastInstant();
    if (latestCommitTime.isPresent()) {
        BaseFileUtils fileUtils = BaseFileUtils.getInstance(this.hoodieTable.getBaseFileFormat());
        Schema schema = new TableSchemaResolver(this.hoodieTable.getMetaClient()).getTableAvroSchema();
        List<FileSlice> fileSlices = this.hoodieTable.getSliceView().getLatestFileSlicesBeforeOrOn(partitionPath, latestCommitTime.get().getTimestamp(), true).collect(toList());
        for (FileSlice fileSlice : fileSlices) {
            if (!shouldLoadFile(fileSlice.getFileId(), maxParallelism, parallelism, taskID)) {
                continue;
            }
            LOG.info("Load records from {}.", fileSlice);
            // load parquet records
            fileSlice.getBaseFile().ifPresent(baseFile -> {
                // filter out crushed files
                if (!isValidFile(baseFile.getFileStatus())) {
                    return;
                }
                try (ClosableIterator<HoodieKey> iterator = fileUtils.getHoodieKeyIterator(this.hadoopConf, new Path(baseFile.getPath()))) {
                    iterator.forEachRemaining(hoodieKey -> {
                        output.collect(new StreamRecord(new IndexRecord(generateHoodieRecord(hoodieKey, fileSlice))));
                    });
                }
            });
            // load avro log records
            List<String> logPaths = fileSlice.getLogFiles().filter(logFile -> isValidFile(logFile.getFileStatus())).map(logFile -> logFile.getPath().toString()).collect(toList());
            HoodieMergedLogRecordScanner scanner = FormatUtils.logScanner(logPaths, schema, latestCommitTime.get().getTimestamp(), writeConfig, hadoopConf);
            try {
                for (String recordKey : scanner.getRecords().keySet()) {
                    output.collect(new StreamRecord(new IndexRecord(generateHoodieRecord(new HoodieKey(recordKey, partitionPath), fileSlice))));
                }
            } catch (Exception e) {
                throw new HoodieException(String.format("Error when loading record keys from files: %s", logPaths), e);
            } finally {
                scanner.close();
            }
        }
    }
    long cost = System.currentTimeMillis() - start;
    LOG.info("Task [{}}:{}}] finish loading the index under partition {} and sending them to downstream, time cost: {} milliseconds.", this.getClass().getSimpleName(), taskID, partitionPath, cost);
}

Example 29 with HoodieException

use of org.apache.hudi.exception.HoodieException in project hudi by apache.

the class CompactionCommitSink method commitIfNecessary.

/**
 * Condition to commit: the commit buffer has equal size with the compaction plan operations
 * and all the compact commit event {@link CompactionCommitEvent} has the same compaction instant time.
 *
 * @param instant Compaction commit instant time
 * @param events  Commit events ever received for the instant
 */
private void commitIfNecessary(String instant, Collection<CompactionCommitEvent> events) throws IOException {
    HoodieCompactionPlan compactionPlan = compactionPlanCache.computeIfAbsent(instant, k -> {
        try {
            return CompactionUtils.getCompactionPlan(this.writeClient.getHoodieTable().getMetaClient(), instant);
        } catch (IOException e) {
            throw new HoodieException(e);
        }
    });
    boolean isReady = compactionPlan.getOperations().size() == events.size();
    if (!isReady) {
        return;
    }
    try {
        doCommit(instant, events);
    } catch (Throwable throwable) {
        // make it fail-safe
        LOG.error("Error while committing compaction instant: " + instant, throwable);
    } finally {
        // reset the status
        reset(instant);
    }
}

Example 30 with HoodieException

use of org.apache.hudi.exception.HoodieException in project hudi by apache.

the class StreamWriteFunction method flushRemaining.

@SuppressWarnings("unchecked, rawtypes")
private void flushRemaining(boolean endInput) {
    this.currentInstant = instantToWrite(hasData());
    if (this.currentInstant == null) {
        // in case there are empty checkpoints that has no input data
        throw new HoodieException("No inflight instant when flushing data!");
    }
    final List<WriteStatus> writeStatus;
    if (buckets.size() > 0) {
        writeStatus = new ArrayList<>();
        this.buckets.values().forEach(bucket -> {
            List<HoodieRecord> records = bucket.writeBuffer();
            if (records.size() > 0) {
                if (config.getBoolean(FlinkOptions.PRE_COMBINE)) {
                    records = FlinkWriteHelper.newInstance().deduplicateRecords(records, (HoodieIndex) null, -1);
                }
                bucket.preWrite(records);
                writeStatus.addAll(writeFunction.apply(records, currentInstant));
                records.clear();
                bucket.reset();
            }
        });
    } else {
        LOG.info("No data to write in subtask [{}] for instant [{}]", taskID, currentInstant);
        writeStatus = Collections.emptyList();
    }
    final WriteMetadataEvent event = WriteMetadataEvent.builder().taskID(taskID).instantTime(currentInstant).writeStatus(writeStatus).lastBatch(true).endInput(endInput).build();
    this.eventGateway.sendEventToCoordinator(event);
    this.buckets.clear();
    this.tracer.reset();
    this.writeClient.cleanHandles();
    this.writeStatuses.addAll(writeStatus);
    // blocks flushing until the coordinator starts a new instant
    this.confirming = true;
}