Examples with WriteMetadataEvent org.apache.hudi.sink.event.WriteMetadataEvent used on opensource projects

Example 1 with WriteMetadataEvent

use of org.apache.hudi.sink.event.WriteMetadataEvent in project hudi by apache.

the class BulkInsertWriteFunction method sendBootstrapEvent.

private void sendBootstrapEvent() {
    WriteMetadataEvent event = WriteMetadataEvent.builder().taskID(taskID).writeStatus(Collections.emptyList()).instantTime("").bootstrap(true).build();
    this.eventGateway.sendEventToCoordinator(event);
    LOG.info("Send bootstrap write metadata event to coordinator, task[{}].", taskID);
}

Example 2 with WriteMetadataEvent

use of org.apache.hudi.sink.event.WriteMetadataEvent in project hudi by apache.

the class AbstractStreamWriteFunction method reloadWriteMetaState.

/**
 * Reload the write metadata state as the current checkpoint.
 */
private void reloadWriteMetaState() throws Exception {
    this.writeMetadataState.clear();
    WriteMetadataEvent event = WriteMetadataEvent.builder().taskID(taskID).instantTime(currentInstant).writeStatus(new ArrayList<>(writeStatuses)).bootstrap(true).build();
    this.writeMetadataState.add(event);
    writeStatuses.clear();
}

Example 3 with WriteMetadataEvent

use of org.apache.hudi.sink.event.WriteMetadataEvent in project hudi by apache.

the class StreamWriteFunction method flushRemaining.

@SuppressWarnings("unchecked, rawtypes")
private void flushRemaining(boolean endInput) {
    this.currentInstant = instantToWrite(hasData());
    if (this.currentInstant == null) {
        // in case there are empty checkpoints that has no input data
        throw new HoodieException("No inflight instant when flushing data!");
    }
    final List<WriteStatus> writeStatus;
    if (buckets.size() > 0) {
        writeStatus = new ArrayList<>();
        this.buckets.values().forEach(bucket -> {
            List<HoodieRecord> records = bucket.writeBuffer();
            if (records.size() > 0) {
                if (config.getBoolean(FlinkOptions.PRE_COMBINE)) {
                    records = FlinkWriteHelper.newInstance().deduplicateRecords(records, (HoodieIndex) null, -1);
                }
                bucket.preWrite(records);
                writeStatus.addAll(writeFunction.apply(records, currentInstant));
                records.clear();
                bucket.reset();
            }
        });
    } else {
        LOG.info("No data to write in subtask [{}] for instant [{}]", taskID, currentInstant);
        writeStatus = Collections.emptyList();
    }
    final WriteMetadataEvent event = WriteMetadataEvent.builder().taskID(taskID).instantTime(currentInstant).writeStatus(writeStatus).lastBatch(true).endInput(endInput).build();
    this.eventGateway.sendEventToCoordinator(event);
    this.buckets.clear();
    this.tracer.reset();
    this.writeClient.cleanHandles();
    this.writeStatuses.addAll(writeStatus);
    // blocks flushing until the coordinator starts a new instant
    this.confirming = true;
}

Example 4 with WriteMetadataEvent

use of org.apache.hudi.sink.event.WriteMetadataEvent in project hudi by apache.

the class StreamWriteOperatorCoordinator method handleEventFromOperator.

@Override
public void handleEventFromOperator(int i, OperatorEvent operatorEvent) {
    ValidationUtils.checkState(operatorEvent instanceof WriteMetadataEvent, "The coordinator can only handle WriteMetaEvent");
    WriteMetadataEvent event = (WriteMetadataEvent) operatorEvent;
    if (event.isEndInput()) {
        // handle end input event synchronously
        handleEndInputEvent(event);
    } else {
        executor.execute(() -> {
            if (event.isBootstrap()) {
                handleBootstrapEvent(event);
            } else {
                handleWriteMetaEvent(event);
            }
        }, "handle write metadata event for instant %s", this.instant);
    }
}

Example 5 with WriteMetadataEvent

use of org.apache.hudi.sink.event.WriteMetadataEvent in project hudi by apache.

the class AppendWriteFunction method flushData.

private void flushData(boolean endInput) {
    final List<WriteStatus> writeStatus;
    final String instant;
    if (this.writerHelper != null) {
        writeStatus = this.writerHelper.getWriteStatuses(this.taskID);
        instant = this.writerHelper.getInstantTime();
    } else {
        LOG.info("No data to write in subtask [{}] for instant [{}]", taskID, currentInstant);
        writeStatus = Collections.emptyList();
        instant = instantToWrite(false);
    }
    final WriteMetadataEvent event = WriteMetadataEvent.builder().taskID(taskID).instantTime(instant).writeStatus(writeStatus).lastBatch(true).endInput(endInput).build();
    this.eventGateway.sendEventToCoordinator(event);
    // nullify the write helper for next ckp
    this.writerHelper = null;
    this.writeStatuses.addAll(writeStatus);
    // blocks flushing until the coordinator starts a new instant
    this.confirming = true;
}