Examples with WatermarkAwareWriter org.apache.gobblin.writer.WatermarkAwareWriter used on opensource projects

Example 1 with WatermarkAwareWriter

use of org.apache.gobblin.writer.WatermarkAwareWriter in project incubator-gobblin by apache.

the class Fork method processRecord.

protected void processRecord(Object record) throws IOException, DataConversionException {
    if (this.forkState.compareAndSet(ForkState.FAILED, ForkState.FAILED)) {
        throw new IllegalStateException(String.format("Fork %d of task %s has failed and is no longer running", this.index, this.taskId));
    }
    if (record == null || record == SHUTDOWN_RECORD) {
        /**
         * null record indicates a timeout on record acquisition, SHUTDOWN_RECORD is sent during shutdown.
         * Will loop unless the parent task has indicated that it is already done pulling records.
         */
        if (this.parentTaskDone) {
            return;
        }
    } else {
        if (isStreamingMode()) {
            // Unpack the record from its container
            RecordEnvelope recordEnvelope = (RecordEnvelope) record;
            // Convert the record, check its data quality, and finally write it out if quality checking passes.
            for (Object convertedRecord : this.converter.convertRecord(this.convertedSchema, recordEnvelope.getRecord(), this.taskState)) {
                if (this.rowLevelPolicyChecker.executePolicies(convertedRecord, this.rowLevelPolicyCheckingResult)) {
                    // for each additional record we pass down, increment the acks needed
                    ((WatermarkAwareWriter) this.writer.get()).writeEnvelope(recordEnvelope.withRecord(convertedRecord));
                }
            }
            // ack this fork's processing done
            recordEnvelope.ack();
        } else {
            buildWriterIfNotPresent();
            // Convert the record, check its data quality, and finally write it out if quality checking passes.
            for (Object convertedRecord : this.converter.convertRecord(this.convertedSchema, record, this.taskState)) {
                if (this.rowLevelPolicyChecker.executePolicies(convertedRecord, this.rowLevelPolicyCheckingResult)) {
                    this.writer.get().writeEnvelope(new RecordEnvelope<>(convertedRecord));
                }
            }
        }
    }
}