Examples with ActiveWaterMarksCancelledException com.thinkbiganalytics.nifi.core.api.metadata.ActiveWaterMarksCancelledException used on opensource projects

Example 1 with ActiveWaterMarksCancelledException

use of com.thinkbiganalytics.nifi.core.api.metadata.ActiveWaterMarksCancelledException in project kylo by Teradata.

the class MetadataClientRecorder method commitAllWaterMarks.

/* (non-Javadoc)
     * @see com.thinkbiganalytics.nifi.core.api.metadata.MetadataRecorder#commitAllWaterMarks(org.apache.nifi.processor.ProcessSession, org.apache.nifi.flowfile.FlowFile, java.lang.String)
     */
@Override
public FlowFile commitAllWaterMarks(ProcessSession session, FlowFile ff, String feedId) {
    FlowFile resultFF = ff;
    Set<String> cancelledWaterMarks = new HashSet<>();
    // TODO do more efficiently
    for (String waterMarkName : new HashSet<String>(getCurrentWaterMarksAttr(ff).keySet())) {
        try {
            resultFF = commitWaterMark(session, resultFF, feedId, waterMarkName);
        } catch (ActiveWaterMarksCancelledException e) {
            cancelledWaterMarks.addAll(e.getWaterMarkNames());
        }
    }
    if (cancelledWaterMarks.size() > 0) {
        throw new ActiveWaterMarksCancelledException(feedId, cancelledWaterMarks);
    } else {
        return resultFF;
    }
}

Example 2 with ActiveWaterMarksCancelledException

use of com.thinkbiganalytics.nifi.core.api.metadata.ActiveWaterMarksCancelledException in project kylo by Teradata.

the class ReleaseHighWaterMark method onTrigger.

/* (non-Javadoc)
     * @see org.apache.nifi.processor.AbstractProcessor#onTrigger(org.apache.nifi.processor.ProcessContext, org.apache.nifi.processor.ProcessSession)
     */
@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    MetadataRecorder recorder = context.getProperty(CommonProperties.METADATA_SERVICE).asControllerService(MetadataProviderService.class).getRecorder();
    FlowFile ff = session.get();
    if (ff != null) {
        try {
            ff = initialize(context, session, ff);
        } catch (Exception e) {
            getLog().error("Failure during initialization", e);
            session.transfer(ff, CommonProperties.REL_FAILURE);
        }
        String mode = context.getProperty(MODE).toString();
        try {
            if (mode.equals("COMMIT")) {
                if (context.getProperty(RELEASE_ALL).asBoolean()) {
                    ff = recorder.commitAllWaterMarks(session, ff, getFeedId(context, ff));
                } else {
                    String waterMarkName = context.getProperty(HIGH_WATER_MARK).evaluateAttributeExpressions(ff).toString();
                    ff = recorder.commitWaterMark(session, ff, getFeedId(context, ff), waterMarkName);
                }
            } else {
                if (context.getProperty(RELEASE_ALL).asBoolean()) {
                    ff = recorder.releaseAllWaterMarks(session, ff, getFeedId(context, ff));
                } else {
                    String waterMarkName = context.getProperty(HIGH_WATER_MARK).evaluateAttributeExpressions(ff).toString();
                    ff = recorder.releaseWaterMark(session, ff, getFeedId(context, ff), waterMarkName);
                }
            }
            session.transfer(ff, CommonProperties.REL_SUCCESS);
        } catch (ActiveWaterMarksCancelledException e) {
            transferForCancelledWaterMarks(context, session, ff, e);
        } catch (Exception e) {
            getLog().warn("Failure during release of high-water mark(s)", e);
            session.transfer(ff, CommonProperties.REL_FAILURE);
        }
    }
}

Example 3 with ActiveWaterMarksCancelledException

use of com.thinkbiganalytics.nifi.core.api.metadata.ActiveWaterMarksCancelledException in project kylo by Teradata.

the class MetadataClientRecorder method commitWaterMark.

/* (non-Javadoc)
     * @see com.thinkbiganalytics.nifi.core.api.metadata.MetadataRecorder#commitWaterMark(org.apache.nifi.processor.ProcessSession, org.apache.nifi.flowfile.FlowFile, java.lang.String, java.lang.String)
     */
@Override
public FlowFile commitWaterMark(ProcessSession session, FlowFile ff, String feedId, String waterMarkName) {
    FlowFile resultFF = ff;
    Map<String, WaterMarkParam> ffWaterMarks = getCurrentWaterMarksAttr(ff);
    WaterMarkParam param = ffWaterMarks.get(waterMarkName);
    Long activeTimestamp = getActiveWaterMarkTimestamp(feedId, waterMarkName);
    if (param == null) {
        log.error("Received request to commit a water mark that does not exist in the flowfile: {}", waterMarkName);
        return resultFF;
    } else if (activeTimestamp == null) {
        log.warn("Received request to commit a water mark that is not active: {}", waterMarkName);
        return resultFF;
    } else if (param.timestamp != activeTimestamp) {
        // If the water mark timestamp does not match the one recorded as an active water mark this means
        // this flowfile's water mark has been canceled and another flow file should be considered the active one.
        // So this water mark value has been superseded and its value should not be committed and a canceled exception thrown.
        log.info("Received request to commit a water mark version that is no longer active: {}/{}", waterMarkName, param.timestamp);
        throw new ActiveWaterMarksCancelledException(feedId, waterMarkName);
    }
    try {
        String value = resultFF.getAttribute(param.name);
        updateHighWaterMarkValue(feedId, waterMarkName, value);
    } finally {
        releaseActiveWaterMark(feedId, waterMarkName, activeTimestamp);
        resultFF = removeFromCurrentWaterMarksAttr(session, resultFF, waterMarkName, param.name);
    }
    return resultFF;
}