Examples with InvalidSetpointException com.thinkbiganalytics.nifi.v2.core.savepoint.InvalidSetpointException used on opensource projects

Example 1 with InvalidSetpointException

use of com.thinkbiganalytics.nifi.v2.core.savepoint.InvalidSetpointException in project kylo by Teradata.

the class TriggerSavepoint method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    // Fetch the controller
    final SavepointController controller = context.getProperty(SAVEPOINT_SERVICE).asControllerService(SavepointController.class);
    final SavepointProvider provider = controller.getProvider();
    final ComponentLog logger = getLogger();
    final PropertyValue pvSavepointId = context.getProperty(SAVEPOINT_ID);
    final PropertyValue pvBehavior = context.getProperty(BEHAVIOR);
    final PropertyValue pvMaxRetries = context.getProperty(MAX_RETRIES);
    // We do processing on each flowfile here
    String behavior = pvBehavior.getValue();
    if (!FAIL.equals(behavior)) {
        final String savepointIdStr = pvSavepointId.evaluateAttributeExpressions(flowFile).getValue();
        Lock lock = null;
        try {
            lock = provider.lock(savepointIdStr);
            if (lock != null) {
                if (RELEASE.equals(behavior)) {
                    provider.release(savepointIdStr, lock, true);
                    flowFile = session.putAttribute(flowFile, SavepointProvenanceProperties.SAVE_POINT_BEHAVIOR_STATUS, behavior);
                    session.transfer(flowFile, REL_SUCCESS);
                } else if (RETRY.equals(behavior)) {
                    // Check the retry count from the SetSavepoint
                    String sRetryCount = flowFile.getAttribute(SetSavepoint.SAVEPOINT_RETRY_COUNT);
                    int retryCount = 0;
                    try {
                        if (sRetryCount != null) {
                            retryCount = Integer.parseInt(sRetryCount);
                        }
                    } catch (NumberFormatException nfe) {
                        logger.warn("{} has an invalid value '{}' on FlowFile {}", new Object[] { SetSavepoint.SAVEPOINT_RETRY_COUNT, sRetryCount, flowFile });
                    }
                    // Check retries
                    if (retryCount > pvMaxRetries.asInteger()) {
                        flowFile = session.putAttribute(flowFile, TriggerSavepoint.SAVE_POINT_MAX_RETRIES_EXCEEDED, sRetryCount);
                        session.transfer(flowFile, REL_MAX_RETRIES_EXCEEDED);
                        return;
                    }
                    // Penalize the flowfile once before retry is processed
                    String sRetryMarker = flowFile.getAttribute(SAVEPOINT_RETRY_MARKER);
                    if (StringUtils.isEmpty(sRetryMarker)) {
                        flowFile = session.penalize(flowFile);
                        flowFile = session.putAttribute(flowFile, SAVEPOINT_RETRY_MARKER, "1");
                        session.transfer(flowFile, REL_SELF);
                        return;
                    }
                    provider.retry(savepointIdStr, lock);
                    session.transfer(flowFile, REL_SUCCESS);
                }
            } else {
                // Unable to obtain lock. Try again
                session.transfer(flowFile, REL_SELF);
            }
        } catch (IOException | InvalidLockException | InvalidSetpointException e) {
            logger.info("Exception occurred for FlowFile {} exception {}", new Object[] { flowFile, e.getLocalizedMessage() }, e);
            // Check the retry count from the SetSavepoint
            String sTriggerFailureCount = flowFile.getAttribute(TriggerSavepoint.SAVEPOINT_TRIGGER_FAILURE_COUNT);
            int triggerFailureCount = 1;
            try {
                triggerFailureCount = (sTriggerFailureCount == null ? 0 : Integer.parseInt(sTriggerFailureCount));
                triggerFailureCount += 1;
            } catch (NumberFormatException nfe) {
                logger.info("Invalid attribute {}", new Object[] { TriggerSavepoint.SAVEPOINT_TRIGGER_FAILURE_COUNT });
            }
            flowFile = session.putAttribute(flowFile, TriggerSavepoint.SAVEPOINT_TRIGGER_FAILURE_COUNT, String.valueOf(triggerFailureCount));
            if (triggerFailureCount > MAX_FAILURES_ALLOWED) {
                logger.info("Maximum failures reached for sp {}, will route to fail.", new String[] { savepointIdStr });
                flowFile = session.putAttribute(flowFile, SavepointProvenanceProperties.SAVE_POINT_BEHAVIOR_STATUS, FAIL);
                flowFile = session.putAttribute(flowFile, TriggerSavepoint.SAVE_POINT_BEHAVIOR_STATUS_DESC, "Maximum failures at " + triggerFailureCount + " were reached.  Failing the flow");
                // add in the trigger flow id so ops manager can get the key to retry if needed
                String triggerFlowFile = flowFile.getAttribute(SavepointProvenanceProperties.PARENT_FLOWFILE_ID);
                if (StringUtils.isNotBlank(triggerFlowFile)) {
                    flowFile = session.putAttribute(flowFile, SavepointProvenanceProperties.SAVE_POINT_TRIGGER_FLOWFILE, triggerFlowFile);
                }
                session.transfer(flowFile, REL_FAILURE);
            } else {
                logger.info("Failed to process flowfile for savepoint {}", new String[] { savepointIdStr }, e);
                flowFile = session.penalize(flowFile);
                session.transfer(flowFile, REL_SELF);
            }
        } finally {
            if (lock != null) {
                try {
                    provider.unlock(lock);
                } catch (IOException e) {
                    logger.warn("Unable to unlock {}", new String[] { savepointIdStr });
                }
            }
        }
    } else {
        // Route to failure
        flowFile = session.putAttribute(flowFile, SavepointProvenanceProperties.SAVE_POINT_BEHAVIOR_STATUS, behavior);
        String triggerFlowFile = flowFile.getAttribute(SavepointProvenanceProperties.PARENT_FLOWFILE_ID);
        if (StringUtils.isNotBlank(triggerFlowFile)) {
            flowFile = session.putAttribute(flowFile, SavepointProvenanceProperties.SAVE_POINT_TRIGGER_FLOWFILE, triggerFlowFile);
        }
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 2 with InvalidSetpointException

use of com.thinkbiganalytics.nifi.v2.core.savepoint.InvalidSetpointException in project kylo by Teradata.

the class SetSavepointTest method testRetryCount.

@Test
public void testRetryCount() throws InitializationException, IOException, InvalidLockException, InterruptedException, InvalidSetpointException {
    // Set expiration to 2s
    enqueue("sp1");
    final TestIteration testIteration = new TestIteration();
    testIteration.expectedQueueSize = 1;
    testIteration.expectedRetry.add("1");
    testIteration.run();
    for (int i = 1; i <= 5; i++) {
        Lock lock = provider.lock(savepointId);
        provider.retry(savepointId, lock);
        provider.unlock(lock);
        testIteration.doNotClearRunnerState();
        testIteration.expectedQueueSize = 1;
        testIteration.expectedRetry.add("1");
        testIteration.run();
        String retries = runner.getFlowFilesForRelationship(SetSavepoint.REL_TRY).get(0).getAttribute(SetSavepoint.SAVEPOINT_RETRY_COUNT);
        assertEquals(String.valueOf(i), retries);
        runner.clearTransferState();
    }
}

Example 3 with InvalidSetpointException

use of com.thinkbiganalytics.nifi.v2.core.savepoint.InvalidSetpointException in project kylo by Teradata.

the class SetSavepointTest method testRetry.

@Test
public void testRetry() throws InitializationException, IOException, InvalidLockException, InvalidSetpointException {
    enqueue("sp1");
    final TestIteration testIteration = new TestIteration();
    testIteration.expectedQueueSize = 1;
    testIteration.expectedRetry.add("1");
    testIteration.run();
    Lock lock = provider.lock(savepointId);
    assertNotNull("Expecting lock", lock);
    provider.retry(savepointId, lock);
    provider.unlock(lock);
    testIteration.expectedRetry.add("1");
    testIteration.expectedQueueSize = 1;
    testIteration.run();
    lock = provider.lock(savepointId);
    assertNotNull("Expecting lock", lock);
    provider.release(savepointId, lock, true);
    provider.unlock(lock);
    testIteration.expectedReleasedSuccess.add("1");
    testIteration.expectedQueueSize = 0;
    testIteration.run();
    String retryCount = testIteration.releasedSuccess.get(0).getAttribute(SetSavepoint.SAVEPOINT_RETRY_COUNT);
    assertEquals("Expecting retry count of 1", "1", retryCount);
}

Example 4 with InvalidSetpointException

use of com.thinkbiganalytics.nifi.v2.core.savepoint.InvalidSetpointException in project kylo by Teradata.

the class SetSavepointTest method testReleaseOn2ndStep.

@Test
public /**
 * Expect arrival on new setpoint will release prior setpoints
 */
void testReleaseOn2ndStep() throws InitializationException, IOException, InvalidLockException, InvalidSetpointException {
    // Simulate an existing setpoint by another processor
    Lock lock = provider.lock(savepointId);
    provider.register(savepointId, "processor1", "flowFile1", lock);
    provider.unlock(lock);
    enqueue(savepointId);
    final TestIteration testIteration = new TestIteration();
    testIteration.expectedQueueSize = 1;
    testIteration.expectedRetry.add("1");
    testIteration.run();
    SavepointEntry entry = provider.lookupEntry(savepointId);
    Assert.assertEquals(SavepointEntry.SavePointState.RELEASE_SUCCESS, entry.getState("processor1"));
}

Example 5 with InvalidSetpointException

use of com.thinkbiganalytics.nifi.v2.core.savepoint.InvalidSetpointException in project kylo by Teradata.

the class SetSavepointTest method testReleaseFailure.

@Test
public void testReleaseFailure() throws InitializationException, IOException, InvalidLockException, InvalidSetpointException {
    enqueue("sp1");
    final TestIteration testIteration = new TestIteration();
    testIteration.expectedQueueSize = 1;
    testIteration.expectedRetry.add("1");
    testIteration.run();
    testIteration.expectedQueueSize = 1;
    testIteration.run();
    Lock lock = provider.lock(savepointId);
    assertNotNull("Expecting lock", lock);
    provider.release(savepointId, lock, false);
    provider.unlock(lock);
    testIteration.expectedQueueSize = 0;
    testIteration.expectedReleasedFailures.add("1");
    testIteration.run();
}