Examples with Lock com.thinkbiganalytics.nifi.v2.core.savepoint.Lock used on opensource projects

Example 6 with Lock

use of com.thinkbiganalytics.nifi.v2.core.savepoint.Lock in project kylo by Teradata.

the class SetSavepointTest method testReleaseOn2ndStep.

@Test
public /**
 * Expect arrival on new setpoint will release prior setpoints
 */
void testReleaseOn2ndStep() throws InitializationException, IOException, InvalidLockException, InvalidSetpointException {
    // Simulate an existing setpoint by another processor
    Lock lock = provider.lock(savepointId);
    provider.register(savepointId, "processor1", "flowFile1", lock);
    provider.unlock(lock);
    enqueue(savepointId);
    final TestIteration testIteration = new TestIteration();
    testIteration.expectedQueueSize = 1;
    testIteration.expectedRetry.add("1");
    testIteration.run();
    SavepointEntry entry = provider.lookupEntry(savepointId);
    Assert.assertEquals(SavepointEntry.SavePointState.RELEASE_SUCCESS, entry.getState("processor1"));
}

Example 7 with Lock

use of com.thinkbiganalytics.nifi.v2.core.savepoint.Lock in project kylo by Teradata.

the class SetSavepointTest method testReleaseFailure.

@Test
public void testReleaseFailure() throws InitializationException, IOException, InvalidLockException, InvalidSetpointException {
    enqueue("sp1");
    final TestIteration testIteration = new TestIteration();
    testIteration.expectedQueueSize = 1;
    testIteration.expectedRetry.add("1");
    testIteration.run();
    testIteration.expectedQueueSize = 1;
    testIteration.run();
    Lock lock = provider.lock(savepointId);
    assertNotNull("Expecting lock", lock);
    provider.release(savepointId, lock, false);
    provider.unlock(lock);
    testIteration.expectedQueueSize = 0;
    testIteration.expectedReleasedFailures.add("1");
    testIteration.run();
}

Example 8 with Lock

use of com.thinkbiganalytics.nifi.v2.core.savepoint.Lock in project kylo by Teradata.

the class TriggerSavepointTest method testRetry2ndAttempt.

@Test
public void testRetry2ndAttempt() throws InitializationException, IOException, InvalidLockException, InterruptedException {
    final String savepointId = "sp1";
    final Map<String, String> props1 = new HashMap<>();
    props1.put("savepointid", savepointId);
    props1.put(TriggerSavepoint.SAVEPOINT_RETRY_MARKER, "1");
    runner.enqueue(new byte[] {}, props1);
    Lock lock = provider.lock(savepointId);
    provider.register(savepointId, "p1", "flowFile1", lock);
    provider.unlock(lock);
    TestIteration iteration = new TestIteration();
    iteration.expectedPenalizedCount = 0;
    iteration.expectedQueueSize = 0;
    iteration.expectedSuccess.add("1");
    iteration.run();
}

Example 9 with Lock

use of com.thinkbiganalytics.nifi.v2.core.savepoint.Lock in project kylo by Teradata.

the class SetSavepoint method processFlowFile.

private void processFlowFile(FlowFile flowFile, final ProcessContext context, final ProcessSession session, final SavepointContextData contextData) {
    final ComponentLog logger = getLogger();
    final SavepointController controller = contextData.getController();
    final SavepointProvider provider = contextData.getProvider();
    final PropertyValue pvSavepointId = contextData.getSavepointId();
    final String processorId = contextData.getProcessorId();
    // We do processing on each flowfile here
    final String savepointIdStr = pvSavepointId.evaluateAttributeExpressions(flowFile).getValue();
    final String flowfileId = flowFile.getAttribute(CoreAttributes.UUID.key());
    Lock lock = null;
    try {
        lock = provider.lock(savepointIdStr);
        if (lock != null) {
            SavepointEntry entry = provider.lookupEntry(savepointIdStr);
            if (isExpired(context, session, provider, flowFile, savepointIdStr, lock)) {
                return;
            }
            String waitStartTimestamp;
            // add the processor id for the current savepoint
            // this will be used to check on the next save point if the flow file should be examined and processed.
            flowFile = session.putAttribute(flowFile, SAVEPOINT_PROCESSOR_ID, getIdentifier());
            if (entry == null || entry.getState(processorId) == null) {
                // Register new
                provider.register(savepointIdStr, processorId, flowfileId, lock);
                flowFile = tryFlowFile(session, flowFile, "-1");
                // add in timestamps
                // Set wait start timestamp if it's not set yet
                waitStartTimestamp = flowFile.getAttribute(SAVEPOINT_START_TIMESTAMP);
                if (waitStartTimestamp == null) {
                    waitStartTimestamp = String.valueOf(System.currentTimeMillis());
                    flowFile = session.putAttribute(flowFile, SAVEPOINT_START_TIMESTAMP, waitStartTimestamp);
                }
                session.transfer(flowFile);
            } else {
                SavepointEntry.SavePointState state = entry.getState(processorId);
                switch(state) {
                    case RELEASE_SUCCESS:
                        provider.commitRelease(savepointIdStr, processorId, lock);
                        // add provenance to indicate success
                        flowFile = session.putAttribute(flowFile, SavepointProvenanceProperties.RELEASE_STATUS_KEY, SavepointProvenanceProperties.RELEASE_STATUS.SUCCESS.name());
                        session.transfer(flowFile, REL_RELEASE_SUCCESS);
                        break;
                    case RELEASE_FAILURE:
                        provider.commitRelease(savepointIdStr, processorId, lock);
                        // add provenance to indicate failure
                        flowFile = session.putAttribute(flowFile, SavepointProvenanceProperties.RELEASE_STATUS_KEY, SavepointProvenanceProperties.RELEASE_STATUS.FAILURE.name());
                        session.transfer(flowFile, REL_RELEASE_FAILURE);
                        break;
                    case RETRY:
                        String retryCount = flowFile.getAttribute(SAVEPOINT_RETRY_COUNT);
                        if (retryCount == null) {
                            retryCount = "0";
                        }
                        provider.commitRetry(savepointIdStr, processorId, lock);
                        flowFile = tryFlowFile(session, flowFile, retryCount);
                        session.transfer(flowFile);
                        break;
                    case WAIT:
                        session.transfer(flowFile, REL_SELF);
                        break;
                    default:
                        logger.warn("Unexpected savepoint state.");
                        session.transfer(flowFile, REL_FAILURE);
                }
            }
        } else {
            // Lock busy so try again later
            // add it back to cache
            controller.putFlowfileBack(processorId, flowfileId, true);
            logger.info("Unable to obtain lock.  It is already locked by another process.  Adding back to queue {} ", new Object[] { flowfileId });
            session.transfer(flowFile, REL_SELF);
        }
    } catch (IOException | InvalidLockException | InvalidSetpointException e) {
        logger.warn("Failed to process flowfile {} for savepoint {}", new String[] { flowfileId, savepointIdStr }, e);
        flowFile = session.putAttribute(flowFile, SAVEPOINT_EXCEPTION, "Failed to process flowfile " + flowfileId + " for savepoint " + savepointIdStr + ". " + e.getMessage());
        session.transfer(flowFile, REL_FAILURE);
    } finally {
        if (lock != null) {
            try {
                provider.unlock(lock);
            } catch (IOException e) {
                logger.warn("Unable to unlock {}", new String[] { savepointIdStr });
            }
        }
    }
}

Example 10 with Lock

use of com.thinkbiganalytics.nifi.v2.core.savepoint.Lock in project kylo by Teradata.

the class SetSavepointTest method testWaitRelease.

@Test
public void testWaitRelease() throws InitializationException, IOException, InvalidLockException, InvalidSetpointException {
    enqueue("sp1");
    final TestIteration testIteration = new TestIteration();
    testIteration.expectedQueueSize = 1;
    testIteration.expectedRetry.add("1");
    testIteration.run();
    testIteration.expectedQueueSize = 1;
    testIteration.run();
    Lock lock = provider.lock(savepointId);
    assertNotNull("Expecting lock", lock);
    provider.release(savepointId, lock, true);
    provider.unlock(lock);
    testIteration.expectedQueueSize = 0;
    testIteration.expectedReleasedSuccess.add("1");
    testIteration.run();
}