use of com.thinkbiganalytics.nifi.v2.core.savepoint.InvalidLockException in project kylo by Teradata.
the class SetSavepointTest method testReleaseOn2ndStep.
@Test
public /**
* Expect arrival on new setpoint will release prior setpoints
*/
void testReleaseOn2ndStep() throws InitializationException, IOException, InvalidLockException, InvalidSetpointException {
// Simulate an existing setpoint by another processor
Lock lock = provider.lock(savepointId);
provider.register(savepointId, "processor1", "flowFile1", lock);
provider.unlock(lock);
enqueue(savepointId);
final TestIteration testIteration = new TestIteration();
testIteration.expectedQueueSize = 1;
testIteration.expectedRetry.add("1");
testIteration.run();
SavepointEntry entry = provider.lookupEntry(savepointId);
Assert.assertEquals(SavepointEntry.SavePointState.RELEASE_SUCCESS, entry.getState("processor1"));
}
use of com.thinkbiganalytics.nifi.v2.core.savepoint.InvalidLockException in project kylo by Teradata.
the class SetSavepointTest method testReleaseFailure.
@Test
public void testReleaseFailure() throws InitializationException, IOException, InvalidLockException, InvalidSetpointException {
enqueue("sp1");
final TestIteration testIteration = new TestIteration();
testIteration.expectedQueueSize = 1;
testIteration.expectedRetry.add("1");
testIteration.run();
testIteration.expectedQueueSize = 1;
testIteration.run();
Lock lock = provider.lock(savepointId);
assertNotNull("Expecting lock", lock);
provider.release(savepointId, lock, false);
provider.unlock(lock);
testIteration.expectedQueueSize = 0;
testIteration.expectedReleasedFailures.add("1");
testIteration.run();
}
use of com.thinkbiganalytics.nifi.v2.core.savepoint.InvalidLockException in project kylo by Teradata.
the class TriggerSavepointTest method testRetry2ndAttempt.
@Test
public void testRetry2ndAttempt() throws InitializationException, IOException, InvalidLockException, InterruptedException {
final String savepointId = "sp1";
final Map<String, String> props1 = new HashMap<>();
props1.put("savepointid", savepointId);
props1.put(TriggerSavepoint.SAVEPOINT_RETRY_MARKER, "1");
runner.enqueue(new byte[] {}, props1);
Lock lock = provider.lock(savepointId);
provider.register(savepointId, "p1", "flowFile1", lock);
provider.unlock(lock);
TestIteration iteration = new TestIteration();
iteration.expectedPenalizedCount = 0;
iteration.expectedQueueSize = 0;
iteration.expectedSuccess.add("1");
iteration.run();
}
use of com.thinkbiganalytics.nifi.v2.core.savepoint.InvalidLockException in project kylo by Teradata.
the class SetSavepoint method processFlowFile.
private void processFlowFile(FlowFile flowFile, final ProcessContext context, final ProcessSession session, final SavepointContextData contextData) {
final ComponentLog logger = getLogger();
final SavepointController controller = contextData.getController();
final SavepointProvider provider = contextData.getProvider();
final PropertyValue pvSavepointId = contextData.getSavepointId();
final String processorId = contextData.getProcessorId();
// We do processing on each flowfile here
final String savepointIdStr = pvSavepointId.evaluateAttributeExpressions(flowFile).getValue();
final String flowfileId = flowFile.getAttribute(CoreAttributes.UUID.key());
Lock lock = null;
try {
lock = provider.lock(savepointIdStr);
if (lock != null) {
SavepointEntry entry = provider.lookupEntry(savepointIdStr);
if (isExpired(context, session, provider, flowFile, savepointIdStr, lock)) {
return;
}
String waitStartTimestamp;
// add the processor id for the current savepoint
// this will be used to check on the next save point if the flow file should be examined and processed.
flowFile = session.putAttribute(flowFile, SAVEPOINT_PROCESSOR_ID, getIdentifier());
if (entry == null || entry.getState(processorId) == null) {
// Register new
provider.register(savepointIdStr, processorId, flowfileId, lock);
flowFile = tryFlowFile(session, flowFile, "-1");
// add in timestamps
// Set wait start timestamp if it's not set yet
waitStartTimestamp = flowFile.getAttribute(SAVEPOINT_START_TIMESTAMP);
if (waitStartTimestamp == null) {
waitStartTimestamp = String.valueOf(System.currentTimeMillis());
flowFile = session.putAttribute(flowFile, SAVEPOINT_START_TIMESTAMP, waitStartTimestamp);
}
session.transfer(flowFile);
} else {
SavepointEntry.SavePointState state = entry.getState(processorId);
switch(state) {
case RELEASE_SUCCESS:
provider.commitRelease(savepointIdStr, processorId, lock);
// add provenance to indicate success
flowFile = session.putAttribute(flowFile, SavepointProvenanceProperties.RELEASE_STATUS_KEY, SavepointProvenanceProperties.RELEASE_STATUS.SUCCESS.name());
session.transfer(flowFile, REL_RELEASE_SUCCESS);
break;
case RELEASE_FAILURE:
provider.commitRelease(savepointIdStr, processorId, lock);
// add provenance to indicate failure
flowFile = session.putAttribute(flowFile, SavepointProvenanceProperties.RELEASE_STATUS_KEY, SavepointProvenanceProperties.RELEASE_STATUS.FAILURE.name());
session.transfer(flowFile, REL_RELEASE_FAILURE);
break;
case RETRY:
String retryCount = flowFile.getAttribute(SAVEPOINT_RETRY_COUNT);
if (retryCount == null) {
retryCount = "0";
}
provider.commitRetry(savepointIdStr, processorId, lock);
flowFile = tryFlowFile(session, flowFile, retryCount);
session.transfer(flowFile);
break;
case WAIT:
session.transfer(flowFile, REL_SELF);
break;
default:
logger.warn("Unexpected savepoint state.");
session.transfer(flowFile, REL_FAILURE);
}
}
} else {
// Lock busy so try again later
// add it back to cache
controller.putFlowfileBack(processorId, flowfileId, true);
logger.info("Unable to obtain lock. It is already locked by another process. Adding back to queue {} ", new Object[] { flowfileId });
session.transfer(flowFile, REL_SELF);
}
} catch (IOException | InvalidLockException | InvalidSetpointException e) {
logger.warn("Failed to process flowfile {} for savepoint {}", new String[] { flowfileId, savepointIdStr }, e);
flowFile = session.putAttribute(flowFile, SAVEPOINT_EXCEPTION, "Failed to process flowfile " + flowfileId + " for savepoint " + savepointIdStr + ". " + e.getMessage());
session.transfer(flowFile, REL_FAILURE);
} finally {
if (lock != null) {
try {
provider.unlock(lock);
} catch (IOException e) {
logger.warn("Unable to unlock {}", new String[] { savepointIdStr });
}
}
}
}
use of com.thinkbiganalytics.nifi.v2.core.savepoint.InvalidLockException in project kylo by Teradata.
the class SetSavepointTest method testWaitRelease.
@Test
public void testWaitRelease() throws InitializationException, IOException, InvalidLockException, InvalidSetpointException {
enqueue("sp1");
final TestIteration testIteration = new TestIteration();
testIteration.expectedQueueSize = 1;
testIteration.expectedRetry.add("1");
testIteration.run();
testIteration.expectedQueueSize = 1;
testIteration.run();
Lock lock = provider.lock(savepointId);
assertNotNull("Expecting lock", lock);
provider.release(savepointId, lock, true);
provider.unlock(lock);
testIteration.expectedQueueSize = 0;
testIteration.expectedReleasedSuccess.add("1");
testIteration.run();
}
Aggregations