Examples with SavepointEntry com.thinkbiganalytics.nifi.v2.core.savepoint.SavepointEntry used on opensource projects

Example 1 with SavepointEntry

use of com.thinkbiganalytics.nifi.v2.core.savepoint.SavepointEntry in project kylo by Teradata.

the class SetSavepoint method getNextFlowFilex.

private Optional<FlowFile> getNextFlowFilex(ProcessContext context, ProcessSession session, SavepointProvider provider, PropertyValue pvSavepointId) {
    long expirationDuration = context.getProperty(EXPIRATION_DURATION).asTimePeriod(TimeUnit.MILLISECONDS);
    List<FlowFile> match = new ArrayList<>();
    List<FlowFile> noMatch = new LinkedList<>();
    session.get(session.getQueueSize().getObjectCount()).stream().sorted(Comparator.comparing(FlowFile::getLastQueueDate).reversed()).forEach(f -> {
        boolean isMatch = false;
        if (match.isEmpty()) {
            final String savepointIdStr = pvSavepointId.evaluateAttributeExpressions(f).getValue();
            String processorId = getIdentifier();
            SavepointEntry entry = provider.lookupEntry(savepointIdStr);
            if (entry == null || entry.getState(processorId) == null || isExpired(f, expirationDuration)) {
                isMatch = true;
            } else if (SavepointEntry.SavePointState.WAIT != entry.getState(processorId)) {
                isMatch = true;
            }
            // add it
            if (isMatch) {
                match.add(f);
            } else {
                noMatch.add(f);
            }
        } else {
            noMatch.add(f);
        }
    });
    // clear those that failed
    session.transfer(noMatch);
    return match.isEmpty() ? Optional.empty() : Optional.of(match.get(0));
}

Example 2 with SavepointEntry

use of com.thinkbiganalytics.nifi.v2.core.savepoint.SavepointEntry in project kylo by Teradata.

the class SavepointCacheInitializingFilter method filter.

public FlowFileFilterResult filter(FlowFile f) {
    final String savepointIdStr = contextData.getSavepointId().evaluateAttributeExpressions(f).getValue();
    SavepointEntry entry = contextData.getProvider().lookupEntry(savepointIdStr);
    if (entry == null || entry.getState(contextData.getProcessorId()) == null) {
        flowfilesToCache.put(f.getLastQueueDate(), f.getAttribute(CoreAttributes.UUID.key()));
    } else if (SavepointFlowFileFilterUtil.isExpired(f, contextData.getExpirationDuration())) {
        savepointsToCache.put(f.getLastQueueDate(), entry);
    } else if (SavepointEntry.SavePointState.WAIT != entry.getState(contextData.getProcessorId())) {
        savepointsToCache.put(f.getLastQueueDate(), entry);
    }
    return FlowFileFilterResult.REJECT_AND_CONTINUE;
}

Example 3 with SavepointEntry

use of com.thinkbiganalytics.nifi.v2.core.savepoint.SavepointEntry in project kylo by Teradata.

the class SetSavepointTest method testReleaseOn2ndStep.

@Test
public /**
 * Expect arrival on new setpoint will release prior setpoints
 */
void testReleaseOn2ndStep() throws InitializationException, IOException, InvalidLockException, InvalidSetpointException {
    // Simulate an existing setpoint by another processor
    Lock lock = provider.lock(savepointId);
    provider.register(savepointId, "processor1", "flowFile1", lock);
    provider.unlock(lock);
    enqueue(savepointId);
    final TestIteration testIteration = new TestIteration();
    testIteration.expectedQueueSize = 1;
    testIteration.expectedRetry.add("1");
    testIteration.run();
    SavepointEntry entry = provider.lookupEntry(savepointId);
    Assert.assertEquals(SavepointEntry.SavePointState.RELEASE_SUCCESS, entry.getState("processor1"));
}

Example 4 with SavepointEntry

use of com.thinkbiganalytics.nifi.v2.core.savepoint.SavepointEntry in project kylo by Teradata.

the class SavepointCacheInitializingFilter method toSavepoint.

private SavepointEntry toSavepoint(String processorId, String flowfileId) {
    SavepointEntry savepointEntry = new SavepointEntry();
    savepointEntry.register(processorId, flowfileId);
    savepointEntry.retry();
    return savepointEntry;
}

Example 5 with SavepointEntry

use of com.thinkbiganalytics.nifi.v2.core.savepoint.SavepointEntry in project kylo by Teradata.

the class SetSavepoint method processFlowFile.

private void processFlowFile(FlowFile flowFile, final ProcessContext context, final ProcessSession session, final SavepointContextData contextData) {
    final ComponentLog logger = getLogger();
    final SavepointController controller = contextData.getController();
    final SavepointProvider provider = contextData.getProvider();
    final PropertyValue pvSavepointId = contextData.getSavepointId();
    final String processorId = contextData.getProcessorId();
    // We do processing on each flowfile here
    final String savepointIdStr = pvSavepointId.evaluateAttributeExpressions(flowFile).getValue();
    final String flowfileId = flowFile.getAttribute(CoreAttributes.UUID.key());
    Lock lock = null;
    try {
        lock = provider.lock(savepointIdStr);
        if (lock != null) {
            SavepointEntry entry = provider.lookupEntry(savepointIdStr);
            if (isExpired(context, session, provider, flowFile, savepointIdStr, lock)) {
                return;
            }
            String waitStartTimestamp;
            // add the processor id for the current savepoint
            // this will be used to check on the next save point if the flow file should be examined and processed.
            flowFile = session.putAttribute(flowFile, SAVEPOINT_PROCESSOR_ID, getIdentifier());
            if (entry == null || entry.getState(processorId) == null) {
                // Register new
                provider.register(savepointIdStr, processorId, flowfileId, lock);
                flowFile = tryFlowFile(session, flowFile, "-1");
                // add in timestamps
                // Set wait start timestamp if it's not set yet
                waitStartTimestamp = flowFile.getAttribute(SAVEPOINT_START_TIMESTAMP);
                if (waitStartTimestamp == null) {
                    waitStartTimestamp = String.valueOf(System.currentTimeMillis());
                    flowFile = session.putAttribute(flowFile, SAVEPOINT_START_TIMESTAMP, waitStartTimestamp);
                }
                session.transfer(flowFile);
            } else {
                SavepointEntry.SavePointState state = entry.getState(processorId);
                switch(state) {
                    case RELEASE_SUCCESS:
                        provider.commitRelease(savepointIdStr, processorId, lock);
                        // add provenance to indicate success
                        flowFile = session.putAttribute(flowFile, SavepointProvenanceProperties.RELEASE_STATUS_KEY, SavepointProvenanceProperties.RELEASE_STATUS.SUCCESS.name());
                        session.transfer(flowFile, REL_RELEASE_SUCCESS);
                        break;
                    case RELEASE_FAILURE:
                        provider.commitRelease(savepointIdStr, processorId, lock);
                        // add provenance to indicate failure
                        flowFile = session.putAttribute(flowFile, SavepointProvenanceProperties.RELEASE_STATUS_KEY, SavepointProvenanceProperties.RELEASE_STATUS.FAILURE.name());
                        session.transfer(flowFile, REL_RELEASE_FAILURE);
                        break;
                    case RETRY:
                        String retryCount = flowFile.getAttribute(SAVEPOINT_RETRY_COUNT);
                        if (retryCount == null) {
                            retryCount = "0";
                        }
                        provider.commitRetry(savepointIdStr, processorId, lock);
                        flowFile = tryFlowFile(session, flowFile, retryCount);
                        session.transfer(flowFile);
                        break;
                    case WAIT:
                        session.transfer(flowFile, REL_SELF);
                        break;
                    default:
                        logger.warn("Unexpected savepoint state.");
                        session.transfer(flowFile, REL_FAILURE);
                }
            }
        } else {
            // Lock busy so try again later
            // add it back to cache
            controller.putFlowfileBack(processorId, flowfileId, true);
            logger.info("Unable to obtain lock.  It is already locked by another process.  Adding back to queue {} ", new Object[] { flowfileId });
            session.transfer(flowFile, REL_SELF);
        }
    } catch (IOException | InvalidLockException | InvalidSetpointException e) {
        logger.warn("Failed to process flowfile {} for savepoint {}", new String[] { flowfileId, savepointIdStr }, e);
        flowFile = session.putAttribute(flowFile, SAVEPOINT_EXCEPTION, "Failed to process flowfile " + flowfileId + " for savepoint " + savepointIdStr + ". " + e.getMessage());
        session.transfer(flowFile, REL_FAILURE);
    } finally {
        if (lock != null) {
            try {
                provider.unlock(lock);
            } catch (IOException e) {
                logger.warn("Unable to unlock {}", new String[] { savepointIdStr });
            }
        }
    }
}