Examples with ProcessSession org.apache.nifi.processor.ProcessSession used on opensource projects

Example 76 with ProcessSession

use of org.apache.nifi.processor.ProcessSession in project nifi by apache.

the class StandardProcessSession method rollback.

private void rollback(final boolean penalize, final boolean rollbackCheckpoint) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("{} session rollback called, FlowFile records are {} {}", this, loggableFlowfileInfo(), new Throwable("Stack Trace on rollback"));
    }
    deleteOnCommit.clear();
    closeStreams(openInputStreams, "rolled back", "input");
    closeStreams(openOutputStreams, "rolled back", "output");
    try {
        claimCache.reset();
    } catch (IOException e1) {
        LOG.warn("{} Attempted to close Output Stream for {} due to session rollback but close failed", this, this.connectableDescription, e1);
    }
    final Set<StandardRepositoryRecord> recordsToHandle = new HashSet<>();
    recordsToHandle.addAll(records.values());
    if (rollbackCheckpoint) {
        final Checkpoint existingCheckpoint = this.checkpoint;
        this.checkpoint = null;
        if (existingCheckpoint != null && existingCheckpoint.records != null) {
            recordsToHandle.addAll(existingCheckpoint.records.values());
        }
    }
    resetWriteClaims();
    resetReadClaim();
    if (recordsToHandle.isEmpty()) {
        LOG.trace("{} was rolled back, but no events were performed by this ProcessSession", this);
        acknowledgeRecords();
        resetState();
        return;
    }
    for (final StandardRepositoryRecord record : recordsToHandle) {
        // remove the working claims if they are different than the originals.
        removeTemporaryClaim(record);
    }
    final Set<RepositoryRecord> abortedRecords = new HashSet<>();
    final Set<StandardRepositoryRecord> transferRecords = new HashSet<>();
    for (final StandardRepositoryRecord record : recordsToHandle) {
        if (record.isMarkedForAbort()) {
            decrementClaimCount(record.getWorkingClaim());
            if (record.getCurrentClaim() != null && !record.getCurrentClaim().equals(record.getWorkingClaim())) {
                // if working & original claim are same, don't remove twice; we only want to remove the original
                // if it's different from the working. Otherwise, we remove two claimant counts. This causes
                // an issue if we only updated the flowfile attributes.
                decrementClaimCount(record.getCurrentClaim());
            }
            abortedRecords.add(record);
        } else {
            transferRecords.add(record);
        }
    }
    // Put the FlowFiles that are not marked for abort back to their original queues
    for (final StandardRepositoryRecord record : transferRecords) {
        if (record.getOriginal() != null) {
            final FlowFileQueue originalQueue = record.getOriginalQueue();
            if (originalQueue != null) {
                if (penalize) {
                    final long expirationEpochMillis = System.currentTimeMillis() + context.getConnectable().getPenalizationPeriod(TimeUnit.MILLISECONDS);
                    final FlowFileRecord newFile = new StandardFlowFileRecord.Builder().fromFlowFile(record.getOriginal()).penaltyExpirationTime(expirationEpochMillis).build();
                    originalQueue.put(newFile);
                } else {
                    originalQueue.put(record.getOriginal());
                }
            }
        }
    }
    if (!abortedRecords.isEmpty()) {
        try {
            context.getFlowFileRepository().updateRepository(abortedRecords);
        } catch (final IOException ioe) {
            LOG.error("Unable to update FlowFile repository for aborted records due to {}", ioe.toString());
            if (LOG.isDebugEnabled()) {
                LOG.error("", ioe);
            }
        }
    }
    // If we have transient claims that need to be cleaned up, do so.
    final List<ContentClaim> transientClaims = recordsToHandle.stream().flatMap(record -> record.getTransientClaims().stream()).collect(Collectors.toList());
    if (!transientClaims.isEmpty()) {
        final RepositoryRecord repoRecord = new TransientClaimRepositoryRecord(transientClaims);
        try {
            context.getFlowFileRepository().updateRepository(Collections.singletonList(repoRecord));
        } catch (final IOException ioe) {
            LOG.error("Unable to update FlowFile repository to cleanup transient claims due to {}", ioe.toString());
            if (LOG.isDebugEnabled()) {
                LOG.error("", ioe);
            }
        }
    }
    final Connectable connectable = context.getConnectable();
    final StandardFlowFileEvent flowFileEvent = new StandardFlowFileEvent(connectable.getIdentifier());
    flowFileEvent.setBytesRead(bytesRead);
    flowFileEvent.setBytesWritten(bytesWritten);
    flowFileEvent.setCounters(immediateCounters);
    // update event repository
    try {
        context.getFlowFileEventRepository().updateRepository(flowFileEvent);
    } catch (final Exception e) {
        LOG.error("Failed to update FlowFileEvent Repository due to " + e);
        if (LOG.isDebugEnabled()) {
            LOG.error("", e);
        }
    }
    acknowledgeRecords();
    resetState();
}

Example 77 with ProcessSession

use of org.apache.nifi.processor.ProcessSession in project nifi by apache.

the class WeakHashMapProcessSessionFactory method terminateActiveSessions.

@Override
public synchronized void terminateActiveSessions() {
    terminated = true;
    for (final ProcessSession session : sessionMap.keySet()) {
        try {
            session.rollback();
        } catch (final TerminatedTaskException tte) {
        // ignore
        }
    }
    sessionMap.clear();
}

Example 78 with ProcessSession

use of org.apache.nifi.processor.ProcessSession in project nifi by apache.

the class QueryCassandra method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile fileToProcess = null;
    if (context.hasIncomingConnection()) {
        fileToProcess = session.get();
        // we know that we should run only if we have a FlowFile.
        if (fileToProcess == null && context.hasNonLoopConnection()) {
            return;
        }
    }
    final ComponentLog logger = getLogger();
    final String selectQuery = context.getProperty(CQL_SELECT_QUERY).evaluateAttributeExpressions(fileToProcess).getValue();
    final long queryTimeout = context.getProperty(QUERY_TIMEOUT).evaluateAttributeExpressions(fileToProcess).asTimePeriod(TimeUnit.MILLISECONDS);
    final String outputFormat = context.getProperty(OUTPUT_FORMAT).getValue();
    final Charset charset = Charset.forName(context.getProperty(CHARSET).evaluateAttributeExpressions(fileToProcess).getValue());
    final StopWatch stopWatch = new StopWatch(true);
    if (fileToProcess == null) {
        fileToProcess = session.create();
    }
    try {
        // The documentation for the driver recommends the session remain open the entire time the processor is running
        // and states that it is thread-safe. This is why connectionSession is not in a try-with-resources.
        final Session connectionSession = cassandraSession.get();
        final ResultSetFuture queryFuture = connectionSession.executeAsync(selectQuery);
        final AtomicLong nrOfRows = new AtomicLong(0L);
        fileToProcess = session.write(fileToProcess, new OutputStreamCallback() {

            @Override
            public void process(final OutputStream out) throws IOException {
                try {
                    logger.debug("Executing CQL query {}", new Object[] { selectQuery });
                    final ResultSet resultSet;
                    if (queryTimeout > 0) {
                        resultSet = queryFuture.getUninterruptibly(queryTimeout, TimeUnit.MILLISECONDS);
                        if (AVRO_FORMAT.equals(outputFormat)) {
                            nrOfRows.set(convertToAvroStream(resultSet, out, queryTimeout, TimeUnit.MILLISECONDS));
                        } else if (JSON_FORMAT.equals(outputFormat)) {
                            nrOfRows.set(convertToJsonStream(resultSet, out, charset, queryTimeout, TimeUnit.MILLISECONDS));
                        }
                    } else {
                        resultSet = queryFuture.getUninterruptibly();
                        if (AVRO_FORMAT.equals(outputFormat)) {
                            nrOfRows.set(convertToAvroStream(resultSet, out, 0, null));
                        } else if (JSON_FORMAT.equals(outputFormat)) {
                            nrOfRows.set(convertToJsonStream(resultSet, out, charset, 0, null));
                        }
                    }
                } catch (final TimeoutException | InterruptedException | ExecutionException e) {
                    throw new ProcessException(e);
                }
            }
        });
        // set attribute how many rows were selected
        fileToProcess = session.putAttribute(fileToProcess, RESULT_ROW_COUNT, String.valueOf(nrOfRows.get()));
        // set mime.type based on output format
        fileToProcess = session.putAttribute(fileToProcess, CoreAttributes.MIME_TYPE.key(), JSON_FORMAT.equals(outputFormat) ? "application/json" : "application/avro-binary");
        logger.info("{} contains {} Avro records; transferring to 'success'", new Object[] { fileToProcess, nrOfRows.get() });
        session.getProvenanceReporter().modifyContent(fileToProcess, "Retrieved " + nrOfRows.get() + " rows", stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(fileToProcess, REL_SUCCESS);
    } catch (final NoHostAvailableException nhae) {
        getLogger().error("No host in the Cassandra cluster can be contacted successfully to execute this query", nhae);
        // Log up to 10 error messages. Otherwise if a 1000-node cluster was specified but there was no connectivity,
        // a thousand error messages would be logged. However we would like information from Cassandra itself, so
        // cap the error limit at 10, format the messages, and don't include the stack trace (it is displayed by the
        // logger message above).
        getLogger().error(nhae.getCustomMessage(10, true, false));
        fileToProcess = session.penalize(fileToProcess);
        session.transfer(fileToProcess, REL_RETRY);
    } catch (final QueryExecutionException qee) {
        logger.error("Cannot execute the query with the requested consistency level successfully", qee);
        fileToProcess = session.penalize(fileToProcess);
        session.transfer(fileToProcess, REL_RETRY);
    } catch (final QueryValidationException qve) {
        if (context.hasIncomingConnection()) {
            logger.error("The CQL query {} is invalid due to syntax error, authorization issue, or another " + "validation problem; routing {} to failure", new Object[] { selectQuery, fileToProcess }, qve);
            fileToProcess = session.penalize(fileToProcess);
            session.transfer(fileToProcess, REL_FAILURE);
        } else {
            // This can happen if any exceptions occur while setting up the connection, statement, etc.
            logger.error("The CQL query {} is invalid due to syntax error, authorization issue, or another " + "validation problem", new Object[] { selectQuery }, qve);
            session.remove(fileToProcess);
            context.yield();
        }
    } catch (final ProcessException e) {
        if (context.hasIncomingConnection()) {
            logger.error("Unable to execute CQL select query {} for {} due to {}; routing to failure", new Object[] { selectQuery, fileToProcess, e });
            fileToProcess = session.penalize(fileToProcess);
            session.transfer(fileToProcess, REL_FAILURE);
        } else {
            logger.error("Unable to execute CQL select query {} due to {}", new Object[] { selectQuery, e });
            session.remove(fileToProcess);
            context.yield();
        }
    }
}

Example 79 with ProcessSession

use of org.apache.nifi.processor.ProcessSession in project nifi by apache.

the class AbstractBinlogEventWriter method writeEvent.

// Default implementation for binlog events
@Override
public long writeEvent(ProcessSession session, String transitUri, T eventInfo, long currentSequenceId, Relationship relationship) {
    FlowFile flowFile = session.create();
    flowFile = session.write(flowFile, (outputStream) -> {
        super.startJson(outputStream, eventInfo);
        writeJson(eventInfo);
        // Nothing in the body
        super.endJson();
    });
    flowFile = session.putAllAttributes(flowFile, getCommonAttributes(currentSequenceId, eventInfo));
    session.transfer(flowFile, relationship);
    session.getProvenanceReporter().receive(flowFile, transitUri);
    return currentSequenceId + 1;
}

Example 80 with ProcessSession

use of org.apache.nifi.processor.ProcessSession in project nifi by apache.

the class AbstractBinlogTableEventWriter method writeEvent.

// Default implementation for table-related binlog events
@Override
public long writeEvent(ProcessSession session, String transitUri, T eventInfo, long currentSequenceId, Relationship relationship) {
    FlowFile flowFile = session.create();
    flowFile = session.write(flowFile, (outputStream) -> {
        super.startJson(outputStream, eventInfo);
        writeJson(eventInfo);
        // Nothing in the body
        super.endJson();
    });
    flowFile = session.putAllAttributes(flowFile, getCommonAttributes(currentSequenceId, eventInfo));
    session.transfer(flowFile, relationship);
    session.getProvenanceReporter().receive(flowFile, transitUri);
    return currentSequenceId + 1;
}