Examples with MergeRecord org.apache.nifi.processors.standard.MergeRecord used on opensource projects

Example 1 with MergeRecord

use of org.apache.nifi.processors.standard.MergeRecord in project nifi by apache.

the class RecordBin method offer.

public boolean offer(final FlowFile flowFile, final RecordReader recordReader, final ProcessSession flowFileSession, final boolean block) throws IOException, MalformedRecordException, SchemaNotFoundException {
    if (isComplete()) {
        logger.debug("RecordBin.offer for id={} returning false because {} is complete", new Object[] { flowFile.getId(), this });
        return false;
    }
    final boolean locked;
    if (block) {
        writeLock.lock();
        locked = true;
    } else {
        locked = writeLock.tryLock();
    }
    if (!locked) {
        logger.debug("RecordBin.offer for id={} returning false because failed to get lock for {}", new Object[] { flowFile.getId(), this });
        return false;
    }
    boolean flowFileMigrated = false;
    try {
        if (isComplete()) {
            logger.debug("RecordBin.offer for id={} returning false because {} is complete", new Object[] { flowFile.getId(), this });
            return false;
        }
        logger.debug("Migrating id={} to {}", new Object[] { flowFile.getId(), this });
        Record record;
        while ((record = recordReader.nextRecord()) != null) {
            if (recordWriter == null) {
                final OutputStream rawOut = session.write(merged);
                logger.debug("Created OutputStream using session {} for {}", new Object[] { session, this });
                this.out = new ByteCountingOutputStream(rawOut);
                recordWriter = writerFactory.createWriter(logger, record.getSchema(), out);
                recordWriter.beginRecordSet();
            }
            recordWriter.write(record);
            recordCount++;
        }
        // This will be closed by the MergeRecord class anyway but we have to close it
        // here because it needs to be closed before we are able to migrate the FlowFile
        // to a new Session.
        recordReader.close();
        flowFileSession.migrate(this.session, Collections.singleton(flowFile));
        flowFileMigrated = true;
        this.flowFiles.add(flowFile);
        if (isFull()) {
            logger.debug(this + " is now full. Completing bin.");
            complete("Bin is full");
        } else if (isOlderThan(thresholds.getMaxBinMillis(), TimeUnit.MILLISECONDS)) {
            logger.debug(this + " is now expired. Completing bin.");
            complete("Bin is older than " + thresholds.getMaxBinAge());
        }
        return true;
    } catch (final Exception e) {
        logger.error("Failed to create merged FlowFile from " + (flowFiles.size() + 1) + " input FlowFiles; routing originals to failure", e);
        try {
            // This will be closed by the MergeRecord class anyway but we have to close it
            // here because it needs to be closed before we are able to migrate the FlowFile
            // to a new Session.
            recordReader.close();
            if (recordWriter != null) {
                recordWriter.close();
            }
            if (this.out != null) {
                this.out.close();
            }
            if (!flowFileMigrated) {
                flowFileSession.migrate(this.session, Collections.singleton(flowFile));
                this.flowFiles.add(flowFile);
            }
        } finally {
            complete = true;
            session.remove(merged);
            session.transfer(flowFiles, MergeRecord.REL_FAILURE);
            session.commit();
        }
        return true;
    } finally {
        writeLock.unlock();
    }
}