use of org.apache.nifi.controller.repository.io.FlowFileAccessOutputStream in project nifi by apache.
the class StandardProcessSession method write.
@Override
public FlowFile write(FlowFile source, final StreamCallback writer) {
verifyTaskActive();
source = validateRecordState(source);
final StandardRepositoryRecord record = records.get(source);
final ContentClaim currClaim = record.getCurrentClaim();
long writtenToFlowFile = 0L;
ContentClaim newClaim = null;
try {
newClaim = claimCache.getContentClaim();
claimLog.debug("Creating ContentClaim {} for 'write' for {}", newClaim, source);
ensureNotAppending(newClaim);
if (currClaim != null) {
claimCache.flush(currClaim.getResourceClaim());
}
try (final InputStream is = getInputStream(source, currClaim, record.getCurrentClaimOffset(), true);
final InputStream limitedIn = new LimitedInputStream(is, source.getSize());
final InputStream disableOnCloseIn = new DisableOnCloseInputStream(limitedIn);
final ByteCountingInputStream countingIn = new ByteCountingInputStream(disableOnCloseIn, bytesRead);
final OutputStream os = claimCache.write(newClaim);
final OutputStream disableOnCloseOut = new DisableOnCloseOutputStream(os);
final ByteCountingOutputStream countingOut = new ByteCountingOutputStream(disableOnCloseOut)) {
writeRecursionSet.add(source);
// We want to differentiate between IOExceptions thrown by the repository and IOExceptions thrown from
// Processor code. As a result, as have the FlowFileAccessInputStream that catches IOException from the repository
// and translates into either FlowFileAccessException or ContentNotFoundException. We keep track of any
// ContentNotFoundException because if it is thrown, the Processor code may catch it and do something else with it
// but in reality, if it is thrown, we want to know about it and handle it, even if the Processor code catches it.
final FlowFileAccessInputStream ffais = new FlowFileAccessInputStream(countingIn, source, currClaim);
final FlowFileAccessOutputStream ffaos = new FlowFileAccessOutputStream(countingOut, source);
boolean cnfeThrown = false;
try {
writer.process(createTaskTerminationStream(ffais), createTaskTerminationStream(ffaos));
} catch (final ContentNotFoundException cnfe) {
cnfeThrown = true;
throw cnfe;
} finally {
writtenToFlowFile = countingOut.getBytesWritten();
this.bytesWritten += writtenToFlowFile;
this.bytesRead += countingIn.getBytesRead();
writeRecursionSet.remove(source);
// if cnfeThrown is true, we don't need to re-thrown the Exception; it will propagate.
if (!cnfeThrown && ffais.getContentNotFoundException() != null) {
throw ffais.getContentNotFoundException();
}
}
}
} catch (final ContentNotFoundException nfe) {
destroyContent(newClaim);
handleContentNotFound(nfe, record);
} catch (final IOException ioe) {
destroyContent(newClaim);
throw new ProcessException("IOException thrown from " + connectableDescription + ": " + ioe.toString(), ioe);
} catch (final FlowFileAccessException ffae) {
destroyContent(newClaim);
throw ffae;
} catch (final Throwable t) {
destroyContent(newClaim);
throw t;
}
removeTemporaryClaim(record);
final FlowFileRecord newFile = new StandardFlowFileRecord.Builder().fromFlowFile(record.getCurrent()).contentClaim(newClaim).contentClaimOffset(Math.max(0L, newClaim.getLength() - writtenToFlowFile)).size(writtenToFlowFile).build();
record.setWorking(newFile);
return newFile;
}
use of org.apache.nifi.controller.repository.io.FlowFileAccessOutputStream in project nifi by apache.
the class StandardProcessSession method append.
@Override
public FlowFile append(FlowFile source, final OutputStreamCallback writer) {
verifyTaskActive();
source = validateRecordState(source);
final StandardRepositoryRecord record = records.get(source);
long newSize = 0L;
// Get the current Content Claim from the record and see if we already have
// an OutputStream that we can append to.
final ContentClaim oldClaim = record.getCurrentClaim();
ByteCountingOutputStream outStream = oldClaim == null ? null : appendableStreams.get(oldClaim);
long originalByteWrittenCount = 0;
ContentClaim newClaim = null;
try {
if (outStream == null) {
claimCache.flush(oldClaim);
try (final InputStream oldClaimIn = context.getContentRepository().read(oldClaim)) {
newClaim = context.getContentRepository().create(context.getConnectable().isLossTolerant());
claimLog.debug("Creating ContentClaim {} for 'append' for {}", newClaim, source);
final OutputStream rawOutStream = context.getContentRepository().write(newClaim);
final OutputStream bufferedOutStream = new BufferedOutputStream(rawOutStream);
outStream = new ByteCountingOutputStream(bufferedOutStream);
originalByteWrittenCount = 0;
appendableStreams.put(newClaim, outStream);
// We need to copy all of the data from the old claim to the new claim
StreamUtils.copy(oldClaimIn, outStream);
// wrap our OutputStreams so that the processor cannot close it
try (final OutputStream disableOnClose = new DisableOnCloseOutputStream(outStream)) {
writeRecursionSet.add(source);
writer.process(new FlowFileAccessOutputStream(disableOnClose, source));
} finally {
writeRecursionSet.remove(source);
}
}
} else {
newClaim = oldClaim;
originalByteWrittenCount = outStream.getBytesWritten();
// wrap our OutputStreams so that the processor cannot close it
try (final OutputStream disableOnClose = new DisableOnCloseOutputStream(outStream);
final OutputStream flowFileAccessOutStream = new FlowFileAccessOutputStream(disableOnClose, source)) {
writeRecursionSet.add(source);
writer.process(flowFileAccessOutStream);
} finally {
writeRecursionSet.remove(source);
}
}
// update the newSize to reflect the number of bytes written
newSize = outStream.getBytesWritten();
} catch (final ContentNotFoundException nfe) {
// need to reset write claim before we can remove the claim
resetWriteClaims();
// it here also, we would be decrementing the claimant count twice!
if (newClaim != oldClaim) {
destroyContent(newClaim);
}
handleContentNotFound(nfe, record);
} catch (final IOException ioe) {
// need to reset write claim before we can remove the claim
resetWriteClaims();
// See above explanation for why this is done only if newClaim != oldClaim
if (newClaim != oldClaim) {
destroyContent(newClaim);
}
throw new ProcessException("IOException thrown from " + connectableDescription + ": " + ioe.toString(), ioe);
} catch (final Throwable t) {
// need to reset write claim before we can remove the claim
resetWriteClaims();
// See above explanation for why this is done only if newClaim != oldClaim
if (newClaim != oldClaim) {
destroyContent(newClaim);
}
throw t;
} finally {
if (outStream != null) {
final long bytesWrittenThisIteration = outStream.getBytesWritten() - originalByteWrittenCount;
bytesWritten += bytesWrittenThisIteration;
}
}
// the FlowFile was written to, via #write() and then append() was called.
if (newClaim != oldClaim) {
removeTemporaryClaim(record);
}
final FlowFileRecord newFile = new StandardFlowFileRecord.Builder().fromFlowFile(record.getCurrent()).contentClaim(newClaim).contentClaimOffset(0).size(newSize).build();
record.setWorking(newFile);
return newFile;
}
use of org.apache.nifi.controller.repository.io.FlowFileAccessOutputStream in project nifi by apache.
the class StandardProcessSession method write.
@Override
public FlowFile write(FlowFile source, final OutputStreamCallback writer) {
verifyTaskActive();
source = validateRecordState(source);
final StandardRepositoryRecord record = records.get(source);
long writtenToFlowFile = 0L;
ContentClaim newClaim = null;
try {
newClaim = claimCache.getContentClaim();
claimLog.debug("Creating ContentClaim {} for 'write' for {}", newClaim, source);
ensureNotAppending(newClaim);
try (final OutputStream stream = claimCache.write(newClaim);
final OutputStream disableOnClose = new DisableOnCloseOutputStream(stream);
final ByteCountingOutputStream countingOut = new ByteCountingOutputStream(disableOnClose)) {
try {
writeRecursionSet.add(source);
final OutputStream ffaos = new FlowFileAccessOutputStream(countingOut, source);
writer.process(createTaskTerminationStream(ffaos));
} finally {
writtenToFlowFile = countingOut.getBytesWritten();
bytesWritten += countingOut.getBytesWritten();
}
} finally {
writeRecursionSet.remove(source);
}
} catch (final ContentNotFoundException nfe) {
// need to reset write claim before we can remove the claim
resetWriteClaims();
destroyContent(newClaim);
handleContentNotFound(nfe, record);
} catch (final FlowFileAccessException ffae) {
// need to reset write claim before we can remove the claim
resetWriteClaims();
destroyContent(newClaim);
throw ffae;
} catch (final IOException ioe) {
// need to reset write claim before we can remove the claim
resetWriteClaims();
destroyContent(newClaim);
throw new ProcessException("IOException thrown from " + connectableDescription + ": " + ioe.toString(), ioe);
} catch (final Throwable t) {
// need to reset write claim before we can remove the claim
resetWriteClaims();
destroyContent(newClaim);
throw t;
}
removeTemporaryClaim(record);
final FlowFileRecord newFile = new StandardFlowFileRecord.Builder().fromFlowFile(record.getCurrent()).contentClaim(newClaim).contentClaimOffset(Math.max(0, newClaim.getLength() - writtenToFlowFile)).size(writtenToFlowFile).build();
record.setWorking(newFile);
return newFile;
}
Aggregations