Examples with AttributeStrategy org.apache.nifi.processors.standard.merge.AttributeStrategy used on opensource projects

Example 1 with AttributeStrategy

use of org.apache.nifi.processors.standard.merge.AttributeStrategy in project nifi by apache.

the class MergeContent method processBin.

@Override
protected boolean processBin(final Bin bin, final ProcessContext context) throws ProcessException {
    final String mergeFormat = context.getProperty(MERGE_FORMAT).getValue();
    MergeBin merger;
    switch(mergeFormat) {
        case MERGE_FORMAT_TAR_VALUE:
            merger = new TarMerge();
            break;
        case MERGE_FORMAT_ZIP_VALUE:
            merger = new ZipMerge(context.getProperty(COMPRESSION_LEVEL).asInteger());
            break;
        case MERGE_FORMAT_FLOWFILE_STREAM_V3_VALUE:
            merger = new FlowFileStreamMerger(new FlowFilePackagerV3(), "application/flowfile-v3");
            break;
        case MERGE_FORMAT_FLOWFILE_STREAM_V2_VALUE:
            merger = new FlowFileStreamMerger(new FlowFilePackagerV2(), "application/flowfile-v2");
            break;
        case MERGE_FORMAT_FLOWFILE_TAR_V1_VALUE:
            merger = new FlowFileStreamMerger(new FlowFilePackagerV1(), "application/flowfile-v1");
            break;
        case MERGE_FORMAT_CONCAT_VALUE:
            merger = new BinaryConcatenationMerge();
            break;
        case MERGE_FORMAT_AVRO_VALUE:
            merger = new AvroMerge();
            break;
        default:
            throw new AssertionError();
    }
    final AttributeStrategy attributeStrategy = AttributeStrategyUtil.strategyFor(context);
    final List<FlowFile> contents = bin.getContents();
    final ProcessSession binSession = bin.getSession();
    if (MERGE_STRATEGY_DEFRAGMENT.equals(context.getProperty(MERGE_STRATEGY).getValue())) {
        final String error = getDefragmentValidationError(bin.getContents());
        // Fail the flow files and commit them
        if (error != null) {
            final String binDescription = contents.size() <= 10 ? contents.toString() : contents.size() + " FlowFiles";
            getLogger().error(error + "; routing {} to failure", new Object[] { binDescription });
            binSession.transfer(contents, REL_FAILURE);
            binSession.commit();
            return true;
        }
        Collections.sort(contents, new FragmentComparator());
    }
    FlowFile bundle = merger.merge(bin, context);
    // keep the filename, as it is added to the bundle.
    final String filename = bundle.getAttribute(CoreAttributes.FILENAME.key());
    // merge all of the attributes
    final Map<String, String> bundleAttributes = attributeStrategy.getMergedAttributes(contents);
    bundleAttributes.put(CoreAttributes.MIME_TYPE.key(), merger.getMergedContentType());
    // restore the filename of the bundle
    bundleAttributes.put(CoreAttributes.FILENAME.key(), filename);
    bundleAttributes.put(MERGE_COUNT_ATTRIBUTE, Integer.toString(contents.size()));
    bundleAttributes.put(MERGE_BIN_AGE_ATTRIBUTE, Long.toString(bin.getBinAge()));
    bundle = binSession.putAllAttributes(bundle, bundleAttributes);
    final String inputDescription = contents.size() < 10 ? contents.toString() : contents.size() + " FlowFiles";
    getLogger().info("Merged {} into {}", new Object[] { inputDescription, bundle });
    binSession.transfer(bundle, REL_MERGED);
    for (final FlowFile unmerged : merger.getUnmergedFlowFiles()) {
        final FlowFile unmergedCopy = binSession.clone(unmerged);
        binSession.transfer(unmergedCopy, REL_FAILURE);
    }
    // We haven't committed anything, parent will take care of it
    return false;
}