Examples with StripHeaderSupport com.thinkbiganalytics.ingest.StripHeaderSupport used on opensource projects

Example 1 with StripHeaderSupport

use of com.thinkbiganalytics.ingest.StripHeaderSupport in project kylo by Teradata.

the class StripHeader method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final StripHeaderSupport headerSupport = new StripHeaderSupport();
    final FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final boolean isEnabled = context.getProperty(ENABLED).evaluateAttributeExpressions(flowFile).asBoolean();
    final int headerCount = context.getProperty(HEADER_LINE_COUNT).evaluateAttributeExpressions(flowFile).asInteger();
    // Empty files and no work to do will simply pass along content
    if (!isEnabled || headerCount == 0 || flowFile.getSize() == 0L) {
        final FlowFile contentFlowFile = session.clone(flowFile);
        session.transfer(contentFlowFile, REL_CONTENT);
        session.transfer(flowFile, REL_ORIGINAL);
        return;
    }
    final MutableLong headerBoundaryInBytes = new MutableLong(-1);
    session.read(flowFile, false, rawIn -> {
        try {
            // Identify the byte boundary of the header
            long bytes = headerSupport.findHeaderBoundary(headerCount, rawIn);
            headerBoundaryInBytes.setValue(bytes);
            if (bytes < 0) {
                getLog().error("Unable to strip header {} expecting at least {} lines in file", new Object[] { flowFile, headerCount });
            }
        } catch (IOException e) {
            getLog().error("Unable to strip header {} due to {}; routing to failure", new Object[] { flowFile, e.getLocalizedMessage() }, e);
        }
    });
    long headerBytes = headerBoundaryInBytes.getValue();
    if (headerBytes < 0) {
        session.transfer(flowFile, REL_FAILURE);
    } else {
        // Transfer header
        final FlowFile headerFlowFile = session.clone(flowFile, 0, headerBytes);
        session.transfer(headerFlowFile, REL_HEADER);
        // Transfer content
        long contentBytes = flowFile.getSize() - headerBytes;
        final FlowFile contentFlowFile = session.clone(flowFile, headerBytes, contentBytes);
        session.transfer(contentFlowFile, REL_CONTENT);
        session.transfer(flowFile, REL_ORIGINAL);
    }
}