Examples with FlowFile org.apache.nifi.flowfile.FlowFile used on opensource projects

Example 91 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class ParseEvtxTest method testGetBasenameExtension.

@Test
public void testGetBasenameExtension() {
    String basename = "basename.wrongextension";
    FlowFile flowFile = mock(FlowFile.class);
    ComponentLog componentLog = mock(ComponentLog.class);
    when(flowFile.getAttribute(CoreAttributes.FILENAME.key())).thenReturn(basename);
    assertEquals(basename, parseEvtx.getBasename(flowFile, componentLog));
    verify(componentLog).warn(anyString(), isA(Object[].class));
}

Example 92 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class ParseEvtxTest method testProcessFileGranularity.

@Test
public void testProcessFileGranularity() throws IOException, MalformedChunkException, XMLStreamException {
    String basename = "basename";
    int chunkNum = 5;
    int offset = 10001;
    byte[] badChunk = { 8 };
    RootNodeHandler rootNodeHandler = mock(RootNodeHandler.class);
    when(rootNodeHandlerFactory.create(out)).thenReturn(rootNodeHandler);
    ChunkHeader chunkHeader1 = mock(ChunkHeader.class);
    ChunkHeader chunkHeader2 = mock(ChunkHeader.class);
    Record record1 = mock(Record.class);
    Record record2 = mock(Record.class);
    Record record3 = mock(Record.class);
    RootNode rootNode1 = mock(RootNode.class);
    RootNode rootNode2 = mock(RootNode.class);
    RootNode rootNode3 = mock(RootNode.class);
    ProcessSession session = mock(ProcessSession.class);
    FlowFile flowFile = mock(FlowFile.class);
    AtomicReference<Exception> reference = new AtomicReference<>();
    MalformedChunkException malformedChunkException = new MalformedChunkException("Test", null, offset, chunkNum, badChunk);
    when(record1.getRootNode()).thenReturn(rootNode1);
    when(record2.getRootNode()).thenReturn(rootNode2);
    when(record3.getRootNode()).thenReturn(rootNode3);
    when(fileHeader.hasNext()).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false);
    when(fileHeader.next()).thenThrow(malformedChunkException).thenReturn(chunkHeader1).thenReturn(chunkHeader2).thenReturn(null);
    when(chunkHeader1.hasNext()).thenReturn(true).thenReturn(false);
    when(chunkHeader1.next()).thenReturn(record1).thenReturn(null);
    when(chunkHeader2.hasNext()).thenReturn(true).thenReturn(true).thenReturn(false);
    when(chunkHeader2.next()).thenReturn(record2).thenReturn(record3).thenReturn(null);
    parseEvtx.processFileGranularity(session, componentLog, flowFile, basename, reference, in, out);
    verify(malformedChunkHandler).handle(flowFile, session, parseEvtx.getName(basename, chunkNum, null, ParseEvtx.EVTX_EXTENSION), badChunk);
    verify(rootNodeHandler).handle(rootNode1);
    verify(rootNodeHandler).handle(rootNode2);
    verify(rootNodeHandler).handle(rootNode3);
    verify(rootNodeHandler).close();
}

Example 93 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class ResultProcessorTest method testProcessResultFileSuccess.

@Test
public void testProcessResultFileSuccess() {
    ProcessSession processSession = mock(ProcessSession.class);
    ComponentLog componentLog = mock(ComponentLog.class);
    FlowFile flowFile = mock(FlowFile.class);
    Exception exception = null;
    String name = "basename";
    when(processSession.putAttribute(eq(flowFile), anyString(), anyString())).thenReturn(flowFile);
    resultProcessor.process(processSession, componentLog, flowFile, exception, name);
    verify(processSession).putAttribute(flowFile, CoreAttributes.FILENAME.key(), name);
    verify(processSession).putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), MediaType.APPLICATION_XML_UTF_8.toString());
    verify(processSession).transfer(flowFile, successRelationship);
    verifyNoMoreInteractions(componentLog);
}

Example 94 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class BinFiles method processBins.

private int processBins(final ProcessContext context) {
    final ComponentLog logger = getLogger();
    int processedBins = 0;
    Bin bin;
    while ((bin = readyBins.poll()) != null) {
        boolean binAlreadyCommitted;
        try {
            binAlreadyCommitted = this.processBin(bin, context);
        } catch (final ProcessException e) {
            logger.error("Failed to process bundle of {} files due to {}", new Object[] { bin.getContents().size(), e });
            final ProcessSession binSession = bin.getSession();
            for (final FlowFile flowFile : bin.getContents()) {
                binSession.transfer(flowFile, REL_FAILURE);
            }
            binSession.commit();
            continue;
        } catch (final Exception e) {
            logger.error("Failed to process bundle of {} files due to {}; rolling back sessions", new Object[] { bin.getContents().size(), e });
            bin.getSession().rollback();
            continue;
        }
        // If this bin's session has been committed, move on.
        if (!binAlreadyCommitted) {
            final ProcessSession binSession = bin.getSession();
            binSession.transfer(bin.getContents(), REL_ORIGINAL);
            binSession.commit();
        }
        processedBins++;
    }
    return processedBins;
}

Example 95 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class BinFiles method binFlowFiles.

private int binFlowFiles(final ProcessContext context, final ProcessSessionFactory sessionFactory) {
    int flowFilesBinned = 0;
    while (binManager.getBinCount() <= context.getProperty(MAX_BIN_COUNT).asInteger()) {
        if (!isScheduled()) {
            break;
        }
        final ProcessSession session = sessionFactory.createSession();
        final List<FlowFile> flowFiles = session.get(1000);
        if (flowFiles.isEmpty()) {
            break;
        }
        final Map<String, List<FlowFile>> flowFileGroups = new HashMap<>();
        for (FlowFile flowFile : flowFiles) {
            flowFile = this.preprocessFlowFile(context, session, flowFile);
            try {
                final String groupingIdentifier = getGroupId(context, flowFile, session);
                flowFileGroups.computeIfAbsent(groupingIdentifier, id -> new ArrayList<>()).add(flowFile);
            } catch (final Exception e) {
                getLogger().error("Could not determine which Bin to add {} to; will route to failure", new Object[] { flowFile }, e);
                session.transfer(flowFile, REL_FAILURE);
                continue;
            }
        }
        for (final Map.Entry<String, List<FlowFile>> entry : flowFileGroups.entrySet()) {
            final Set<FlowFile> unbinned = binManager.offer(entry.getKey(), entry.getValue(), session, sessionFactory);
            for (final FlowFile flowFile : unbinned) {
                Bin bin = new Bin(sessionFactory.createSession(), 0, Long.MAX_VALUE, 0, Integer.MAX_VALUE, null);
                bin.offer(flowFile, session);
                this.readyBins.add(bin);
            }
            flowFilesBinned += entry.getValue().size();
        }
    }
    return flowFilesBinned;
}