Search in sources :

Example 1 with ChunkHeader

use of org.apache.nifi.processors.evtx.parser.ChunkHeader in project nifi by apache.

the class ParseEvtx method processRecordGranularity.

protected void processRecordGranularity(ProcessSession session, ComponentLog componentLog, FlowFile flowFile, String basename, InputStream in) throws IOException {
    FileHeader fileHeader = fileHeaderFactory.create(in, componentLog);
    while (fileHeader.hasNext()) {
        try {
            ChunkHeader chunkHeader = fileHeader.next();
            while (chunkHeader.hasNext()) {
                FlowFile updated = session.create(flowFile);
                AtomicReference<Exception> exceptionReference = new AtomicReference<>(null);
                try {
                    Record record = chunkHeader.next();
                    updated = session.write(updated, out -> {
                        try (RootNodeHandler rootNodeHandler = rootNodeHandlerFactory.create(out)) {
                            try {
                                rootNodeHandler.handle(record.getRootNode());
                            } catch (IOException e) {
                                exceptionReference.set(e);
                            }
                        } catch (IOException e) {
                            exceptionReference.set(e);
                        }
                    });
                    resultProcessor.process(session, componentLog, updated, exceptionReference.get(), getName(basename, chunkHeader.getChunkNumber(), record.getRecordNum(), XML_EXTENSION));
                } catch (Exception e) {
                    exceptionReference.set(e);
                    session.remove(updated);
                }
                if (exceptionReference.get() != null) {
                    malformedChunkHandler.handle(flowFile, session, getName(basename, chunkHeader.getChunkNumber(), null, EVTX_EXTENSION), chunkHeader.getBinaryReader().getBytes());
                }
            }
        } catch (MalformedChunkException e) {
            malformedChunkHandler.handle(flowFile, session, getName(basename, e.getChunkNum(), null, EVTX_EXTENSION), e.getBadChunk());
        }
    }
}
Also used : Arrays(java.util.Arrays) CapabilityDescription(org.apache.nifi.annotation.documentation.CapabilityDescription) FileHeaderFactory(org.apache.nifi.processors.evtx.parser.FileHeaderFactory) FileHeader(org.apache.nifi.processors.evtx.parser.FileHeader) EventDriven(org.apache.nifi.annotation.behavior.EventDriven) ComponentLog(org.apache.nifi.logging.ComponentLog) SideEffectFree(org.apache.nifi.annotation.behavior.SideEffectFree) AtomicReference(java.util.concurrent.atomic.AtomicReference) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) ProcessException(org.apache.nifi.processor.exception.ProcessException) HashSet(java.util.HashSet) WritesAttributes(org.apache.nifi.annotation.behavior.WritesAttributes) Relationship(org.apache.nifi.processor.Relationship) Requirement(org.apache.nifi.annotation.behavior.InputRequirement.Requirement) ReadsAttributes(org.apache.nifi.annotation.behavior.ReadsAttributes) Record(org.apache.nifi.processors.evtx.parser.Record) ChunkHeader(org.apache.nifi.processors.evtx.parser.ChunkHeader) OutputStream(java.io.OutputStream) MalformedChunkException(org.apache.nifi.processors.evtx.parser.MalformedChunkException) FlowFile(org.apache.nifi.flowfile.FlowFile) ProcessContext(org.apache.nifi.processor.ProcessContext) Set(java.util.Set) ProcessSession(org.apache.nifi.processor.ProcessSession) IOException(java.io.IOException) WritesAttribute(org.apache.nifi.annotation.behavior.WritesAttribute) InputRequirement(org.apache.nifi.annotation.behavior.InputRequirement) List(java.util.List) SupportsBatching(org.apache.nifi.annotation.behavior.SupportsBatching) AbstractProcessor(org.apache.nifi.processor.AbstractProcessor) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Tags(org.apache.nifi.annotation.documentation.Tags) CoreAttributes(org.apache.nifi.flowfile.attributes.CoreAttributes) Collections(java.util.Collections) ReadsAttribute(org.apache.nifi.annotation.behavior.ReadsAttribute) InputStream(java.io.InputStream) FlowFile(org.apache.nifi.flowfile.FlowFile) ChunkHeader(org.apache.nifi.processors.evtx.parser.ChunkHeader) AtomicReference(java.util.concurrent.atomic.AtomicReference) Record(org.apache.nifi.processors.evtx.parser.Record) IOException(java.io.IOException) MalformedChunkException(org.apache.nifi.processors.evtx.parser.MalformedChunkException) FileHeader(org.apache.nifi.processors.evtx.parser.FileHeader) ProcessException(org.apache.nifi.processor.exception.ProcessException) MalformedChunkException(org.apache.nifi.processors.evtx.parser.MalformedChunkException) IOException(java.io.IOException)

Example 2 with ChunkHeader

use of org.apache.nifi.processors.evtx.parser.ChunkHeader in project nifi by apache.

the class ParseEvtxTest method testProcessChunkGranularity.

@Test
public void testProcessChunkGranularity() throws IOException, MalformedChunkException, XMLStreamException {
    String basename = "basename";
    int chunkNum = 5;
    int offset = 10001;
    byte[] badChunk = { 8 };
    RootNodeHandler rootNodeHandler1 = mock(RootNodeHandler.class);
    RootNodeHandler rootNodeHandler2 = mock(RootNodeHandler.class);
    OutputStream out2 = mock(OutputStream.class);
    when(rootNodeHandlerFactory.create(out)).thenReturn(rootNodeHandler1);
    when(rootNodeHandlerFactory.create(out2)).thenReturn(rootNodeHandler2);
    ChunkHeader chunkHeader1 = mock(ChunkHeader.class);
    ChunkHeader chunkHeader2 = mock(ChunkHeader.class);
    Record record1 = mock(Record.class);
    Record record2 = mock(Record.class);
    Record record3 = mock(Record.class);
    RootNode rootNode1 = mock(RootNode.class);
    RootNode rootNode2 = mock(RootNode.class);
    RootNode rootNode3 = mock(RootNode.class);
    ProcessSession session = mock(ProcessSession.class);
    FlowFile flowFile = mock(FlowFile.class);
    FlowFile created1 = mock(FlowFile.class);
    FlowFile updated1 = mock(FlowFile.class);
    FlowFile created2 = mock(FlowFile.class);
    FlowFile updated2 = mock(FlowFile.class);
    MalformedChunkException malformedChunkException = new MalformedChunkException("Test", null, offset, chunkNum, badChunk);
    when(session.create(flowFile)).thenReturn(created1).thenReturn(created2).thenReturn(null);
    when(session.write(eq(created1), any(OutputStreamCallback.class))).thenAnswer(invocation -> {
        ((OutputStreamCallback) invocation.getArguments()[1]).process(out);
        return updated1;
    });
    when(session.write(eq(created2), any(OutputStreamCallback.class))).thenAnswer(invocation -> {
        ((OutputStreamCallback) invocation.getArguments()[1]).process(out2);
        return updated2;
    });
    when(record1.getRootNode()).thenReturn(rootNode1);
    when(record2.getRootNode()).thenReturn(rootNode2);
    when(record3.getRootNode()).thenReturn(rootNode3);
    when(fileHeader.hasNext()).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false);
    when(fileHeader.next()).thenThrow(malformedChunkException).thenReturn(chunkHeader1).thenReturn(chunkHeader2).thenReturn(null);
    when(chunkHeader1.hasNext()).thenReturn(true).thenReturn(false);
    when(chunkHeader1.next()).thenReturn(record1).thenReturn(null);
    when(chunkHeader2.hasNext()).thenReturn(true).thenReturn(true).thenReturn(false);
    when(chunkHeader2.next()).thenReturn(record2).thenReturn(record3).thenReturn(null);
    parseEvtx.processChunkGranularity(session, componentLog, flowFile, basename, in);
    verify(malformedChunkHandler).handle(flowFile, session, parseEvtx.getName(basename, chunkNum, null, ParseEvtx.EVTX_EXTENSION), badChunk);
    verify(rootNodeHandler1).handle(rootNode1);
    verify(rootNodeHandler1).close();
    verify(rootNodeHandler2).handle(rootNode2);
    verify(rootNodeHandler2).handle(rootNode3);
    verify(rootNodeHandler2).close();
}
Also used : ProcessSession(org.apache.nifi.processor.ProcessSession) RootNode(org.apache.nifi.processors.evtx.parser.bxml.RootNode) FlowFile(org.apache.nifi.flowfile.FlowFile) MockFlowFile(org.apache.nifi.util.MockFlowFile) OutputStream(java.io.OutputStream) ChunkHeader(org.apache.nifi.processors.evtx.parser.ChunkHeader) Record(org.apache.nifi.processors.evtx.parser.Record) Mockito.anyString(org.mockito.Mockito.anyString) MalformedChunkException(org.apache.nifi.processors.evtx.parser.MalformedChunkException) OutputStreamCallback(org.apache.nifi.processor.io.OutputStreamCallback) Test(org.junit.Test)

Example 3 with ChunkHeader

use of org.apache.nifi.processors.evtx.parser.ChunkHeader in project nifi by apache.

the class ParseEvtxTest method testProcessFileGranularity.

@Test
public void testProcessFileGranularity() throws IOException, MalformedChunkException, XMLStreamException {
    String basename = "basename";
    int chunkNum = 5;
    int offset = 10001;
    byte[] badChunk = { 8 };
    RootNodeHandler rootNodeHandler = mock(RootNodeHandler.class);
    when(rootNodeHandlerFactory.create(out)).thenReturn(rootNodeHandler);
    ChunkHeader chunkHeader1 = mock(ChunkHeader.class);
    ChunkHeader chunkHeader2 = mock(ChunkHeader.class);
    Record record1 = mock(Record.class);
    Record record2 = mock(Record.class);
    Record record3 = mock(Record.class);
    RootNode rootNode1 = mock(RootNode.class);
    RootNode rootNode2 = mock(RootNode.class);
    RootNode rootNode3 = mock(RootNode.class);
    ProcessSession session = mock(ProcessSession.class);
    FlowFile flowFile = mock(FlowFile.class);
    AtomicReference<Exception> reference = new AtomicReference<>();
    MalformedChunkException malformedChunkException = new MalformedChunkException("Test", null, offset, chunkNum, badChunk);
    when(record1.getRootNode()).thenReturn(rootNode1);
    when(record2.getRootNode()).thenReturn(rootNode2);
    when(record3.getRootNode()).thenReturn(rootNode3);
    when(fileHeader.hasNext()).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false);
    when(fileHeader.next()).thenThrow(malformedChunkException).thenReturn(chunkHeader1).thenReturn(chunkHeader2).thenReturn(null);
    when(chunkHeader1.hasNext()).thenReturn(true).thenReturn(false);
    when(chunkHeader1.next()).thenReturn(record1).thenReturn(null);
    when(chunkHeader2.hasNext()).thenReturn(true).thenReturn(true).thenReturn(false);
    when(chunkHeader2.next()).thenReturn(record2).thenReturn(record3).thenReturn(null);
    parseEvtx.processFileGranularity(session, componentLog, flowFile, basename, reference, in, out);
    verify(malformedChunkHandler).handle(flowFile, session, parseEvtx.getName(basename, chunkNum, null, ParseEvtx.EVTX_EXTENSION), badChunk);
    verify(rootNodeHandler).handle(rootNode1);
    verify(rootNodeHandler).handle(rootNode2);
    verify(rootNodeHandler).handle(rootNode3);
    verify(rootNodeHandler).close();
}
Also used : ProcessSession(org.apache.nifi.processor.ProcessSession) RootNode(org.apache.nifi.processors.evtx.parser.bxml.RootNode) FlowFile(org.apache.nifi.flowfile.FlowFile) MockFlowFile(org.apache.nifi.util.MockFlowFile) AtomicReference(java.util.concurrent.atomic.AtomicReference) Mockito.anyString(org.mockito.Mockito.anyString) MalformedChunkException(org.apache.nifi.processors.evtx.parser.MalformedChunkException) XMLStreamException(javax.xml.stream.XMLStreamException) MalformedChunkException(org.apache.nifi.processors.evtx.parser.MalformedChunkException) IOException(java.io.IOException) ParserConfigurationException(javax.xml.parsers.ParserConfigurationException) SAXException(org.xml.sax.SAXException) ChunkHeader(org.apache.nifi.processors.evtx.parser.ChunkHeader) Record(org.apache.nifi.processors.evtx.parser.Record) Test(org.junit.Test)

Example 4 with ChunkHeader

use of org.apache.nifi.processors.evtx.parser.ChunkHeader in project nifi by apache.

the class ParseEvtx method processChunkGranularity.

protected void processChunkGranularity(ProcessSession session, ComponentLog componentLog, FlowFile flowFile, String basename, InputStream in) throws IOException {
    FileHeader fileHeader = fileHeaderFactory.create(in, componentLog);
    while (fileHeader.hasNext()) {
        try {
            ChunkHeader chunkHeader = fileHeader.next();
            FlowFile updated = session.create(flowFile);
            AtomicReference<Exception> exceptionReference = new AtomicReference<>(null);
            updated = session.write(updated, out -> {
                try (RootNodeHandler rootNodeHandler = rootNodeHandlerFactory.create(out)) {
                    while (chunkHeader.hasNext()) {
                        try {
                            rootNodeHandler.handle(chunkHeader.next().getRootNode());
                        } catch (IOException e) {
                            exceptionReference.set(e);
                            break;
                        }
                    }
                } catch (IOException e) {
                    exceptionReference.set(e);
                }
            });
            Exception exception = exceptionReference.get();
            resultProcessor.process(session, componentLog, updated, exception, getName(basename, chunkHeader.getChunkNumber(), null, XML_EXTENSION));
            if (exception != null) {
                malformedChunkHandler.handle(flowFile, session, getName(basename, chunkHeader.getChunkNumber(), null, EVTX_EXTENSION), chunkHeader.getBinaryReader().getBytes());
            }
        } catch (MalformedChunkException e) {
            malformedChunkHandler.handle(flowFile, session, getName(basename, e.getChunkNum(), null, EVTX_EXTENSION), e.getBadChunk());
        }
    }
}
Also used : Arrays(java.util.Arrays) CapabilityDescription(org.apache.nifi.annotation.documentation.CapabilityDescription) FileHeaderFactory(org.apache.nifi.processors.evtx.parser.FileHeaderFactory) FileHeader(org.apache.nifi.processors.evtx.parser.FileHeader) EventDriven(org.apache.nifi.annotation.behavior.EventDriven) ComponentLog(org.apache.nifi.logging.ComponentLog) SideEffectFree(org.apache.nifi.annotation.behavior.SideEffectFree) AtomicReference(java.util.concurrent.atomic.AtomicReference) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) ProcessException(org.apache.nifi.processor.exception.ProcessException) HashSet(java.util.HashSet) WritesAttributes(org.apache.nifi.annotation.behavior.WritesAttributes) Relationship(org.apache.nifi.processor.Relationship) Requirement(org.apache.nifi.annotation.behavior.InputRequirement.Requirement) ReadsAttributes(org.apache.nifi.annotation.behavior.ReadsAttributes) Record(org.apache.nifi.processors.evtx.parser.Record) ChunkHeader(org.apache.nifi.processors.evtx.parser.ChunkHeader) OutputStream(java.io.OutputStream) MalformedChunkException(org.apache.nifi.processors.evtx.parser.MalformedChunkException) FlowFile(org.apache.nifi.flowfile.FlowFile) ProcessContext(org.apache.nifi.processor.ProcessContext) Set(java.util.Set) ProcessSession(org.apache.nifi.processor.ProcessSession) IOException(java.io.IOException) WritesAttribute(org.apache.nifi.annotation.behavior.WritesAttribute) InputRequirement(org.apache.nifi.annotation.behavior.InputRequirement) List(java.util.List) SupportsBatching(org.apache.nifi.annotation.behavior.SupportsBatching) AbstractProcessor(org.apache.nifi.processor.AbstractProcessor) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Tags(org.apache.nifi.annotation.documentation.Tags) CoreAttributes(org.apache.nifi.flowfile.attributes.CoreAttributes) Collections(java.util.Collections) ReadsAttribute(org.apache.nifi.annotation.behavior.ReadsAttribute) InputStream(java.io.InputStream) FlowFile(org.apache.nifi.flowfile.FlowFile) ChunkHeader(org.apache.nifi.processors.evtx.parser.ChunkHeader) AtomicReference(java.util.concurrent.atomic.AtomicReference) IOException(java.io.IOException) MalformedChunkException(org.apache.nifi.processors.evtx.parser.MalformedChunkException) FileHeader(org.apache.nifi.processors.evtx.parser.FileHeader) ProcessException(org.apache.nifi.processor.exception.ProcessException) MalformedChunkException(org.apache.nifi.processors.evtx.parser.MalformedChunkException) IOException(java.io.IOException)

Example 5 with ChunkHeader

use of org.apache.nifi.processors.evtx.parser.ChunkHeader in project nifi by apache.

the class ParseEvtxTest method testProcess1RecordGranularity.

@Test
public void testProcess1RecordGranularity() throws IOException, MalformedChunkException, XMLStreamException {
    String basename = "basename";
    int chunkNum = 5;
    int offset = 10001;
    byte[] badChunk = { 8 };
    RootNodeHandler rootNodeHandler1 = mock(RootNodeHandler.class);
    RootNodeHandler rootNodeHandler2 = mock(RootNodeHandler.class);
    RootNodeHandler rootNodeHandler3 = mock(RootNodeHandler.class);
    OutputStream out2 = mock(OutputStream.class);
    OutputStream out3 = mock(OutputStream.class);
    when(rootNodeHandlerFactory.create(out)).thenReturn(rootNodeHandler1);
    when(rootNodeHandlerFactory.create(out2)).thenReturn(rootNodeHandler2);
    when(rootNodeHandlerFactory.create(out3)).thenReturn(rootNodeHandler3);
    ChunkHeader chunkHeader1 = mock(ChunkHeader.class);
    ChunkHeader chunkHeader2 = mock(ChunkHeader.class);
    Record record1 = mock(Record.class);
    Record record2 = mock(Record.class);
    Record record3 = mock(Record.class);
    RootNode rootNode1 = mock(RootNode.class);
    RootNode rootNode2 = mock(RootNode.class);
    RootNode rootNode3 = mock(RootNode.class);
    ProcessSession session = mock(ProcessSession.class);
    FlowFile flowFile = mock(FlowFile.class);
    FlowFile created1 = mock(FlowFile.class);
    FlowFile updated1 = mock(FlowFile.class);
    FlowFile created2 = mock(FlowFile.class);
    FlowFile updated2 = mock(FlowFile.class);
    FlowFile created3 = mock(FlowFile.class);
    FlowFile updated3 = mock(FlowFile.class);
    MalformedChunkException malformedChunkException = new MalformedChunkException("Test", null, offset, chunkNum, badChunk);
    when(session.create(flowFile)).thenReturn(created1).thenReturn(created2).thenReturn(created3).thenReturn(null);
    when(session.write(eq(created1), any(OutputStreamCallback.class))).thenAnswer(invocation -> {
        ((OutputStreamCallback) invocation.getArguments()[1]).process(out);
        return updated1;
    });
    when(session.write(eq(created2), any(OutputStreamCallback.class))).thenAnswer(invocation -> {
        ((OutputStreamCallback) invocation.getArguments()[1]).process(out2);
        return updated2;
    });
    when(session.write(eq(created3), any(OutputStreamCallback.class))).thenAnswer(invocation -> {
        ((OutputStreamCallback) invocation.getArguments()[1]).process(out3);
        return updated3;
    });
    when(record1.getRootNode()).thenReturn(rootNode1);
    when(record2.getRootNode()).thenReturn(rootNode2);
    when(record3.getRootNode()).thenReturn(rootNode3);
    when(fileHeader.hasNext()).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false);
    when(fileHeader.next()).thenThrow(malformedChunkException).thenReturn(chunkHeader1).thenReturn(chunkHeader2).thenReturn(null);
    when(chunkHeader1.hasNext()).thenReturn(true).thenReturn(false);
    when(chunkHeader1.next()).thenReturn(record1).thenReturn(null);
    when(chunkHeader2.hasNext()).thenReturn(true).thenReturn(true).thenReturn(false);
    when(chunkHeader2.next()).thenReturn(record2).thenReturn(record3).thenReturn(null);
    parseEvtx.processRecordGranularity(session, componentLog, flowFile, basename, in);
    verify(malformedChunkHandler).handle(flowFile, session, parseEvtx.getName(basename, chunkNum, null, ParseEvtx.EVTX_EXTENSION), badChunk);
    verify(rootNodeHandler1).handle(rootNode1);
    verify(rootNodeHandler1).close();
    verify(rootNodeHandler2).handle(rootNode2);
    verify(rootNodeHandler2).close();
    verify(rootNodeHandler3).handle(rootNode3);
    verify(rootNodeHandler3).close();
}
Also used : ProcessSession(org.apache.nifi.processor.ProcessSession) RootNode(org.apache.nifi.processors.evtx.parser.bxml.RootNode) FlowFile(org.apache.nifi.flowfile.FlowFile) MockFlowFile(org.apache.nifi.util.MockFlowFile) OutputStream(java.io.OutputStream) ChunkHeader(org.apache.nifi.processors.evtx.parser.ChunkHeader) Record(org.apache.nifi.processors.evtx.parser.Record) Mockito.anyString(org.mockito.Mockito.anyString) MalformedChunkException(org.apache.nifi.processors.evtx.parser.MalformedChunkException) OutputStreamCallback(org.apache.nifi.processor.io.OutputStreamCallback) Test(org.junit.Test)

Aggregations

FlowFile (org.apache.nifi.flowfile.FlowFile)5 ProcessSession (org.apache.nifi.processor.ProcessSession)5 ChunkHeader (org.apache.nifi.processors.evtx.parser.ChunkHeader)5 MalformedChunkException (org.apache.nifi.processors.evtx.parser.MalformedChunkException)5 Record (org.apache.nifi.processors.evtx.parser.Record)5 OutputStream (java.io.OutputStream)4 IOException (java.io.IOException)3 AtomicReference (java.util.concurrent.atomic.AtomicReference)3 RootNode (org.apache.nifi.processors.evtx.parser.bxml.RootNode)3 MockFlowFile (org.apache.nifi.util.MockFlowFile)3 Test (org.junit.Test)3 Mockito.anyString (org.mockito.Mockito.anyString)3 VisibleForTesting (com.google.common.annotations.VisibleForTesting)2 InputStream (java.io.InputStream)2 Arrays (java.util.Arrays)2 Collections (java.util.Collections)2 HashSet (java.util.HashSet)2 List (java.util.List)2 Set (java.util.Set)2 EventDriven (org.apache.nifi.annotation.behavior.EventDriven)2