Examples with JmsProcessingSummary org.apache.nifi.processors.standard.util.JmsProcessingSummary used on opensource projects

Example 1 with JmsProcessingSummary

use of org.apache.nifi.processors.standard.util.JmsProcessingSummary in project nifi by apache.

the class JmsConsumer method map2FlowFile.

public static JmsProcessingSummary map2FlowFile(final ProcessContext context, final ProcessSession session, final Message message, final boolean addAttributes, ComponentLog logger) throws Exception {
    // Currently not very useful, because always one Message == one FlowFile
    final AtomicInteger msgsThisFlowFile = new AtomicInteger(1);
    FlowFile flowFile = session.create();
    try {
        // MapMessage is exception, add only name-value pairs to FlowFile attributes
        if (message instanceof MapMessage) {
            MapMessage mapMessage = (MapMessage) message;
            flowFile = session.putAllAttributes(flowFile, createMapMessageValues(mapMessage));
        } else {
            // all other message types, write Message body to FlowFile content
            flowFile = session.write(flowFile, new OutputStreamCallback() {

                @Override
                public void process(final OutputStream rawOut) throws IOException {
                    try (final OutputStream out = new BufferedOutputStream(rawOut, 65536)) {
                        final byte[] messageBody = JmsFactory.createByteArray(message);
                        out.write(messageBody);
                    } catch (final JMSException e) {
                        throw new ProcessException("Failed to receive JMS Message due to " + e.getMessage(), e);
                    }
                }
            });
        }
        if (addAttributes) {
            flowFile = session.putAllAttributes(flowFile, JmsFactory.createAttributeMap(message));
        }
        session.getProvenanceReporter().receive(flowFile, context.getProperty(URL).getValue());
        session.transfer(flowFile, REL_SUCCESS);
        logger.info("Created {} from {} messages received from JMS Server and transferred to 'success'", new Object[] { flowFile, msgsThisFlowFile.get() });
        return new JmsProcessingSummary(flowFile.getSize(), message, flowFile);
    } catch (Exception e) {
        session.remove(flowFile);
        throw e;
    }
}

Example 2 with JmsProcessingSummary

use of org.apache.nifi.processors.standard.util.JmsProcessingSummary in project nifi by apache.

the class TestJmsConsumer method testMap2FlowFileBytesMessage.

/**
 * Test BytesMessage to FlowFile conversion
 *
 * @throws java.lang.Exception ex
 */
@Test
public void testMap2FlowFileBytesMessage() throws Exception {
    TestRunner runner = TestRunners.newTestRunner(GetJMSQueue.class);
    BytesMessage bytesMessage = new ActiveMQBytesMessage();
    String sourceString = "Apache NiFi is an easy to use, powerful, and reliable system to process and distribute data.!";
    byte[] payload = sourceString.getBytes("UTF-8");
    bytesMessage.writeBytes(payload);
    bytesMessage.reset();
    ProcessContext context = runner.getProcessContext();
    ProcessSession session = runner.getProcessSessionFactory().createSession();
    ProcessorInitializationContext pic = new MockProcessorInitializationContext(runner.getProcessor(), (MockProcessContext) runner.getProcessContext());
    JmsProcessingSummary summary = JmsConsumer.map2FlowFile(context, session, bytesMessage, true, pic.getLogger());
    assertEquals("BytesMessage content length should equal to FlowFile content size", payload.length, summary.getLastFlowFile().getSize());
    final byte[] buffer = new byte[payload.length];
    runner.clearTransferState();
    session.read(summary.getLastFlowFile(), new InputStreamCallback() {

        @Override
        public void process(InputStream in) throws IOException {
            StreamUtils.fillBuffer(in, buffer, false);
        }
    });
    String contentString = new String(buffer, "UTF-8");
    assertEquals("", sourceString, contentString);
}

Example 3 with JmsProcessingSummary

use of org.apache.nifi.processors.standard.util.JmsProcessingSummary in project nifi by apache.

the class TestJmsConsumer method testMap2FlowFileMapMessage.

/**
 * Test MapMessage to FlowFile conversion
 *
 * @throws java.lang.Exception ex
 */
@Test
public void testMap2FlowFileMapMessage() throws Exception {
    TestRunner runner = TestRunners.newTestRunner(GetJMSQueue.class);
    MapMessage mapMessage = createMapMessage();
    ProcessContext context = runner.getProcessContext();
    ProcessSession session = runner.getProcessSessionFactory().createSession();
    ProcessorInitializationContext pic = new MockProcessorInitializationContext(runner.getProcessor(), (MockProcessContext) runner.getProcessContext());
    JmsProcessingSummary summary = JmsConsumer.map2FlowFile(context, session, mapMessage, true, pic.getLogger());
    assertEquals("MapMessage should not create FlowFile content", 0, summary.getBytesReceived());
    Map<String, String> attributes = summary.getLastFlowFile().getAttributes();
    assertEquals("", "Arnold", attributes.get(JmsConsumer.MAP_MESSAGE_PREFIX + "name"));
    assertEquals("", "97", attributes.get(JmsConsumer.MAP_MESSAGE_PREFIX + "age"));
    assertEquals("", "89686.564", attributes.get(JmsConsumer.MAP_MESSAGE_PREFIX + "xyz"));
    assertEquals("", "true", attributes.get(JmsConsumer.MAP_MESSAGE_PREFIX + "good"));
}

Example 4 with JmsProcessingSummary

use of org.apache.nifi.processors.standard.util.JmsProcessingSummary in project nifi by apache.

the class JmsConsumer method consume.

public void consume(final ProcessContext context, final ProcessSession session, final WrappedMessageConsumer wrappedConsumer) throws ProcessException {
    final ComponentLog logger = getLogger();
    final MessageConsumer consumer = wrappedConsumer.getConsumer();
    final boolean clientAcknowledge = context.getProperty(ACKNOWLEDGEMENT_MODE).getValue().equalsIgnoreCase(ACK_MODE_CLIENT);
    final long timeout = context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS);
    final boolean addAttributes = context.getProperty(JMS_PROPS_TO_ATTRIBUTES).asBoolean();
    final int batchSize = context.getProperty(BATCH_SIZE).asInteger();
    final JmsProcessingSummary processingSummary = new JmsProcessingSummary();
    final StopWatch stopWatch = new StopWatch(true);
    for (int i = 0; i < batchSize; i++) {
        final Message message;
        try {
            // all messages that are immediately available.
            if (processingSummary.getMessagesReceived() == 0) {
                message = consumer.receive(timeout);
            } else {
                message = consumer.receiveNoWait();
            }
        } catch (final JMSException e) {
            logger.error("Failed to receive JMS Message due to {}", e);
            wrappedConsumer.close(logger);
            break;
        }
        if (message == null) {
            // if no messages, we're done
            break;
        }
        try {
            processingSummary.add(map2FlowFile(context, session, message, addAttributes, logger));
        } catch (Exception e) {
            logger.error("Failed to receive JMS Message due to {}", e);
            wrappedConsumer.close(logger);
            break;
        }
    }
    if (processingSummary.getFlowFilesCreated() == 0) {
        context.yield();
        return;
    }
    session.commit();
    stopWatch.stop();
    if (processingSummary.getFlowFilesCreated() > 0) {
        final float secs = (stopWatch.getDuration(TimeUnit.MILLISECONDS) / 1000F);
        float messagesPerSec = (processingSummary.getMessagesReceived()) / secs;
        final String dataRate = stopWatch.calculateDataRate(processingSummary.getBytesReceived());
        logger.info("Received {} messages in {} milliseconds, at a rate of {} messages/sec or {}", new Object[] { processingSummary.getMessagesReceived(), stopWatch.getDuration(TimeUnit.MILLISECONDS), messagesPerSec, dataRate });
    }
    // if we need to acknowledge the messages, do so now.
    final Message lastMessage = processingSummary.getLastMessageReceived();
    if (clientAcknowledge && lastMessage != null) {
        try {
            // acknowledge all received messages by acknowledging only the last.
            lastMessage.acknowledge();
        } catch (final JMSException e) {
            logger.error("Failed to acknowledge {} JMS Message(s). This may result in duplicate messages. Reason for failure: {}", new Object[] { processingSummary.getMessagesReceived(), e });
        }
    }
}

Example 5 with JmsProcessingSummary

use of org.apache.nifi.processors.standard.util.JmsProcessingSummary in project nifi by apache.

the class TestJmsConsumer method testMap2FlowFileTextMessage.

@Test
public void testMap2FlowFileTextMessage() throws Exception {
    TestRunner runner = TestRunners.newTestRunner(GetJMSQueue.class);
    TextMessage textMessage = new ActiveMQTextMessage();
    String payload = "Hello world!";
    textMessage.setText(payload);
    ProcessContext context = runner.getProcessContext();
    ProcessSession session = runner.getProcessSessionFactory().createSession();
    ProcessorInitializationContext pic = new MockProcessorInitializationContext(runner.getProcessor(), (MockProcessContext) runner.getProcessContext());
    JmsProcessingSummary summary = JmsConsumer.map2FlowFile(context, session, textMessage, true, pic.getLogger());
    assertEquals("TextMessage content length should equal to FlowFile content size", payload.length(), summary.getLastFlowFile().getSize());
    final byte[] buffer = new byte[payload.length()];
    runner.clearTransferState();
    session.read(summary.getLastFlowFile(), new InputStreamCallback() {

        @Override
        public void process(InputStream in) throws IOException {
            StreamUtils.fillBuffer(in, buffer, false);
        }
    });
    String contentString = new String(buffer, "UTF-8");
    assertEquals("", payload, contentString);
}