Examples with AbstractPipe nl.nn.adapterframework.pipes.AbstractPipe used on opensource projects

Example 1 with AbstractPipe

use of nl.nn.adapterframework.pipes.AbstractPipe in project iaf by ibissource.

the class CheckMessageSizePipeProcessor method checkMessageSize.

private void checkMessageSize(Object message, PipeLine pipeLine, IPipe pipe, boolean input) {
    if (message != null && message instanceof String) {
        int messageLength = message.toString().length();
        if (pipe instanceof AbstractPipe) {
            AbstractPipe aPipe = (AbstractPipe) pipe;
            StatisticsKeeper sizeStat = null;
            if (input) {
                if (aPipe.getInSizeStatDummyObject() != null) {
                    sizeStat = pipeLine.getPipeSizeStatistics(aPipe.getInSizeStatDummyObject());
                }
            } else {
                if (aPipe.getOutSizeStatDummyObject() != null) {
                    sizeStat = pipeLine.getPipeSizeStatistics(aPipe.getOutSizeStatDummyObject());
                }
            }
            if (sizeStat != null) {
                sizeStat.addValue(messageLength);
            }
        }
        if (pipeLine.getMessageSizeWarnNum() >= 0) {
            if (messageLength >= pipeLine.getMessageSizeWarnNum()) {
                String logMessage = "pipe [" + pipe.getName() + "] of adapter [" + pipeLine.getOwner().getName() + "], " + (input ? "input" : "result") + " message size [" + Misc.toFileSize(messageLength) + "] exceeds [" + Misc.toFileSize(pipeLine.getMessageSizeWarnNum()) + "]";
                log.warn(logMessage);
                if (pipe instanceof IExtendedPipe) {
                    IExtendedPipe pe = (IExtendedPipe) pipe;
                    pe.throwEvent(IExtendedPipe.MESSAGE_SIZE_MONITORING_EVENT);
                }
            }
        }
    }
}

Example 2 with AbstractPipe

use of nl.nn.adapterframework.pipes.AbstractPipe in project iaf by ibissource.

the class CorePipeLineProcessor method processPipeLine.

public PipeLineResult processPipeLine(PipeLine pipeLine, String messageId, String message, IPipeLineSession pipeLineSession, String firstPipe) throws PipeRunException {
    // Object is the object that is passed to and returned from Pipes
    Object object = (Object) message;
    PipeRunResult pipeRunResult;
    // the PipeLineResult
    PipeLineResult pipeLineResult = new PipeLineResult();
    if (object == null || (object instanceof String && StringUtils.isEmpty(object.toString()))) {
        if (StringUtils.isNotEmpty(pipeLine.getAdapterToRunBeforeOnEmptyInput())) {
            log.debug("running adapterBeforeOnEmptyInput");
            IAdapter adapter = pipeLine.getAdapter().getConfiguration().getIbisManager().getRegisteredAdapter(pipeLine.getAdapterToRunBeforeOnEmptyInput());
            if (adapter == null) {
                log.warn("adapterToRunBefore with specified name [" + pipeLine.getAdapterToRunBeforeOnEmptyInput() + "] could not be retrieved");
            } else {
                PipeLineResult plr = adapter.processMessage(messageId, message, pipeLineSession);
                if (plr == null || !plr.getState().equals("success")) {
                    throw new PipeRunException(null, "adapterToRunBefore [" + pipeLine.getAdapterToRunBeforeOnEmptyInput() + "] ended with state [" + plr.getState() + "]");
                }
                message = plr.getResult();
                log.debug("input after running adapterBeforeOnEmptyInput [" + message + "]");
                object = (Object) message;
            }
        }
    }
    // ready indicates wether the pipeline processing is complete
    boolean ready = false;
    // get the first pipe to run
    IPipe pipeToRun = pipeLine.getPipe(pipeLine.getFirstPipe());
    boolean inputValidateError = false;
    IPipe inputValidator = pipeLine.getInputValidator();
    if (inputValidator != null) {
        log.debug("validating input");
        PipeRunResult validationResult = pipeProcessor.processPipe(pipeLine, inputValidator, messageId, message, pipeLineSession);
        if (validationResult != null) {
            if (!validationResult.getPipeForward().getName().equals("success")) {
                PipeForward validationForward = validationResult.getPipeForward();
                if (validationForward.getPath() == null) {
                    throw new PipeRunException(pipeToRun, "forward [" + validationForward.getName() + "] of inputValidator has emtpy forward path");
                }
                log.warn("setting first pipe to [" + validationForward.getPath() + "] due to validation fault");
                inputValidateError = true;
                pipeToRun = pipeLine.getPipe(validationForward.getPath());
                if (pipeToRun == null) {
                    throw new PipeRunException(pipeToRun, "forward [" + validationForward.getName() + "], path [" + validationForward.getPath() + "] does not correspond to a pipe");
                }
            }
            Object validatedMessage = validationResult.getResult();
            if (validatedMessage != null) {
                object = validatedMessage;
                message = validatedMessage.toString();
            }
        }
    }
    if (!inputValidateError) {
        IPipe inputWrapper = pipeLine.getInputWrapper();
        if (inputWrapper != null) {
            log.debug("wrapping input");
            PipeRunResult wrapResult = pipeProcessor.processPipe(pipeLine, inputWrapper, messageId, message, pipeLineSession);
            if (wrapResult != null && !wrapResult.getPipeForward().getName().equals("success")) {
                PipeForward wrapForward = wrapResult.getPipeForward();
                if (wrapForward.getPath() == null) {
                    throw new PipeRunException(pipeToRun, "forward [" + wrapForward.getName() + "] of inputWrapper has emtpy forward path");
                }
                log.warn("setting first pipe to [" + wrapForward.getPath() + "] due to wrap fault");
                pipeToRun = pipeLine.getPipe(wrapForward.getPath());
                if (pipeToRun == null) {
                    throw new PipeRunException(pipeToRun, "forward [" + wrapForward.getName() + "], path [" + wrapForward.getPath() + "] does not correspond to a pipe");
                }
            } else {
                message = wrapResult.getResult().toString();
            }
            log.debug("input after wrapping [" + message + "]");
            object = (Object) message;
        }
    }
    pipeLine.getRequestSizeStats().addValue(message.length());
    if (pipeLine.isStoreOriginalMessageWithoutNamespaces()) {
        if (XmlUtils.isWellFormed(message)) {
            String removeNamespaces_xslt = XmlUtils.makeRemoveNamespacesXslt(true, true);
            try {
                String xsltResult = null;
                Transformer transformer = XmlUtils.createTransformer(removeNamespaces_xslt);
                xsltResult = XmlUtils.transformXml(transformer, message);
                pipeLineSession.put("originalMessageWithoutNamespaces", xsltResult);
            } catch (IOException e) {
                throw new PipeRunException(pipeToRun, "cannot retrieve removeNamespaces", e);
            } catch (TransformerConfigurationException te) {
                throw new PipeRunException(pipeToRun, "got error creating transformer from removeNamespaces", te);
            } catch (TransformerException te) {
                throw new PipeRunException(pipeToRun, "got error transforming removeNamespaces", te);
            } catch (DomBuilderException te) {
                throw new PipeRunException(pipeToRun, "caught DomBuilderException", te);
            }
        } else {
            log.warn("original message is not well-formed");
            pipeLineSession.put("originalMessageWithoutNamespaces", message);
        }
    }
    boolean outputValidated = false;
    try {
        while (!ready) {
            pipeRunResult = pipeProcessor.processPipe(pipeLine, pipeToRun, messageId, object, pipeLineSession);
            object = pipeRunResult.getResult();
            if (!(pipeToRun instanceof AbstractPipe)) {
                if (object != null && object instanceof String) {
                    StatisticsKeeper sizeStat = pipeLine.getPipeSizeStatistics(pipeToRun);
                    if (sizeStat != null) {
                        sizeStat.addValue(((String) object).length());
                    }
                }
            }
            PipeForward pipeForward = pipeRunResult.getPipeForward();
            if (pipeForward == null) {
                throw new PipeRunException(pipeToRun, "Pipeline of [" + pipeLine.getOwner().getName() + "] received result from pipe [" + pipeToRun.getName() + "] without a pipeForward");
            }
            // get the next pipe to run
            String nextPath = pipeForward.getPath();
            if ((null == nextPath) || (nextPath.length() == 0)) {
                throw new PipeRunException(pipeToRun, "Pipeline of [" + pipeLine.getOwner().getName() + "] got an path that equals null or has a zero-length value from pipe [" + pipeToRun.getName() + "]. Check the configuration, probably forwards are not defined for this pipe.");
            }
            PipeLineExit plExit = pipeLine.getPipeLineExits().get(nextPath);
            if (null != plExit) {
                boolean outputWrapError = false;
                IPipe outputWrapper = pipeLine.getOutputWrapper();
                if (outputWrapper != null) {
                    log.debug("wrapping PipeLineResult");
                    PipeRunResult wrapResult = pipeProcessor.processPipe(pipeLine, outputWrapper, messageId, object, pipeLineSession);
                    if (wrapResult != null && !wrapResult.getPipeForward().getName().equals("success")) {
                        PipeForward wrapForward = wrapResult.getPipeForward();
                        if (wrapForward.getPath() == null) {
                            throw new PipeRunException(pipeToRun, "forward [" + wrapForward.getName() + "] of outputWrapper has emtpy forward path");
                        }
                        log.warn("setting next pipe to [" + wrapForward.getPath() + "] due to wrap fault");
                        outputWrapError = true;
                        pipeToRun = pipeLine.getPipe(wrapForward.getPath());
                        if (pipeToRun == null) {
                            throw new PipeRunException(pipeToRun, "forward [" + wrapForward.getName() + "], path [" + wrapForward.getPath() + "] does not correspond to a pipe");
                        }
                    } else {
                        log.debug("wrap succeeded");
                        object = wrapResult.getResult();
                    }
                    log.debug("PipeLineResult after wrapping [" + object.toString() + "]");
                }
                if (!outputWrapError) {
                    IPipe outputValidator = pipeLine.getOutputValidator();
                    if ((outputValidator != null) && !outputValidated) {
                        outputValidated = true;
                        log.debug("validating PipeLineResult");
                        PipeRunResult validationResult;
                        validationResult = pipeProcessor.processPipe(pipeLine, outputValidator, messageId, object, pipeLineSession);
                        if (validationResult != null && !validationResult.getPipeForward().getName().equals("success")) {
                            PipeForward validationForward = validationResult.getPipeForward();
                            if (validationForward.getPath() == null) {
                                throw new PipeRunException(pipeToRun, "forward [" + validationForward.getName() + "] of outputValidator has emtpy forward path");
                            }
                            log.warn("setting next pipe to [" + validationForward.getPath() + "] due to validation fault");
                            pipeToRun = pipeLine.getPipe(validationForward.getPath());
                            if (pipeToRun == null) {
                                throw new PipeRunException(pipeToRun, "forward [" + validationForward.getName() + "], path [" + validationForward.getPath() + "] does not correspond to a pipe");
                            }
                        } else {
                            log.debug("validation succeeded");
                            object = validationResult.getResult();
                            ready = true;
                        }
                    } else {
                        ready = true;
                    }
                } else {
                    ready = true;
                }
                if (ready) {
                    String state = plExit.getState();
                    pipeLineResult.setState(state);
                    pipeLineResult.setExitCode(plExit.getExitCode());
                    if (object != null && !plExit.getEmptyResult()) {
                        pipeLineResult.setResult(object.toString());
                    } else {
                        pipeLineResult.setResult(null);
                    }
                    ready = true;
                    if (log.isDebugEnabled()) {
                        // for performance reasons
                        String skString = "";
                        for (Iterator it = pipeLineSession.keySet().iterator(); it.hasNext(); ) {
                            String key = (String) it.next();
                            Object value = pipeLineSession.get(key);
                            skString = skString + "\n " + key + "=[" + value + "]";
                        }
                        log.debug("Available session keys at finishing pipeline of adapter [" + pipeLine.getOwner().getName() + "]:" + skString);
                        log.debug("Pipeline of adapter [" + pipeLine.getOwner().getName() + "] finished processing messageId [" + messageId + "] result: [" + object + "] with exit-state [" + state + "]");
                    }
                }
            } else {
                pipeToRun = pipeLine.getPipe(pipeForward.getPath());
                if (pipeToRun == null) {
                    throw new PipeRunException(null, "Pipeline of adapter [" + pipeLine.getOwner().getName() + "] got an erroneous definition. Pipe to execute [" + pipeForward.getPath() + "] is not defined.");
                }
            }
        }
    } finally {
        for (int i = 0; i < pipeLine.getExitHandlers().size(); i++) {
            IPipeLineExitHandler exitHandler = pipeLine.getExitHandlers().get(i);
            try {
                if (log.isDebugEnabled())
                    log.debug("processing ExitHandler [" + exitHandler.getName() + "]");
                exitHandler.atEndOfPipeLine(messageId, pipeLineResult, pipeLineSession);
            } catch (Throwable t) {
                log.warn("Caught Exception processing ExitHandler [" + exitHandler.getName() + "]", t);
            }
        }
    }
    return pipeLineResult;
}

Example 3 with AbstractPipe

use of nl.nn.adapterframework.pipes.AbstractPipe in project iaf by ibissource.

the class PipeLine method configure.

public void configure(IPipe pipe) throws ConfigurationException {
    try {
        if (pipe instanceof IExtendedPipe) {
            IExtendedPipe epipe = (IExtendedPipe) pipe;
            epipe.configure(this);
            if (epipe.getDurationThreshold() >= 0) {
                epipe.registerEvent(IExtendedPipe.LONG_DURATION_MONITORING_EVENT);
            }
            epipe.registerEvent(IExtendedPipe.PIPE_EXCEPTION_MONITORING_EVENT);
            if (getMessageSizeWarnNum() >= 0) {
                epipe.registerEvent(IExtendedPipe.MESSAGE_SIZE_MONITORING_EVENT);
            }
            if (epipe.hasSizeStatistics()) {
                if (pipe instanceof AbstractPipe) {
                    AbstractPipe aPipe = (AbstractPipe) pipe;
                    if (aPipe.getInSizeStatDummyObject() != null) {
                        pipeSizeStats.put(aPipe.getInSizeStatDummyObject().getName(), new SizeStatisticsKeeper(aPipe.getInSizeStatDummyObject().getName()));
                    }
                    if (aPipe.getOutSizeStatDummyObject() != null) {
                        pipeSizeStats.put(aPipe.getOutSizeStatDummyObject().getName(), new SizeStatisticsKeeper(aPipe.getOutSizeStatDummyObject().getName()));
                    }
                } else {
                    pipeSizeStats.put(pipe.getName(), new SizeStatisticsKeeper(pipe.getName()));
                }
            }
        } else {
            pipe.configure();
        }
        if (pipe instanceof MessageSendingPipe) {
            MessageSendingPipe messageSendingPipe = (MessageSendingPipe) pipe;
            if (messageSendingPipe.getInputValidator() != null) {
                configure(messageSendingPipe.getInputValidator());
            }
            if (messageSendingPipe.getOutputValidator() != null) {
                configure(messageSendingPipe.getOutputValidator());
            }
            if (messageSendingPipe.getInputWrapper() != null) {
                configure(messageSendingPipe.getInputWrapper());
            }
            if (messageSendingPipe.getOutputWrapper() != null) {
                configure(messageSendingPipe.getOutputWrapper());
            }
            if (messageSendingPipe.getMessageLog() != null) {
                pipeStatistics.put(messageSendingPipe.getMessageLog().getName(), new StatisticsKeeper(messageSendingPipe.getMessageLog().getName()));
            }
        }
        pipeStatistics.put(pipe.getName(), new StatisticsKeeper(pipe.getName()));
    // congestionSensors.addSensor(pipe);
    } catch (Throwable t) {
        if (t instanceof ConfigurationException) {
            throw (ConfigurationException) t;
        }
        throw new ConfigurationException("Exception configuring Pipe [" + pipe.getName() + "]", t);
    }
    if (log.isDebugEnabled()) {
        log.debug("Pipeline of [" + owner.getName() + "]: Pipe [" + pipe.getName() + "] successfully configured: [" + pipe.toString() + "]");
    }
}

Example 4 with AbstractPipe

use of nl.nn.adapterframework.pipes.AbstractPipe in project iaf by ibissource.

the class PipeLine method iterateOverStatistics.

@Override
public void iterateOverStatistics(StatisticsKeeperIterationHandler hski, Object data, int action) throws SenderException {
    Object pipeStatsData = hski.openGroup(data, null, "pipeStats");
    handlePipeStat(getInputValidator(), pipeStatistics, pipeStatsData, hski, true, action);
    handlePipeStat(getOutputValidator(), pipeStatistics, pipeStatsData, hski, true, action);
    handlePipeStat(getInputWrapper(), pipeStatistics, pipeStatsData, hski, true, action);
    handlePipeStat(getOutputWrapper(), pipeStatistics, pipeStatsData, hski, true, action);
    for (IPipe pipe : adapter.getPipeLine().getPipes()) {
        handlePipeStat(pipe, pipeStatistics, pipeStatsData, hski, true, action);
        if (pipe instanceof MessageSendingPipe) {
            MessageSendingPipe messageSendingPipe = (MessageSendingPipe) pipe;
            if (messageSendingPipe.getInputValidator() != null) {
                handlePipeStat(messageSendingPipe.getInputValidator(), pipeStatistics, pipeStatsData, hski, true, action);
            }
            if (messageSendingPipe.getOutputValidator() != null) {
                handlePipeStat(messageSendingPipe.getOutputValidator(), pipeStatistics, pipeStatsData, hski, true, action);
            }
            if (messageSendingPipe.getInputWrapper() != null) {
                handlePipeStat(messageSendingPipe.getInputWrapper(), pipeStatistics, pipeStatsData, hski, true, action);
            }
            if (messageSendingPipe.getOutputWrapper() != null) {
                handlePipeStat(messageSendingPipe.getOutputWrapper(), pipeStatistics, pipeStatsData, hski, true, action);
            }
            if (messageSendingPipe.getMessageLog() != null) {
                handlePipeStat(messageSendingPipe.getMessageLog(), pipeStatistics, pipeStatsData, hski, true, action);
            }
        }
    }
    if (pipeWaitingStatistics.size() > 0) {
        Object waitStatsData = hski.openGroup(data, null, "waitStats");
        for (IPipe pipe : adapter.getPipeLine().getPipes()) {
            handlePipeStat(pipe, pipeWaitingStatistics, waitStatsData, hski, false, action);
        }
    }
    hski.closeGroup(pipeStatsData);
    Object sizeStatsData = hski.openGroup(data, null, "sizeStats");
    hski.handleStatisticsKeeper(sizeStatsData, getRequestSizeStats());
    for (IPipe pipe : adapter.getPipeLine().getPipes()) {
        if (pipe instanceof AbstractPipe) {
            AbstractPipe aPipe = (AbstractPipe) pipe;
            if (aPipe.getInSizeStatDummyObject() != null) {
                handlePipeStat(aPipe.getInSizeStatDummyObject(), pipeSizeStats, sizeStatsData, hski, false, action);
            }
            if (aPipe.getOutSizeStatDummyObject() != null) {
                handlePipeStat(aPipe.getOutSizeStatDummyObject(), pipeSizeStats, sizeStatsData, hski, false, action);
            }
        } else {
            handlePipeStat(pipe, pipeSizeStats, sizeStatsData, hski, false, action);
        }
    }
    hski.closeGroup(sizeStatsData);
}

Example 5 with AbstractPipe

use of nl.nn.adapterframework.pipes.AbstractPipe in project iaf by ibissource.

the class MonitoringPipeProcessor method processPipe.

public PipeRunResult processPipe(PipeLine pipeLine, IPipe pipe, String messageId, Object message, IPipeLineSession pipeLineSession) throws PipeRunException {
    PipeRunResult pipeRunResult = null;
    IExtendedPipe pe = null;
    if (pipe instanceof IExtendedPipe) {
        pe = (IExtendedPipe) pipe;
    }
    long pipeStartTime = System.currentTimeMillis();
    if (log.isDebugEnabled()) {
        // for performance reasons
        StringBuffer sb = new StringBuffer();
        String ownerName = pipeLine.getOwner() == null ? "<null>" : pipeLine.getOwner().getName();
        String pipeName = pipe == null ? "<null>" : pipe.getName();
        sb.append("Pipeline of adapter [" + ownerName + "] messageId [" + messageId + "] is about to call pipe [" + pipeName + "]");
        boolean lir = false;
        if (AppConstants.getInstance().getProperty("log.logIntermediaryResults") != null) {
            if (AppConstants.getInstance().getProperty("log.logIntermediaryResults").equalsIgnoreCase("true")) {
                lir = true;
            }
        }
        if (pipe instanceof AbstractPipe) {
            AbstractPipe ap = (AbstractPipe) pipe;
            if (StringUtils.isNotEmpty(ap.getLogIntermediaryResults())) {
                lir = Boolean.valueOf(ap.getLogIntermediaryResults());
            }
        }
        if (lir) {
            sb.append(" current result [" + message + "] ");
        }
        log.debug(sb.toString());
    }
    // start it
    long pipeDuration = -1;
    try {
        pipeRunResult = pipeProcessor.processPipe(pipeLine, pipe, messageId, message, pipeLineSession);
    } catch (PipeRunException pre) {
        if (pe != null) {
            pe.throwEvent(IExtendedPipe.PIPE_EXCEPTION_MONITORING_EVENT);
        }
        throw pre;
    } catch (RuntimeException re) {
        if (pe != null) {
            pe.throwEvent(IExtendedPipe.PIPE_EXCEPTION_MONITORING_EVENT);
        }
        throw new PipeRunException(pipe, "Uncaught runtime exception running pipe '" + (pipe == null ? "null" : pipe.getName()) + "'", re);
    } finally {
        long pipeEndTime = System.currentTimeMillis();
        pipeDuration = pipeEndTime - pipeStartTime;
        StatisticsKeeper sk = pipeLine.getPipeStatistics(pipe);
        if (sk == null) {
            log.warn("Could not get statistics for pipe [+" + pipe.getName() + "]");
        } else {
            sk.addValue(pipeDuration);
        }
        if (pe != null) {
            if (pe.getDurationThreshold() >= 0 && pipeDuration > pe.getDurationThreshold()) {
                durationLog.info("Pipe [" + pe.getName() + "] of [" + pipeLine.getOwner().getName() + "] duration [" + pipeDuration + "] ms exceeds max [" + pe.getDurationThreshold() + "], message [" + message + "]");
                pe.throwEvent(IExtendedPipe.LONG_DURATION_MONITORING_EVENT);
            }
        }
    }
    return pipeRunResult;
}