Examples with TransformErrorMeta org.apache.hop.pipeline.transform.TransformErrorMeta used on opensource projects

Example 1 with TransformErrorMeta

use of org.apache.hop.pipeline.transform.TransformErrorMeta in project hop by apache.

the class KafkaConsumerInputMeta method getFields.

@Override
public void getFields(IRowMeta rowMeta, String origin, IRowMeta[] info, TransformMeta nextTransform, IVariables variables, IHopMetadataProvider metadataProvider) throws HopTransformException {
    try {
        PipelineMeta pipelineMeta = mappingMetaRetriever.get(this, metadataProvider, variables);
        if (!StringUtils.isEmpty(getSubTransform())) {
            String realSubTransformName = variables.resolve(getSubTransform());
            rowMeta.addRowMeta(pipelineMeta.getPrevTransformFields(variables, realSubTransformName));
            pipelineMeta.getTransforms().stream().filter(transformMeta -> transformMeta.getName().equals(realSubTransformName)).findFirst().ifPresent(transformMeta -> {
                try {
                    transformMeta.getTransform().getFields(rowMeta, origin, info, nextTransform, variables, metadataProvider);
                } catch (HopTransformException e) {
                    throw new RuntimeException(e);
                }
            });
        }
        // Check if we get called from error path and only in that case, show fields that will dump the
        // record coming from the kafka queue.
        TransformErrorMeta transformErrorMeta = getParentTransformMeta().getTransformErrorMeta();
        if (transformErrorMeta != null && transformErrorMeta.getTargetTransform().getName().equals(nextTransform.getName())) {
            rowMeta.addValueMeta(createValueMetaString(getKeyField().getOutputName()));
            rowMeta.addValueMeta(createValueMetaString(getMessageField().getOutputName()));
            rowMeta.addValueMeta(createValueMetaString(getTopicField().getOutputName()));
            rowMeta.addValueMeta(createValueMetaInteger(getPartitionField().getOutputName()));
            rowMeta.addValueMeta(createValueMetaInteger(getOffsetField().getOutputName()));
            rowMeta.addValueMeta(createValueMetaInteger(getTimestampField().getOutputName()));
        }
    } catch (HopException e) {
        getLog().logDebug("could not get fields, probable AEL");
        rowMeta.addRowMeta(getRowMeta(origin, variables));
    }
}

Example 2 with TransformErrorMeta

use of org.apache.hop.pipeline.transform.TransformErrorMeta in project hop by apache.

the class HopGuiPipelineHopDelegate method delHop.

public void delHop(PipelineMeta pipelineMeta, PipelineHopMeta pipelineHopMeta) {
    int index = pipelineMeta.indexOfPipelineHop(pipelineHopMeta);
    hopGui.undoDelegate.addUndoDelete(pipelineMeta, new Object[] { (PipelineHopMeta) pipelineHopMeta.clone() }, new int[] { index });
    pipelineMeta.removePipelineHop(index);
    TransformMeta fromTransformMeta = pipelineHopMeta.getFromTransform();
    TransformMeta beforeFrom = (TransformMeta) fromTransformMeta.clone();
    int indexFrom = pipelineMeta.indexOfTransform(fromTransformMeta);
    TransformMeta toTransformMeta = pipelineHopMeta.getToTransform();
    TransformMeta beforeTo = (TransformMeta) toTransformMeta.clone();
    int indexTo = pipelineMeta.indexOfTransform(toTransformMeta);
    boolean transformFromNeedAddUndoChange = fromTransformMeta.getTransform().cleanAfterHopFromRemove(pipelineHopMeta.getToTransform());
    boolean transformToNeedAddUndoChange = toTransformMeta.getTransform().cleanAfterHopToRemove(fromTransformMeta);
    // 
    if (pipelineHopMeta.getFromTransform().isDoingErrorHandling()) {
        TransformErrorMeta transformErrorMeta = fromTransformMeta.getTransformErrorMeta();
        // 
        if (transformErrorMeta.getTargetTransform() != null && transformErrorMeta.getTargetTransform().equals(pipelineHopMeta.getToTransform())) {
            // Only if the target transform is where the error handling is going to...
            // 
            transformErrorMeta.setEnabled(false);
            transformFromNeedAddUndoChange = true;
        }
    }
    if (transformFromNeedAddUndoChange) {
        hopGui.undoDelegate.addUndoChange(pipelineMeta, new Object[] { beforeFrom }, new Object[] { fromTransformMeta }, new int[] { indexFrom });
    }
    if (transformToNeedAddUndoChange) {
        hopGui.undoDelegate.addUndoChange(pipelineMeta, new Object[] { beforeTo }, new Object[] { toTransformMeta }, new int[] { indexTo });
    }
    pipelineGraph.redraw();
}

Example 3 with TransformErrorMeta

use of org.apache.hop.pipeline.transform.TransformErrorMeta in project hop by apache.

the class PipelineTestFactory method generateTestTransformationError.

public static PipelineMeta generateTestTransformationError(IVariables parent, ITransformMeta oneMeta, String oneTransformName) {
    PipelineMeta previewMeta = new PipelineMeta();
    if (parent == null) {
        parent = new Variables();
    }
    // First the injector transform...
    TransformMeta zero = getInjectorTransformMeta();
    previewMeta.addTransform(zero);
    // Then the middle transform to test...
    // 
    TransformMeta one = new TransformMeta(registry.getPluginId(TransformPluginType.class, oneMeta), oneTransformName, oneMeta);
    one.setLocation(150, 50);
    // one.setDraw( true );
    previewMeta.addTransform(one);
    // Then we add the dummy transform to read the results from
    TransformMeta two = getReadTransformMeta();
    previewMeta.addTransform(two);
    // error handling transform
    TransformMeta err = getReadTransformMeta(ERROR_TRANSFORMNAME);
    previewMeta.addTransform(err);
    // Add the hops between the 3 transforms.
    PipelineHopMeta zeroOne = new PipelineHopMeta(zero, one);
    previewMeta.addPipelineHop(zeroOne);
    PipelineHopMeta oneTwo = new PipelineHopMeta(one, two);
    previewMeta.addPipelineHop(oneTwo);
    TransformErrorMeta errMeta = new TransformErrorMeta(one, err);
    errMeta.setEnabled(true);
    errMeta.setNrErrorsValuename(NUMBER_ERRORS_FIELD);
    errMeta.setErrorDescriptionsValuename(ERROR_DESC_FIELD);
    errMeta.setErrorFieldsValuename(ERROR_FIELD_VALUE);
    errMeta.setErrorCodesValuename(ERROR_CODE_VALUE);
    one.setTransformErrorMeta(errMeta);
    PipelineHopMeta oneErr = new PipelineHopMeta(one, err);
    previewMeta.addPipelineHop(oneErr);
    return previewMeta;
}

Example 4 with TransformErrorMeta

use of org.apache.hop.pipeline.transform.TransformErrorMeta in project hop by apache.

the class HopGuiPipelineClipboardDelegate method pasteXml.

public void pasteXml(PipelineMeta pipelineMeta, String clipcontent, Point loc) {
    try {
        Document doc = XmlHandler.loadXmlString(clipcontent);
        Node pipelineNode = XmlHandler.getSubNode(doc, XML_TAG_PIPELINE_TRANSFORMS);
        // De-select all, re-select pasted transforms...
        pipelineMeta.unselectAll();
        Node transformsNode = XmlHandler.getSubNode(pipelineNode, "transforms");
        int nr = XmlHandler.countNodes(transformsNode, "transform");
        if (log.isDebug()) {
            // "I found "+nr+" transforms to paste on location: "
            log.logDebug(BaseMessages.getString(PKG, "HopGui.Log.FoundTransforms", "" + nr) + loc);
        }
        TransformMeta[] transforms = new TransformMeta[nr];
        ArrayList<String> transformOldNames = new ArrayList<>(nr);
        Point min = new Point(99999999, 99999999);
        // Load the transforms...
        for (int i = 0; i < nr; i++) {
            Node transformNode = XmlHandler.getSubNodeByNr(transformsNode, "transform", i);
            transforms[i] = new TransformMeta(transformNode, hopGui.getMetadataProvider());
            if (loc != null) {
                Point p = transforms[i].getLocation();
                if (min.x > p.x) {
                    min.x = p.x;
                }
                if (min.y > p.y) {
                    min.y = p.y;
                }
            }
        }
        // Load the hops...
        Node hopsNode = XmlHandler.getSubNode(pipelineNode, "order");
        nr = XmlHandler.countNodes(hopsNode, "hop");
        if (log.isDebug()) {
            // "I found "+nr+" hops to paste."
            log.logDebug(BaseMessages.getString(PKG, "HopGui.Log.FoundHops", "" + nr));
        }
        PipelineHopMeta[] hops = new PipelineHopMeta[nr];
        for (int i = 0; i < nr; i++) {
            Node hopNode = XmlHandler.getSubNodeByNr(hopsNode, "hop", i);
            hops[i] = new PipelineHopMeta(hopNode, Arrays.asList(transforms));
        }
        // This is the offset:
        Point offset = new Point(loc.x - min.x, loc.y - min.y);
        // Undo/redo object positions...
        int[] position = new int[transforms.length];
        for (int i = 0; i < transforms.length; i++) {
            Point p = transforms[i].getLocation();
            String name = transforms[i].getName();
            PropsUi.setLocation(transforms[i], p.x + offset.x, p.y + offset.y);
            // Check the name, find alternative...
            transformOldNames.add(name);
            transforms[i].setName(pipelineMeta.getAlternativeTransformName(name));
            pipelineMeta.addTransform(transforms[i]);
            position[i] = pipelineMeta.indexOfTransform(transforms[i]);
            transforms[i].setSelected(true);
        }
        // Add the hops too...
        for (PipelineHopMeta hop : hops) {
            pipelineMeta.addPipelineHop(hop);
        }
        // Load the notes...
        Node notesNode = XmlHandler.getSubNode(pipelineNode, "notepads");
        nr = XmlHandler.countNodes(notesNode, "notepad");
        if (log.isDebug()) {
            // "I found "+nr+" notepads to paste."
            log.logDebug(BaseMessages.getString(PKG, "HopGui.Log.FoundNotepads", "" + nr));
        }
        NotePadMeta[] notes = new NotePadMeta[nr];
        for (int i = 0; i < notes.length; i++) {
            Node noteNode = XmlHandler.getSubNodeByNr(notesNode, "notepad", i);
            notes[i] = new NotePadMeta(noteNode);
            Point p = notes[i].getLocation();
            PropsUi.setLocation(notes[i], p.x + offset.x, p.y + offset.y);
            pipelineMeta.addNote(notes[i]);
            notes[i].setSelected(true);
        }
        // Set the source and target transforms ...
        for (TransformMeta transform : transforms) {
            ITransformMeta smi = transform.getTransform();
            smi.searchInfoAndTargetTransforms(pipelineMeta.getTransforms());
        }
        // Set the error handling hops
        Node errorHandlingNode = XmlHandler.getSubNode(pipelineNode, PipelineMeta.XML_TAG_TRANSFORM_ERROR_HANDLING);
        int nrErrorHandlers = XmlHandler.countNodes(errorHandlingNode, TransformErrorMeta.XML_ERROR_TAG);
        for (int i = 0; i < nrErrorHandlers; i++) {
            Node transformErrorMetaNode = XmlHandler.getSubNodeByNr(errorHandlingNode, TransformErrorMeta.XML_ERROR_TAG, i);
            TransformErrorMeta transformErrorMeta = new TransformErrorMeta(transformErrorMetaNode, pipelineMeta.getTransforms());
            // Handle pasting multiple times, need to update source and target transform names
            int srcTransformPos = transformOldNames.indexOf(transformErrorMeta.getSourceTransform().getName());
            int tgtTransformPos = -1;
            if (transformErrorMeta.getTargetTransform() != null) {
                tgtTransformPos = transformOldNames.indexOf(transformErrorMeta.getTargetTransform().getName());
            }
            TransformMeta sourceTransform = pipelineMeta.findTransform(transforms[srcTransformPos].getName());
            if (sourceTransform != null) {
                sourceTransform.setTransformErrorMeta(transformErrorMeta);
            }
            sourceTransform.setTransformErrorMeta(null);
            if (tgtTransformPos >= 0) {
                sourceTransform.setTransformErrorMeta(transformErrorMeta);
                TransformMeta targetTransform = pipelineMeta.findTransform(transforms[tgtTransformPos].getName());
                transformErrorMeta.setSourceTransform(sourceTransform);
                transformErrorMeta.setTargetTransform(targetTransform);
            }
        }
        // Save undo information too...
        hopGui.undoDelegate.addUndoNew(pipelineMeta, transforms, position, false);
        int[] hopPos = new int[hops.length];
        for (int i = 0; i < hops.length; i++) {
            hopPos[i] = pipelineMeta.indexOfPipelineHop(hops[i]);
        }
        hopGui.undoDelegate.addUndoNew(pipelineMeta, hops, hopPos, true);
        int[] notePos = new int[notes.length];
        for (int i = 0; i < notes.length; i++) {
            notePos[i] = pipelineMeta.indexOfNote(notes[i]);
        }
        hopGui.undoDelegate.addUndoNew(pipelineMeta, notes, notePos, true);
    } catch (HopException e) {
        // "Error pasting transforms...",
        // "I was unable to paste transforms to this pipeline"
        new ErrorDialog(hopGui.getShell(), BaseMessages.getString(PKG, "HopGui.Dialog.UnablePasteTransforms.Title"), BaseMessages.getString(PKG, "HopGui.Dialog.UnablePasteTransforms.Message"), e);
    }
    pipelineGraph.redraw();
}

Example 5 with TransformErrorMeta

use of org.apache.hop.pipeline.transform.TransformErrorMeta in project hop by apache.

the class JsonInputTest method testErrorRedirect.

@Test
public void testErrorRedirect() throws Exception {
    JsonInputField field = new JsonInputField("value");
    field.setPath("$.value");
    field.setType(IValueMeta.TYPE_STRING);
    String input1 = "{{";
    String input2 = "{ \"value\": \"ok\" }";
    JsonInputMeta meta = createSimpleMeta("json", field);
    meta.setRemoveSourceField(true);
    when(helper.transformMeta.isDoingErrorHandling()).thenReturn(true);
    JsonInput jsonInput = createJsonInput("json", meta, new Object[] { input1 }, new Object[] { input2 });
    TransformErrorMeta errMeta = new TransformErrorMeta(helper.transformMeta);
    errMeta.setEnabled(true);
    errMeta.setErrorFieldsValuename("err field");
    when(helper.transformMeta.getTransformErrorMeta()).thenReturn(errMeta);
    final List<Object[]> errorLines = new ArrayList<>();
    jsonInput.addRowListener(new RowComparatorListener(new Object[] { "ok" }) {

        @Override
        public void errorRowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
            errorLines.add(row);
        }
    });
    processRows(jsonInput, 3);
    Assert.assertEquals("fwd error", 1, errorLines.size());
    Assert.assertEquals("input in err line", input1, errorLines.get(0)[0]);
    Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten());
}