Examples with PipelineUnitTestSetLocation org.apache.hop.testing.PipelineUnitTestSetLocation used on opensource projects

Example 1 with PipelineUnitTestSetLocation

use of org.apache.hop.testing.PipelineUnitTestSetLocation in project hop by apache.

the class InjectDataSetIntoTransformExtensionPoint method callExtensionPoint.

@Override
public void callExtensionPoint(ILogChannel log, IVariables variables, final IPipelineEngine<PipelineMeta> pipeline) throws HopException {
    if (!(pipeline instanceof LocalPipelineEngine)) {
        throw new HopPluginException("Unit tests can only run using a local pipeline engine type");
    }
    final PipelineMeta pipelineMeta = pipeline.getPipelineMeta();
    boolean dataSetEnabled = "Y".equalsIgnoreCase(pipeline.getVariable(DataSetConst.VAR_RUN_UNIT_TEST));
    if (log.isDetailed()) {
        log.logDetailed("Data Set enabled? " + dataSetEnabled);
    }
    if (!dataSetEnabled) {
        return;
    }
    String unitTestName = pipeline.getVariable(DataSetConst.VAR_UNIT_TEST_NAME);
    if (log.isDetailed()) {
        log.logDetailed("Unit test name: " + unitTestName);
    }
    try {
        IHopMetadataProvider metadataProvider = pipelineMeta.getMetadataProvider();
        // 
        if (StringUtil.isEmpty(unitTestName)) {
            return;
        }
        PipelineUnitTest unitTest = metadataProvider.getSerializer(PipelineUnitTest.class).load(unitTestName);
        if (unitTest == null) {
            if (log.isDetailed()) {
                log.logDetailed("Unit test '" + unitTestName + "' could not be found");
            }
            return;
        }
        // 
        for (final TransformMeta transformMeta : pipeline.getPipelineMeta().getTransforms()) {
            String transformName = transformMeta.getName();
            PipelineUnitTestSetLocation inputLocation = unitTest.findInputLocation(transformName);
            if (inputLocation != null && StringUtils.isNotEmpty(inputLocation.getDataSetName())) {
                String inputDataSetName = inputLocation.getDataSetName();
                log.logDetailed("Data Set location found for transform '" + transformName + "' and data set  " + inputDataSetName);
                // We need to inject data from the data set with the specified name into the transform
                // 
                injectDataSetIntoTransform((LocalPipelineEngine) pipeline, inputDataSetName, metadataProvider, transformMeta, inputLocation);
            }
            // How about capturing rows for golden data review?
            // 
            PipelineUnitTestSetLocation goldenLocation = unitTest.findGoldenLocation(transformName);
            if (goldenLocation != null) {
                String goldenDataSetName = goldenLocation.getDataSetName();
                if (!StringUtil.isEmpty(goldenDataSetName)) {
                    log.logDetailed("Capturing rows for validation at pipeline end, transform='" + transformMeta.getName() + "', golden set '" + goldenDataSetName);
                    final RowCollection rowCollection = new RowCollection();
                    // Create a row collection map if it's missing...
                    // 
                    @SuppressWarnings("unchecked") Map<String, RowCollection> collectionMap = (Map<String, RowCollection>) pipeline.getExtensionDataMap().get(DataSetConst.ROW_COLLECTION_MAP);
                    if (collectionMap == null) {
                        collectionMap = new HashMap<>();
                        pipeline.getExtensionDataMap().put(DataSetConst.ROW_COLLECTION_MAP, collectionMap);
                    }
                    // Keep the map for safe keeping...
                    // 
                    collectionMap.put(transformMeta.getName(), rowCollection);
                    // We'll capture the rows from this one and then evaluate them after execution...
                    // 
                    IEngineComponent component = pipeline.findComponent(transformMeta.getName(), 0);
                    component.addRowListener(new RowAdapter() {

                        @Override
                        public void rowReadEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
                            if (rowCollection.getRowMeta() == null) {
                                rowCollection.setRowMeta(rowMeta);
                            }
                            rowCollection.getRows().add(row);
                        }
                    });
                }
            }
        }
    } catch (Throwable e) {
        throw new HopException("Unable to inject data set rows", e);
    }
}

Example 2 with PipelineUnitTestSetLocation

use of org.apache.hop.testing.PipelineUnitTestSetLocation in project hop by apache.

the class DrawGoldenDataSetOnTransformExtensionPoint method drawGoldenSetMarker.

protected void drawGoldenSetMarker(PipelinePainterExtension ext, TransformMeta transformMeta, PipelineUnitTest unitTest, List<AreaOwner> areaOwners) {
    PipelineUnitTestSetLocation location = unitTest.findGoldenLocation(transformMeta.getName());
    if (location == null) {
        return;
    }
    String dataSetName = Const.NVL(location.getDataSetName(), "");
    // Now we're here, draw a marker and indicate the name of the unit test
    // 
    IGc gc = ext.gc;
    int iconSize = ext.iconSize;
    int x = ext.x1;
    int y = ext.y1;
    gc.setLineWidth(transformMeta.isSelected() ? 2 : 1);
    gc.setForeground(IGc.EColor.CRYSTAL);
    gc.setBackground(IGc.EColor.LIGHTGRAY);
    gc.setFont(IGc.EFont.GRAPH);
    Point textExtent = gc.textExtent(dataSetName);
    // add a tiny bit of a margin
    textExtent.x += 6;
    textExtent.y += 6;
    // Draw it at the right hand side
    // 
    int arrowSize = textExtent.y;
    Point point = new Point(x + iconSize, y + (iconSize - textExtent.y) / 2);
    int[] arrow = new int[] { point.x, point.y + textExtent.y / 2, point.x + arrowSize, point.y, point.x + textExtent.x + arrowSize, point.y, point.x + textExtent.x + arrowSize, point.y + textExtent.y, point.x + arrowSize, point.y + textExtent.y };
    gc.fillPolygon(arrow);
    gc.drawPolygon(arrow);
    gc.drawText(dataSetName, point.x + arrowSize + 3, point.y + 3);
    // Leave a trace of what we drew, for memory reasons, just the name of the data set here.
    // 
    areaOwners.add(new AreaOwner(AreaOwner.AreaType.CUSTOM, point.x, point.y, textExtent.x, textExtent.y, new Point(0, 0), DataSetConst.AREA_DRAWN_GOLDEN_DATA_SET, transformMeta.getName()));
    // 
    if (ext.stateMap != null) {
        Map<String, Boolean> results = (Map<String, Boolean>) ext.stateMap.get(DataSetConst.STATE_KEY_GOLDEN_DATASET_RESULTS);
        if (results != null) {
            Boolean result = results.get(dataSetName);
            if (result != null) {
                try {
                    int iconX = point.x + arrowSize + textExtent.x - 5;
                    int iconY = point.y - 5;
                    if (result) {
                        gc.drawImage(IGc.EImage.SUCCESS, iconX, iconY, gc.getMagnification());
                    } else {
                        gc.drawImage(IGc.EImage.FAILURE, iconX, iconY, gc.getMagnification());
                    }
                    areaOwners.add(new AreaOwner(AreaOwner.AreaType.CUSTOM, iconX + ConstUi.SMALL_ICON_SIZE, iconY + 5, ConstUi.SMALL_ICON_SIZE, ConstUi.SMALL_ICON_SIZE, new Point(0, 0), DataSetConst.AREA_DRAWN_GOLDEN_DATA_RESULT, transformMeta.getName()));
                } catch (Exception e) {
                    LogChannel.UI.logError("Error drawing golden data set result on pipeline", e);
                }
            }
        }
    }
}

Example 3 with PipelineUnitTestSetLocation

use of org.apache.hop.testing.PipelineUnitTestSetLocation in project hop by apache.

the class DrawInputDataSetOnTransformExtensionPoint method drawInputDataSetMarker.

private void drawInputDataSetMarker(PipelinePainterExtension ext, TransformMeta transformMeta, PipelineUnitTest unitTest, List<AreaOwner> areaOwners) {
    // Now we're here, draw a marker and indicate the name of the data set name
    // 
    PipelineUnitTestSetLocation location = unitTest.findInputLocation(transformMeta.getName());
    if (location == null) {
        return;
    }
    String dataSetName = Const.NVL(location.getDataSetName(), "");
    IGc gc = ext.gc;
    int iconSize = ext.iconSize;
    int x = ext.x1;
    int y = ext.y1;
    gc.setLineWidth(transformMeta.isSelected() ? 2 : 1);
    gc.setForeground(IGc.EColor.CRYSTAL);
    gc.setBackground(IGc.EColor.LIGHTGRAY);
    gc.setFont(IGc.EFont.GRAPH);
    Point textExtent = gc.textExtent(dataSetName);
    // add a tiny bit of a margin
    textExtent.x += 6;
    textExtent.y += 6;
    // Draw it to the left as an arrow
    // 
    int arrowSize = textExtent.y;
    Point point = new Point(x - textExtent.x - arrowSize - 2, y + (iconSize - textExtent.y) / 2);
    int[] arrow = new int[] { point.x, point.y, point.x + textExtent.x, point.y, point.x + textExtent.x + arrowSize, point.y + textExtent.y / 2, point.x + textExtent.x, point.y + textExtent.y, point.x, point.y + textExtent.y };
    gc.fillPolygon(arrow);
    gc.drawPolygon(arrow);
    gc.drawText(dataSetName, point.x + 3, point.y + 3);
    // Leave a trace of what we drew, for memory reasons, just the name of the data set here.
    // 
    areaOwners.add(new AreaOwner(AreaOwner.AreaType.CUSTOM, point.x, point.y, textExtent.x, textExtent.y, new Point(0, 0), DataSetConst.AREA_DRAWN_INPUT_DATA_SET, transformMeta.getName()));
}

Example 4 with PipelineUnitTestSetLocation

use of org.apache.hop.testing.PipelineUnitTestSetLocation in project hop by apache.

the class LocationMouseDoubleClickExtensionPoint method callExtensionPoint.

@Override
public void callExtensionPoint(ILogChannel log, IVariables variables, HopGuiPipelineGraphExtension pipelineGraphExtension) throws HopException {
    HopGuiPipelineGraph pipelineGraph = pipelineGraphExtension.getPipelineGraph();
    PipelineMeta pipelineMeta = pipelineGraph.getPipelineMeta();
    PipelineUnitTest unitTest = TestingGuiPlugin.getCurrentUnitTest(pipelineMeta);
    if (unitTest == null) {
        return;
    }
    HopGui hopGui = HopGui.getInstance();
    try {
        List<DataSet> dataSets = hopGui.getMetadataProvider().getSerializer(DataSet.class).loadAll();
        Map<String, IRowMeta> transformFieldsMap = new HashMap<>();
        for (TransformMeta transformMeta : pipelineMeta.getTransforms()) {
            try {
                IRowMeta transformFields = pipelineMeta.getTransformFields(pipelineGraph.getVariables(), transformMeta);
                transformFieldsMap.put(transformMeta.getName(), transformFields);
            } catch (Exception e) {
            // Ignore GUI errors...
            }
        }
        // Find the location that was double clicked on...
        // 
        MouseEvent e = pipelineGraphExtension.getEvent();
        Point point = pipelineGraphExtension.getPoint();
        if (e.button == 1 || e.button == 2) {
            AreaOwner areaOwner = pipelineGraph.getVisibleAreaOwner(point.x, point.y);
            if (areaOwner != null && areaOwner.getAreaType() != null) {
                // 
                if (DataSetConst.AREA_DRAWN_INPUT_DATA_SET.equals(areaOwner.getParent())) {
                    // Open the dataset double clicked on...
                    // 
                    String transformName = (String) areaOwner.getOwner();
                    PipelineUnitTestSetLocation inputLocation = unitTest.findInputLocation(transformName);
                    if (inputLocation != null) {
                        PipelineUnitTestSetLocationDialog dialog = new PipelineUnitTestSetLocationDialog(hopGui.getShell(), variables, hopGui.getMetadataProvider(), inputLocation, dataSets, transformFieldsMap);
                        if (dialog.open()) {
                            hopGui.getMetadataProvider().getSerializer(PipelineUnitTest.class).save(unitTest);
                            pipelineGraph.updateGui();
                        }
                    }
                } else if (DataSetConst.AREA_DRAWN_GOLDEN_DATA_SET.equals(areaOwner.getParent())) {
                    // Open the dataset double clicked on...
                    // 
                    String transformName = (String) areaOwner.getOwner();
                    PipelineUnitTestSetLocation goldenLocation = unitTest.findGoldenLocation(transformName);
                    if (goldenLocation != null) {
                        PipelineUnitTestSetLocationDialog dialog = new PipelineUnitTestSetLocationDialog(hopGui.getShell(), variables, hopGui.getMetadataProvider(), goldenLocation, dataSets, transformFieldsMap);
                        if (dialog.open()) {
                            // Save the unit test
                            hopGui.getMetadataProvider().getSerializer(PipelineUnitTest.class).save(unitTest);
                            pipelineGraph.updateGui();
                        }
                    }
                } else if (DataSetConst.AREA_DRAWN_GOLDEN_DATA_RESULT.equals(areaOwner.getParent())) {
                    // Open the dataset double clicked on...
                    // 
                    String transformName = (String) areaOwner.getOwner();
                    PipelineUnitTestSetLocation goldenLocation = unitTest.findGoldenLocation(transformName);
                    if (goldenLocation != null) {
                        // Find the errors list of the unit test...
                        // 
                        IPipelineEngine<PipelineMeta> pipeline = pipelineGraph.getPipeline();
                        if (pipeline == null) {
                            return;
                        }
                        List<UnitTestResult> results = (List<UnitTestResult>) pipeline.getExtensionDataMap().get(DataSetConst.UNIT_TEST_RESULTS);
                        if (results == null || results.isEmpty()) {
                            return;
                        }
                        ValidatePipelineUnitTestExtensionPoint.showUnitTestErrors(pipeline, results, hopGui);
                    }
                }
            }
        }
    } catch (Exception e) {
        new ErrorDialog(hopGui.getShell(), "Error", "Error editing location", e);
    }
}

Example 5 with PipelineUnitTestSetLocation

use of org.apache.hop.testing.PipelineUnitTestSetLocation in project hop by apache.

the class PipelineUnitTestSetLocationDialog method getFieldMappings.

protected void getFieldMappings() {
    try {
        PipelineUnitTestSetLocation loc = new PipelineUnitTestSetLocation();
        getInfo(loc);
        String transformName = wTransformName.getText();
        String datasetName = wDataset.getText();
        if (StringUtils.isEmpty(transformName) || StringUtils.isEmpty(datasetName)) {
            throw new HopException("Please select a transform and a data set to map fields between");
        }
        IRowMeta transformRowMeta = transformFieldsMap.get(transformName);
        if (transformRowMeta == null) {
            throw new HopException("Unable to find fields for transform " + transformName);
        }
        String[] transformFieldNames = transformRowMeta.getFieldNames();
        DataSet dataSet = findDataSet(datasetName);
        IRowMeta setRowMeta = dataSet.getSetRowMeta();
        String[] setFieldNames = setRowMeta.getFieldNames();
        // Get the current mappings...
        // 
        List<SourceToTargetMapping> currentMappings = new ArrayList<>();
        for (PipelineUnitTestFieldMapping mapping : loc.getFieldMappings()) {
            int transformFieldIndex = transformRowMeta.indexOfValue(mapping.getTransformFieldName());
            int setFieldIndex = transformRowMeta.indexOfValue(mapping.getDataSetFieldName());
            if (transformFieldIndex >= 0 && setFieldIndex >= 0) {
                currentMappings.add(new SourceToTargetMapping(transformFieldIndex, setFieldIndex));
            }
        }
        // Edit them
        // 
        EnterMappingDialog mappingDialog = new EnterMappingDialog(shell, transformFieldNames, setFieldNames, currentMappings);
        List<SourceToTargetMapping> newMappings = mappingDialog.open();
        if (newMappings != null) {
            // Simply clean everything and add the new mappings
            // 
            wFieldMappings.clearAll();
            for (SourceToTargetMapping sourceToTargetMapping : newMappings) {
                TableItem item = new TableItem(wFieldMappings.table, SWT.NONE);
                item.setText(1, transformFieldNames[sourceToTargetMapping.getSourcePosition()]);
                item.setText(2, setFieldNames[sourceToTargetMapping.getTargetPosition()]);
            }
            wFieldMappings.removeEmptyRows();
            wFieldMappings.setRowNums();
            wFieldMappings.optWidth(true);
        }
    } catch (Exception e) {
        new ErrorDialog(shell, "Error", "Error mapping fields from transform to dataset", e);
    }
}