Examples with PipelineUnitTest org.apache.hop.testing.PipelineUnitTest used on opensource projects

Example 1 with PipelineUnitTest

use of org.apache.hop.testing.PipelineUnitTest in project hop by apache.

the class InjectDataSetIntoTransformExtensionPoint method callExtensionPoint.

@Override
public void callExtensionPoint(ILogChannel log, IVariables variables, final IPipelineEngine<PipelineMeta> pipeline) throws HopException {
    if (!(pipeline instanceof LocalPipelineEngine)) {
        throw new HopPluginException("Unit tests can only run using a local pipeline engine type");
    }
    final PipelineMeta pipelineMeta = pipeline.getPipelineMeta();
    boolean dataSetEnabled = "Y".equalsIgnoreCase(pipeline.getVariable(DataSetConst.VAR_RUN_UNIT_TEST));
    if (log.isDetailed()) {
        log.logDetailed("Data Set enabled? " + dataSetEnabled);
    }
    if (!dataSetEnabled) {
        return;
    }
    String unitTestName = pipeline.getVariable(DataSetConst.VAR_UNIT_TEST_NAME);
    if (log.isDetailed()) {
        log.logDetailed("Unit test name: " + unitTestName);
    }
    try {
        IHopMetadataProvider metadataProvider = pipelineMeta.getMetadataProvider();
        // 
        if (StringUtil.isEmpty(unitTestName)) {
            return;
        }
        PipelineUnitTest unitTest = metadataProvider.getSerializer(PipelineUnitTest.class).load(unitTestName);
        if (unitTest == null) {
            if (log.isDetailed()) {
                log.logDetailed("Unit test '" + unitTestName + "' could not be found");
            }
            return;
        }
        // 
        for (final TransformMeta transformMeta : pipeline.getPipelineMeta().getTransforms()) {
            String transformName = transformMeta.getName();
            PipelineUnitTestSetLocation inputLocation = unitTest.findInputLocation(transformName);
            if (inputLocation != null && StringUtils.isNotEmpty(inputLocation.getDataSetName())) {
                String inputDataSetName = inputLocation.getDataSetName();
                log.logDetailed("Data Set location found for transform '" + transformName + "' and data set  " + inputDataSetName);
                // We need to inject data from the data set with the specified name into the transform
                // 
                injectDataSetIntoTransform((LocalPipelineEngine) pipeline, inputDataSetName, metadataProvider, transformMeta, inputLocation);
            }
            // How about capturing rows for golden data review?
            // 
            PipelineUnitTestSetLocation goldenLocation = unitTest.findGoldenLocation(transformName);
            if (goldenLocation != null) {
                String goldenDataSetName = goldenLocation.getDataSetName();
                if (!StringUtil.isEmpty(goldenDataSetName)) {
                    log.logDetailed("Capturing rows for validation at pipeline end, transform='" + transformMeta.getName() + "', golden set '" + goldenDataSetName);
                    final RowCollection rowCollection = new RowCollection();
                    // Create a row collection map if it's missing...
                    // 
                    @SuppressWarnings("unchecked") Map<String, RowCollection> collectionMap = (Map<String, RowCollection>) pipeline.getExtensionDataMap().get(DataSetConst.ROW_COLLECTION_MAP);
                    if (collectionMap == null) {
                        collectionMap = new HashMap<>();
                        pipeline.getExtensionDataMap().put(DataSetConst.ROW_COLLECTION_MAP, collectionMap);
                    }
                    // Keep the map for safe keeping...
                    // 
                    collectionMap.put(transformMeta.getName(), rowCollection);
                    // We'll capture the rows from this one and then evaluate them after execution...
                    // 
                    IEngineComponent component = pipeline.findComponent(transformMeta.getName(), 0);
                    component.addRowListener(new RowAdapter() {

                        @Override
                        public void rowReadEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
                            if (rowCollection.getRowMeta() == null) {
                                rowCollection.setRowMeta(rowMeta);
                            }
                            rowCollection.getRows().add(row);
                        }
                    });
                }
            }
        }
    } catch (Throwable e) {
        throw new HopException("Unable to inject data set rows", e);
    }
}

Example 2 with PipelineUnitTest

use of org.apache.hop.testing.PipelineUnitTest in project hop by apache.

the class PipelineUnitTestEditor method createControl.

@Override
public void createControl(Composite parent) {
    PipelineUnitTest pipelineUnitTest = this.getMetadata();
    int middle = props.getMiddlePct();
    int margin = Const.MARGIN;
    // The name of the unit test...
    // 
    Label wlName = new Label(parent, SWT.RIGHT);
    props.setLook(wlName);
    wlName.setText(BaseMessages.getString(PKG, "PipelineUnitTestDialog.Name.Label"));
    FormData fdlName = new FormData();
    fdlName.top = new FormAttachment(0, margin);
    fdlName.left = new FormAttachment(0, 0);
    fdlName.right = new FormAttachment(middle, -margin);
    wlName.setLayoutData(fdlName);
    wName = new Text(parent, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
    props.setLook(wName);
    FormData fdName = new FormData();
    fdName.top = new FormAttachment(wlName, 0, SWT.CENTER);
    fdName.left = new FormAttachment(middle, 0);
    fdName.right = new FormAttachment(100, 0);
    wName.setLayoutData(fdName);
    Control lastControl = wName;
    // The description of the test...
    // 
    Label wlDescription = new Label(parent, SWT.RIGHT);
    props.setLook(wlDescription);
    wlDescription.setText(BaseMessages.getString(PKG, "PipelineUnitTestDialog.Description.Label"));
    FormData fdlDescription = new FormData();
    fdlDescription.top = new FormAttachment(lastControl, margin);
    fdlDescription.left = new FormAttachment(0, 0);
    fdlDescription.right = new FormAttachment(middle, -margin);
    wlDescription.setLayoutData(fdlDescription);
    wDescription = new Text(parent, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
    props.setLook(wDescription);
    FormData fdDescription = new FormData();
    fdDescription.top = new FormAttachment(wlDescription, 0, SWT.CENTER);
    fdDescription.left = new FormAttachment(middle, 0);
    fdDescription.right = new FormAttachment(100, 0);
    wDescription.setLayoutData(fdDescription);
    lastControl = wDescription;
    // The type of test...
    // 
    Label wlTestType = new Label(parent, SWT.RIGHT);
    props.setLook(wlTestType);
    wlTestType.setText(BaseMessages.getString(PKG, "PipelineUnitTestDialog.TestType.Label"));
    FormData fdlTestType = new FormData();
    fdlTestType.top = new FormAttachment(lastControl, margin);
    fdlTestType.left = new FormAttachment(0, 0);
    fdlTestType.right = new FormAttachment(middle, -margin);
    wlTestType.setLayoutData(fdlTestType);
    wTestType = new Combo(parent, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
    FormData fdTestType = new FormData();
    fdTestType.top = new FormAttachment(wlTestType, 0, SWT.CENTER);
    fdTestType.left = new FormAttachment(middle, 0);
    fdTestType.right = new FormAttachment(100, 0);
    wTestType.setLayoutData(fdTestType);
    wTestType.setItems(DataSetConst.getTestTypeDescriptions());
    lastControl = wTestType;
    // The filename of the pipeline to test
    // 
    Label wlPipelineFilename = new Label(parent, SWT.RIGHT);
    props.setLook(wlPipelineFilename);
    wlPipelineFilename.setText(BaseMessages.getString(PKG, "PipelineUnitTestDialog.PipelineFilename.Label"));
    FormData fdlPipelineFilename = new FormData();
    fdlPipelineFilename.top = new FormAttachment(lastControl, margin);
    fdlPipelineFilename.left = new FormAttachment(0, 0);
    fdlPipelineFilename.right = new FormAttachment(middle, -margin);
    wlPipelineFilename.setLayoutData(fdlPipelineFilename);
    wPipelineFilename = new Text(parent, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
    props.setLook(wPipelineFilename);
    FormData fdPipelineFilename = new FormData();
    fdPipelineFilename.top = new FormAttachment(wlPipelineFilename, 0, SWT.CENTER);
    fdPipelineFilename.left = new FormAttachment(middle, 0);
    fdPipelineFilename.right = new FormAttachment(100, 0);
    wPipelineFilename.setLayoutData(fdPipelineFilename);
    lastControl = wPipelineFilename;
    // The optional filename of the test result...
    // 
    Label wlFilename = new Label(parent, SWT.RIGHT);
    props.setLook(wlFilename);
    wlFilename.setText(BaseMessages.getString(PKG, "PipelineUnitTestDialog.Filename.Label"));
    FormData fdlFilename = new FormData();
    fdlFilename.top = new FormAttachment(lastControl, margin);
    fdlFilename.left = new FormAttachment(0, 0);
    fdlFilename.right = new FormAttachment(middle, -margin);
    wlFilename.setLayoutData(fdlFilename);
    wFilename = new TextVar(manager.getVariables(), parent, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
    props.setLook(wFilename);
    FormData fdFilename = new FormData();
    fdFilename.top = new FormAttachment(wlFilename, 0, SWT.CENTER);
    fdFilename.left = new FormAttachment(middle, 0);
    fdFilename.right = new FormAttachment(100, 0);
    wFilename.setLayoutData(fdFilename);
    lastControl = wFilename;
    // The base path for relative test path resolution
    // 
    Label wlBasePath = new Label(parent, SWT.RIGHT);
    props.setLook(wlBasePath);
    wlBasePath.setText(BaseMessages.getString(PKG, "PipelineUnitTestDialog.BasePath.Label"));
    FormData fdlBasePath = new FormData();
    fdlBasePath.top = new FormAttachment(lastControl, margin);
    fdlBasePath.left = new FormAttachment(0, 0);
    fdlBasePath.right = new FormAttachment(middle, -margin);
    wlBasePath.setLayoutData(fdlBasePath);
    wBasePath = new TextVar(manager.getVariables(), parent, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
    props.setLook(wBasePath);
    FormData fdBasePath = new FormData();
    fdBasePath.top = new FormAttachment(wlBasePath, 0, SWT.CENTER);
    fdBasePath.left = new FormAttachment(middle, 0);
    fdBasePath.right = new FormAttachment(100, 0);
    wBasePath.setLayoutData(fdBasePath);
    lastControl = wBasePath;
    // The base path for relative test path resolution
    // 
    Label wlAutoOpen = new Label(parent, SWT.RIGHT);
    props.setLook(wlAutoOpen);
    wlAutoOpen.setText(BaseMessages.getString(PKG, "PipelineUnitTestDialog.AutoOpen.Label"));
    FormData fdlAutoOpen = new FormData();
    fdlAutoOpen.top = new FormAttachment(lastControl, margin);
    fdlAutoOpen.left = new FormAttachment(0, 0);
    fdlAutoOpen.right = new FormAttachment(middle, -margin);
    wlAutoOpen.setLayoutData(fdlAutoOpen);
    wAutoOpen = new Button(parent, SWT.CHECK);
    props.setLook(wAutoOpen);
    FormData fdAutoOpen = new FormData();
    fdAutoOpen.top = new FormAttachment(wlAutoOpen, 0, SWT.CENTER);
    fdAutoOpen.left = new FormAttachment(middle, 0);
    fdAutoOpen.right = new FormAttachment(100, 0);
    wAutoOpen.setLayoutData(fdAutoOpen);
    lastControl = wAutoOpen;
    // The list of database replacements in the unit test pipeline
    // 
    Label wlFieldMapping = new Label(parent, SWT.NONE);
    wlFieldMapping.setText(BaseMessages.getString(PKG, "PipelineUnitTestDialog.DbReplacements.Label"));
    props.setLook(wlFieldMapping);
    FormData fdlUpIns = new FormData();
    fdlUpIns.left = new FormAttachment(0, 0);
    fdlUpIns.top = new FormAttachment(lastControl, 3 * margin);
    wlFieldMapping.setLayoutData(fdlUpIns);
    lastControl = wlFieldMapping;
    // the database replacements
    // 
    List<String> dbNames;
    try {
        dbNames = metadataProvider.getSerializer(DatabaseMeta.class).listObjectNames();
        Collections.sort(dbNames);
    } catch (HopException e) {
        LogChannel.UI.logError("Error getting list of databases", e);
        dbNames = Collections.emptyList();
    }
    ColumnInfo[] columns = new ColumnInfo[] { new ColumnInfo(BaseMessages.getString(PKG, "PipelineUnitTestDialog.DbReplacement.ColumnInfo.OriginalDb"), ColumnInfo.COLUMN_TYPE_CCOMBO, dbNames.toArray(new String[0]), false), new ColumnInfo(BaseMessages.getString(PKG, "PipelineUnitTestDialog.DbReplacement.ColumnInfo.ReplacementDb"), ColumnInfo.COLUMN_TYPE_CCOMBO, dbNames.toArray(new String[0]), false) };
    columns[0].setUsingVariables(true);
    columns[1].setUsingVariables(true);
    wDbReplacements = new TableView(new Variables(), parent, SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI | SWT.V_SCROLL | SWT.H_SCROLL, columns, pipelineUnitTest.getTweaks().size(), null, props);
    FormData fdDbReplacements = new FormData();
    fdDbReplacements.left = new FormAttachment(0, 0);
    fdDbReplacements.top = new FormAttachment(lastControl, margin);
    fdDbReplacements.right = new FormAttachment(100, 0);
    fdDbReplacements.bottom = new FormAttachment(lastControl, 250);
    wDbReplacements.setLayoutData(fdDbReplacements);
    lastControl = wDbReplacements;
    Label wlVariableValues = new Label(parent, SWT.NONE);
    wlVariableValues.setText(BaseMessages.getString(PKG, "PipelineUnitTestDialog.VariableValues.Label"));
    props.setLook(wlVariableValues);
    FormData fdlVariableValues = new FormData();
    fdlVariableValues.left = new FormAttachment(0, 0);
    fdlVariableValues.top = new FormAttachment(lastControl, margin);
    wlVariableValues.setLayoutData(fdlVariableValues);
    lastControl = wlVariableValues;
    ColumnInfo[] varValColumns = new ColumnInfo[] { new ColumnInfo(BaseMessages.getString(PKG, "PipelineUnitTestDialog.VariableValues.ColumnInfo.VariableName"), ColumnInfo.COLUMN_TYPE_TEXT, false), new ColumnInfo(BaseMessages.getString(PKG, "PipelineUnitTestDialog.VariableValues.ColumnInfo.VariableValue"), ColumnInfo.COLUMN_TYPE_TEXT, false) };
    varValColumns[0].setUsingVariables(true);
    varValColumns[1].setUsingVariables(true);
    wVariableValues = new TableView(new Variables(), parent, SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI | SWT.V_SCROLL | SWT.H_SCROLL, varValColumns, pipelineUnitTest.getVariableValues().size(), null, props);
    FormData fdVariableValues = new FormData();
    fdVariableValues.left = new FormAttachment(0, 0);
    fdVariableValues.top = new FormAttachment(lastControl, margin);
    fdVariableValues.right = new FormAttachment(100, 0);
    fdVariableValues.bottom = new FormAttachment(100, -2 * margin);
    wVariableValues.setLayoutData(fdVariableValues);
    setWidgetsContent();
    // Add listener to detect change after loading data
    Listener modifyListener = e -> setChanged();
    wName.addListener(SWT.Modify, modifyListener);
    wDescription.addListener(SWT.Modify, modifyListener);
    wTestType.addListener(SWT.Modify, modifyListener);
    wPipelineFilename.addListener(SWT.Modify, modifyListener);
    wFilename.addListener(SWT.Modify, modifyListener);
    wBasePath.addListener(SWT.Modify, modifyListener);
    wAutoOpen.addListener(SWT.Selection, modifyListener);
}

Example 3 with PipelineUnitTest

use of org.apache.hop.testing.PipelineUnitTest in project hop by apache.

the class PipelineUnitTestEditor method setWidgetsContent.

@Override
public void setWidgetsContent() {
    PipelineUnitTest pipelineUnitTest = this.getMetadata();
    wName.setText(Const.NVL(pipelineUnitTest.getName(), ""));
    wDescription.setText(Const.NVL(pipelineUnitTest.getDescription(), ""));
    wTestType.setText(Const.NVL(DataSetConst.getTestTypeDescription(pipelineUnitTest.getType()), ""));
    wPipelineFilename.setText(Const.NVL(pipelineUnitTest.getPipelineFilename(), ""));
    wFilename.setText(Const.NVL(pipelineUnitTest.getFilename(), ""));
    wBasePath.setText(Const.NVL(pipelineUnitTest.getBasePath(), ""));
    wAutoOpen.setSelection(pipelineUnitTest.isAutoOpening());
    for (int i = 0; i < pipelineUnitTest.getDatabaseReplacements().size(); i++) {
        PipelineUnitTestDatabaseReplacement dbReplacement = pipelineUnitTest.getDatabaseReplacements().get(i);
        wDbReplacements.setText(Const.NVL(dbReplacement.getOriginalDatabaseName(), ""), 1, i);
        wDbReplacements.setText(Const.NVL(dbReplacement.getReplacementDatabaseName(), ""), 2, i);
    }
    for (int i = 0; i < pipelineUnitTest.getVariableValues().size(); i++) {
        VariableValue variableValue = pipelineUnitTest.getVariableValues().get(i);
        wVariableValues.setText(Const.NVL(variableValue.getKey(), ""), 1, i);
        wVariableValues.setText(Const.NVL(variableValue.getValue(), ""), 2, i);
    }
    wDbReplacements.removeEmptyRows();
    wDbReplacements.setRowNums();
}

Example 4 with PipelineUnitTest

use of org.apache.hop.testing.PipelineUnitTest in project hop by apache.

the class RunPipelineTestsDialog method getTestNames.

private void getTestNames() {
    try {
        IHopMetadataSerializer<PipelineUnitTest> testSerializer = metadataProvider.getSerializer(PipelineUnitTest.class);
        List<String> testNames = testSerializer.listObjectNames();
        // 
        for (String testName : testNames) {
            wTestNames.add(testName);
        }
        wTestNames.optimizeTableView();
    } catch (Exception e) {
        new ErrorDialog(shell, "Error", "Error getting list of pipeline unit test names", e);
    }
}

Example 5 with PipelineUnitTest

use of org.apache.hop.testing.PipelineUnitTest in project hop by apache.

the class ExecuteTests method processRow.

@Override
public boolean processRow() throws HopException {
    if (first) {
        first = false;
        IHopMetadataSerializer<PipelineUnitTest> testSerializer = metadataProvider.getSerializer(PipelineUnitTest.class);
        // 
        if (data.hasPrevious) {
            data.tests = new ArrayList<>();
            Object[] row = getRow();
            if (row == null) {
                // No input and as such no tests to execute. We're all done here.
                // 
                setOutputDone();
                return false;
            }
            int inputFieldIndex = getInputRowMeta().indexOfValue(meta.getTestNameInputField());
            if (inputFieldIndex < 0) {
                throw new HopException("Unable to find test name field '" + meta.getTestNameInputField() + "' in the input");
            }
            while (row != null) {
                String testName = getInputRowMeta().getString(row, inputFieldIndex);
                try {
                    PipelineUnitTest pipelineUnitTest = testSerializer.load(testName);
                    data.tests.add(pipelineUnitTest);
                } catch (Exception e) {
                    throw new HopException("Unable to load test '" + testName + "'", e);
                }
                row = getRow();
            }
        } else {
            // 
            try {
                data.tests = new ArrayList<>();
                for (String testName : testSerializer.listObjectNames()) {
                    PipelineUnitTest pipelineUnitTest = testSerializer.load(testName);
                    if (meta.getTypeToExecute() == null || meta.getTypeToExecute() == pipelineUnitTest.getType()) {
                        data.tests.add(pipelineUnitTest);
                    }
                }
            } catch (HopException e) {
                throw new HopException("Unable to read pipeline unit tests from the metadata", e);
            }
        }
        data.testsIterator = data.tests.iterator();
        data.outputRowMeta = new RowMeta();
        meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider);
    }
    // 
    if (data.testsIterator.hasNext()) {
        PipelineUnitTest test = data.testsIterator.next();
        UnitTestUtil.executeUnitTest(test, this, getLogLevel(), metadataProvider, this, // 
        (pipeline, pipelineResult) -> {
            if (pipelineResult.getNrErrors() != 0) {
                // The pipeline had a failure, report this too.
                // 
                Object[] row = RowDataUtil.allocateRowData(data.outputRowMeta.size());
                int index = 0;
                row[index++] = pipeline.getPipelineMeta().getName();
                row[index++] = null;
                row[index++] = null;
                row[index++] = null;
                row[index++] = Boolean.TRUE;
                row[index++] = pipelineResult.getLogText();
                ExecuteTests.this.putRow(data.outputRowMeta, row);
            }
        }, // 
        (pipeline, testResults) -> {
            for (UnitTestResult testResult : testResults) {
                Object[] row = RowDataUtil.allocateRowData(data.outputRowMeta.size());
                int index = 0;
                row[index++] = testResult.getPipelineName();
                row[index++] = testResult.getUnitTestName();
                row[index++] = testResult.getDataSetName();
                row[index++] = testResult.getTransformName();
                row[index++] = testResult.isError();
                row[index++] = testResult.getComment();
                putRow(data.outputRowMeta, row);
            }
        }, // 
        (test1, testPipelineMeta, e) -> {
            // Some configuration or setup error...
            // 
            Object[] row = RowDataUtil.allocateRowData(data.outputRowMeta.size());
            int index = 0;
            row[index++] = testPipelineMeta == null ? null : testPipelineMeta.getName();
            row[index++] = test1.getName();
            row[index++] = null;
            row[index++] = null;
            // ERROR!
            row[index++] = Boolean.TRUE;
            row[index++] = e.getMessage() + " : " + Const.getStackTracker(e);
            ExecuteTests.this.putRow(data.outputRowMeta, row);
        });
        return true;
    } else {
        setOutputDone();
        return false;
    }
}