Examples with IHopMetadataProvider org.apache.hop.metadata.api.IHopMetadataProvider used on opensource projects

Example 1 with IHopMetadataProvider

use of org.apache.hop.metadata.api.IHopMetadataProvider in project hop by apache.

the class InjectDataSetIntoTransformExtensionPoint method callExtensionPoint.

@Override
public void callExtensionPoint(ILogChannel log, IVariables variables, final IPipelineEngine<PipelineMeta> pipeline) throws HopException {
    if (!(pipeline instanceof LocalPipelineEngine)) {
        throw new HopPluginException("Unit tests can only run using a local pipeline engine type");
    }
    final PipelineMeta pipelineMeta = pipeline.getPipelineMeta();
    boolean dataSetEnabled = "Y".equalsIgnoreCase(pipeline.getVariable(DataSetConst.VAR_RUN_UNIT_TEST));
    if (log.isDetailed()) {
        log.logDetailed("Data Set enabled? " + dataSetEnabled);
    }
    if (!dataSetEnabled) {
        return;
    }
    String unitTestName = pipeline.getVariable(DataSetConst.VAR_UNIT_TEST_NAME);
    if (log.isDetailed()) {
        log.logDetailed("Unit test name: " + unitTestName);
    }
    try {
        IHopMetadataProvider metadataProvider = pipelineMeta.getMetadataProvider();
        // 
        if (StringUtil.isEmpty(unitTestName)) {
            return;
        }
        PipelineUnitTest unitTest = metadataProvider.getSerializer(PipelineUnitTest.class).load(unitTestName);
        if (unitTest == null) {
            if (log.isDetailed()) {
                log.logDetailed("Unit test '" + unitTestName + "' could not be found");
            }
            return;
        }
        // 
        for (final TransformMeta transformMeta : pipeline.getPipelineMeta().getTransforms()) {
            String transformName = transformMeta.getName();
            PipelineUnitTestSetLocation inputLocation = unitTest.findInputLocation(transformName);
            if (inputLocation != null && StringUtils.isNotEmpty(inputLocation.getDataSetName())) {
                String inputDataSetName = inputLocation.getDataSetName();
                log.logDetailed("Data Set location found for transform '" + transformName + "' and data set  " + inputDataSetName);
                // We need to inject data from the data set with the specified name into the transform
                // 
                injectDataSetIntoTransform((LocalPipelineEngine) pipeline, inputDataSetName, metadataProvider, transformMeta, inputLocation);
            }
            // How about capturing rows for golden data review?
            // 
            PipelineUnitTestSetLocation goldenLocation = unitTest.findGoldenLocation(transformName);
            if (goldenLocation != null) {
                String goldenDataSetName = goldenLocation.getDataSetName();
                if (!StringUtil.isEmpty(goldenDataSetName)) {
                    log.logDetailed("Capturing rows for validation at pipeline end, transform='" + transformMeta.getName() + "', golden set '" + goldenDataSetName);
                    final RowCollection rowCollection = new RowCollection();
                    // Create a row collection map if it's missing...
                    // 
                    @SuppressWarnings("unchecked") Map<String, RowCollection> collectionMap = (Map<String, RowCollection>) pipeline.getExtensionDataMap().get(DataSetConst.ROW_COLLECTION_MAP);
                    if (collectionMap == null) {
                        collectionMap = new HashMap<>();
                        pipeline.getExtensionDataMap().put(DataSetConst.ROW_COLLECTION_MAP, collectionMap);
                    }
                    // Keep the map for safe keeping...
                    // 
                    collectionMap.put(transformMeta.getName(), rowCollection);
                    // We'll capture the rows from this one and then evaluate them after execution...
                    // 
                    IEngineComponent component = pipeline.findComponent(transformMeta.getName(), 0);
                    component.addRowListener(new RowAdapter() {

                        @Override
                        public void rowReadEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
                            if (rowCollection.getRowMeta() == null) {
                                rowCollection.setRowMeta(rowMeta);
                            }
                            rowCollection.getRows().add(row);
                        }
                    });
                }
            }
        }
    } catch (Throwable e) {
        throw new HopException("Unable to inject data set rows", e);
    }
}

Example 2 with IHopMetadataProvider

use of org.apache.hop.metadata.api.IHopMetadataProvider in project hop by apache.

the class WriteToDataSetExtensionPoint method callExtensionPoint.

@Override
public void callExtensionPoint(ILogChannel log, IVariables variables, IPipelineEngine<PipelineMeta> pipeline) throws HopException {
    final PipelineMeta pipelineMeta = pipeline.getPipelineMeta();
    boolean writeToDataSet = "Y".equalsIgnoreCase(pipeline.getVariable(DataSetConst.VAR_WRITE_TO_DATASET));
    if (!writeToDataSet) {
        return;
    }
    pipeline.addExecutionFinishedListener(engine -> {
        // Remove the flag when done.
        // We don't want to write to the data set every time we run
        // 
        pipeline.setVariable(DataSetConst.VAR_WRITE_TO_DATASET, null);
        // Prevent memory leaking as well
        // 
        WriteToDataSetExtensionPoint.transformsMap.remove(pipelineMeta.getName());
        WriteToDataSetExtensionPoint.mappingsMap.remove(pipelineMeta.getName());
        WriteToDataSetExtensionPoint.setsMap.remove(pipelineMeta.getName());
    });
    try {
        IHopMetadataProvider metadataProvider = pipelineMeta.getMetadataProvider();
        if (metadataProvider == null) {
            // Nothing to do here, we can't reference data sets.
            return;
        }
        // 
        for (final TransformMeta transformMeta : pipeline.getPipelineMeta().getTransforms()) {
            // We might want to pass the data from this transform into a data set all by itself...
            // For this we want to attach a row listener which writes the data.
            // 
            TransformMeta injectMeta = transformsMap.get(pipelineMeta.getName());
            if (injectMeta != null && injectMeta.equals(transformMeta)) {
                final List<SourceToTargetMapping> mappings = mappingsMap.get(pipelineMeta.getName());
                final DataSet dataSet = setsMap.get(pipelineMeta.getName());
                if (mappings != null && dataSet != null) {
                    passTransformRowsToDataSet(pipeline, pipelineMeta, transformMeta, mappings, dataSet);
                }
            }
        }
    } catch (Throwable e) {
        throw new HopException("Unable to pass rows to data set", e);
    }
}

Example 3 with IHopMetadataProvider

use of org.apache.hop.metadata.api.IHopMetadataProvider in project hop by apache.

the class TestingGuiPlugin method selectUnitTest.

@GuiToolbarElement(root = HopGuiPipelineGraph.GUI_PLUGIN_TOOLBAR_PARENT_ID, id = ID_TOOLBAR_UNIT_TESTS_COMBO, type = GuiToolbarElementType.COMBO, comboValuesMethod = "getUnitTestsList", extraWidth = 200, toolTip = "i18n::TestingGuiPlugin.ToolbarElement.GetUnitTestList.Tooltip")
public void selectUnitTest() {
    HopGui hopGui = HopGui.getInstance();
    try {
        PipelineMeta pipelineMeta = getActivePipelineMeta();
        if (pipelineMeta == null) {
            return;
        }
        IHopMetadataProvider metadataProvider = hopGui.getMetadataProvider();
        Combo combo = getUnitTestsCombo();
        if (combo == null) {
            return;
        }
        IHopMetadataSerializer<PipelineUnitTest> testSerializer = metadataProvider.getSerializer(PipelineUnitTest.class);
        String testName = combo.getText();
        if (testName != null) {
            PipelineUnitTest unitTest = testSerializer.load(testName);
            if (unitTest == null) {
                throw new HopException(BaseMessages.getString(PKG, "TestingGuiPlugin.ToolbarElement.GetUnitTestList.Exception", testName));
            }
            selectUnitTest(pipelineMeta, unitTest);
            // Update the pipeline graph
            // 
            hopGui.getActiveFileTypeHandler().updateGui();
        }
    } catch (Exception e) {
        new ErrorDialog(hopGui.getShell(), BaseMessages.getString(PKG, "TestingGuiPlugin.ToolbarElement.GetUnitTestList.Error.Header"), BaseMessages.getString(PKG, "TestingGuiPlugin.ToolbarElement.GetUnitTestList.Error.Message"), e);
    }
}

Example 4 with IHopMetadataProvider

use of org.apache.hop.metadata.api.IHopMetadataProvider in project hop by apache.

the class TestingGuiPlugin method setGoldenDataSet.

/**
 * We set an golden data set on the selected unit test
 */
@GuiContextAction(id = ACTION_ID_PIPELINE_GRAPH_TRANSFORM_DEFINE_GOLDEN_DATA_SET, parentId = HopGuiPipelineTransformContext.CONTEXT_ID, type = GuiActionType.Modify, name = "i18n::TestingGuiPlugin.ContextAction.SetGoldenDataset.Name", tooltip = "i18n::TestingGuiPlugin.ContextAction.SetGoldenDataset.Tooltip", image = "set-golden-dataset.svg", category = "i18n::TestingGuiPlugin.Category", categoryOrder = "8")
public void setGoldenDataSet(HopGuiPipelineTransformContext context) {
    PipelineMeta sourcePipelineMeta = context.getPipelineMeta();
    TransformMeta transformMeta = context.getTransformMeta();
    HopGuiPipelineGraph pipelineGraph = context.getPipelineGraph();
    IVariables variables = pipelineGraph.getVariables();
    HopGui hopGui = HopGui.getInstance();
    IHopMetadataProvider metadataProvider = hopGui.getMetadataProvider();
    if (checkTestPresent(hopGui, sourcePipelineMeta)) {
        return;
    }
    PipelineUnitTest unitTest = getCurrentUnitTest(sourcePipelineMeta);
    try {
        // Create a copy and modify the pipeline
        // This way we have
        PipelineMetaModifier modifier = new PipelineMetaModifier(variables, sourcePipelineMeta, unitTest);
        PipelineMeta pipelineMeta = modifier.getTestPipeline(LogChannel.UI, variables, metadataProvider);
        IHopMetadataSerializer<DataSet> setSerializer = metadataProvider.getSerializer(DataSet.class);
        List<String> setNames = setSerializer.listObjectNames();
        Collections.sort(setNames);
        EnterSelectionDialog esd = new EnterSelectionDialog(hopGui.getShell(), setNames.toArray(new String[setNames.size()]), BaseMessages.getString(PKG, "TestingGuiPlugin.ContextAction.SetGoldenDataset.Header"), BaseMessages.getString(PKG, "TestingGuiPlugin.ContextAction.SetGoldenDataset.Message"));
        String setName = esd.open();
        if (setName != null) {
            DataSet dataSet = setSerializer.load(setName);
            boolean changed = setGoldenDataSetOnTransform(variables, metadataProvider, pipelineMeta, transformMeta, unitTest, dataSet);
            if (changed) {
                pipelineGraph.updateGui();
            }
        }
    } catch (Exception e) {
        new ErrorDialog(hopGui.getShell(), BaseMessages.getString(PKG, "TestingGuiPlugin.ContextAction.SetGoldenDataset.Error.Header"), BaseMessages.getString(PKG, "TestingGuiPlugin.ContextAction.SetGoldenDataset.Error.Message"), e);
    }
}

Example 5 with IHopMetadataProvider

use of org.apache.hop.metadata.api.IHopMetadataProvider in project hop by apache.

the class TestingGuiPlugin method createDataSetFromTransform.

/**
 * Create a new data set with the output from
 */
@GuiContextAction(id = "pipeline-graph-transform-20400-create-data-set-from-transform", parentId = HopGuiPipelineTransformContext.CONTEXT_ID, type = GuiActionType.Delete, name = "i18n::TestingGuiPlugin.ContextAction.CreateDataset.Name", tooltip = "i18n::TestingGuiPlugin.ContextAction.CreateDataset.Tooltip", image = "create-dataset.svg", category = "i18n::TestingGuiPlugin.Category", categoryOrder = "8")
public void createDataSetFromTransform(HopGuiPipelineTransformContext context) {
    HopGui hopGui = HopGui.getInstance();
    IHopMetadataProvider metadataProvider = hopGui.getMetadataProvider();
    IVariables variables = context.getPipelineGraph().getVariables();
    TransformMeta transformMeta = context.getTransformMeta();
    PipelineMeta pipelineMeta = context.getPipelineMeta();
    try {
        DataSet dataSet = new DataSet();
        IRowMeta rowMeta = pipelineMeta.getTransformFields(variables, transformMeta);
        for (int i = 0; i < rowMeta.size(); i++) {
            IValueMeta valueMeta = rowMeta.getValueMeta(i);
            String setFieldName = valueMeta.getName();
            DataSetField field = new DataSetField(setFieldName, valueMeta.getType(), valueMeta.getLength(), valueMeta.getPrecision(), valueMeta.getComments(), valueMeta.getFormatMask());
            dataSet.getFields().add(field);
        }
        MetadataManager<DataSet> manager = new MetadataManager<>(hopGui.getVariables(), hopGui.getMetadataProvider(), DataSet.class);
        if (manager.newMetadata(dataSet) != null) {
            PipelineUnitTest unitTest = getCurrentUnitTest(pipelineMeta);
            if (unitTest == null) {
                return;
            }
            // Now that the data set is created and we have an active unit test, perhaps the user wants
            // to use it on the transform?
            // 
            MessageBox box = new MessageBox(hopGui.getShell(), SWT.YES | SWT.NO | SWT.CANCEL | SWT.ICON_QUESTION);
            box.setText(BaseMessages.getString(PKG, "TestingGuiPlugin.ContextAction.CreateDataset.DatasetType.Header"));
            box.setMessage(BaseMessages.getString(PKG, "TestingGuiPlugin.ContextAction.CreateDataset.DatasetType.Message", dataSet.getName(), transformMeta.getName()) + Const.CR + BaseMessages.getString(PKG, "TestingGuiPlugin.ContextAction.CreateDataset.DatasetType.Answer1") + Const.CR + BaseMessages.getString(PKG, "TestingGuiPlugin.ContextAction.CreateDataset.DatasetType.Answer2") + Const.CR + BaseMessages.getString(PKG, "TestingGuiPlugin.ContextAction.CreateDataset.DatasetType.Answer3") + Const.CR);
            int answer = box.open();
            if ((answer & SWT.YES) != 0) {
                // set the new data set as an input
                // 
                setInputDataSetOnTransform(variables, metadataProvider, pipelineMeta, transformMeta, unitTest, dataSet);
            }
            if ((answer & SWT.NO) != 0) {
                setGoldenDataSetOnTransform(variables, metadataProvider, pipelineMeta, transformMeta, unitTest, dataSet);
            }
        }
    } catch (Exception e) {
        new ErrorDialog(hopGui.getShell(), BaseMessages.getString(PKG, "TestingGuiPlugin.ContextAction.CreateDataset.Error.Header"), BaseMessages.getString(PKG, "TestingGuiPlugin.ContextAction.CreateDataset.Error.Message"), e);
    }
}