Examples with DataSet org.apache.hop.testing.DataSet used on opensource projects

Example 1 with DataSet

use of org.apache.hop.testing.DataSet in project hop by apache.

the class LocationMouseDoubleClickExtensionPoint method openDataSet.

private void openDataSet(String dataSetName) {
    HopGui hopGui = HopGui.getInstance();
    MetadataManager<DataSet> manager = new MetadataManager<>(hopGui.getVariables(), hopGui.getMetadataProvider(), DataSet.class);
    manager.editMetadata(dataSetName);
}

Example 2 with DataSet

use of org.apache.hop.testing.DataSet in project hop by apache.

the class WriteToDataSetExtensionPoint method callExtensionPoint.

@Override
public void callExtensionPoint(ILogChannel log, IVariables variables, IPipelineEngine<PipelineMeta> pipeline) throws HopException {
    final PipelineMeta pipelineMeta = pipeline.getPipelineMeta();
    boolean writeToDataSet = "Y".equalsIgnoreCase(pipeline.getVariable(DataSetConst.VAR_WRITE_TO_DATASET));
    if (!writeToDataSet) {
        return;
    }
    pipeline.addExecutionFinishedListener(engine -> {
        // Remove the flag when done.
        // We don't want to write to the data set every time we run
        // 
        pipeline.setVariable(DataSetConst.VAR_WRITE_TO_DATASET, null);
        // Prevent memory leaking as well
        // 
        WriteToDataSetExtensionPoint.transformsMap.remove(pipelineMeta.getName());
        WriteToDataSetExtensionPoint.mappingsMap.remove(pipelineMeta.getName());
        WriteToDataSetExtensionPoint.setsMap.remove(pipelineMeta.getName());
    });
    try {
        IHopMetadataProvider metadataProvider = pipelineMeta.getMetadataProvider();
        if (metadataProvider == null) {
            // Nothing to do here, we can't reference data sets.
            return;
        }
        // 
        for (final TransformMeta transformMeta : pipeline.getPipelineMeta().getTransforms()) {
            // We might want to pass the data from this transform into a data set all by itself...
            // For this we want to attach a row listener which writes the data.
            // 
            TransformMeta injectMeta = transformsMap.get(pipelineMeta.getName());
            if (injectMeta != null && injectMeta.equals(transformMeta)) {
                final List<SourceToTargetMapping> mappings = mappingsMap.get(pipelineMeta.getName());
                final DataSet dataSet = setsMap.get(pipelineMeta.getName());
                if (mappings != null && dataSet != null) {
                    passTransformRowsToDataSet(pipeline, pipelineMeta, transformMeta, mappings, dataSet);
                }
            }
        }
    } catch (Throwable e) {
        throw new HopException("Unable to pass rows to data set", e);
    }
}

Example 3 with DataSet

use of org.apache.hop.testing.DataSet in project hop by apache.

the class DataSetEditor method editData.

protected void editData() {
    // If the row count is too high, we don't want to load it into memory...
    // Too high simply means: above the preview size...
    // 
    int previewSize = PropsUi.getInstance().getDefaultPreviewSize();
    try {
        verifySettings();
        DataSet set = new DataSet();
        getWidgetsContent(set);
        // get rows from the data set...
        // 
        List<Object[]> rows = set.getAllRows(manager.getVariables(), LogChannel.UI);
        IRowMeta fieldsRowMeta = set.getSetRowMeta();
        boolean written = false;
        while (!written) {
            try {
                EditRowsDialog editRowsDialog = new EditRowsDialog(getShell(), SWT.NONE, BaseMessages.getString(PKG, "DataSetDialog.EditRows.Title"), BaseMessages.getString(PKG, "DataSetDialog.EditRows.Message", set.getName()), fieldsRowMeta, rows);
                List<Object[]> newList = editRowsDialog.open();
                if (newList != null) {
                    FileObject setFolder = HopVfs.getFileObject(set.getActualDataSetFolder(manager.getVariables()));
                    boolean folderExists = setFolder.exists();
                    if (!folderExists) {
                        MessageBox box = new MessageBox(getShell(), SWT.YES | SWT.NO | SWT.CANCEL | SWT.ICON_QUESTION);
                        box.setText("Create data sets folder?");
                        box.setMessage("The data sets folder does not exist. Do you want to create it?" + Const.CR + set.getActualDataSetFolder(manager.getVariables()));
                        int answer = box.open();
                        if ((answer & SWT.YES) != 0) {
                            setFolder.createFolder();
                            folderExists = true;
                        } else if ((answer & SWT.CANCEL) != 0) {
                            break;
                        }
                    }
                    // 
                    if (folderExists) {
                        DataSetCsvUtil.writeDataSetData(manager.getVariables(), set, fieldsRowMeta, newList);
                        written = true;
                    }
                } else {
                    // User hit cancel
                    break;
                }
            } catch (Exception e) {
                new ErrorDialog(getShell(), "Error", "Error writing data to dataset file " + set.getActualDataSetFilename(manager.getVariables()), e);
            }
        }
    } catch (Exception e) {
        new ErrorDialog(getShell(), "Error", "Error previewing data from dataset table", e);
    }
}

Example 4 with DataSet

use of org.apache.hop.testing.DataSet in project hop by apache.

the class PipelineUnitTestSetLocationDialog method getSortFields.

protected void getSortFields() {
    try {
        String datasetName = wDataset.getText();
        if (StringUtils.isEmpty(datasetName)) {
            throw new HopException("Please select a data set to get order fields from");
        }
        DataSet dataSet = findDataSet(datasetName);
        IRowMeta setRowMeta = dataSet.getSetRowMeta();
        String[] setFieldNames = setRowMeta.getFieldNames();
        wFieldOrder.clearAll();
        for (String setFieldName : setFieldNames) {
            TableItem item = new TableItem(wFieldOrder.table, SWT.NONE);
            item.setText(1, setFieldName);
        }
        wFieldOrder.removeEmptyRows();
        wFieldOrder.setRowNums();
        wFieldOrder.optWidth(true);
    } catch (Exception e) {
        new ErrorDialog(shell, "Error", "Error getting sort fields", e);
    }
}

Example 5 with DataSet

use of org.apache.hop.testing.DataSet in project hop by apache.

the class DataSetEditor method setWidgetsContent.

@Override
public void setWidgetsContent() {
    DataSet dataSet = getMetadata();
    wName.setText(Const.NVL(dataSet.getName(), ""));
    wDescription.setText(Const.NVL(dataSet.getDescription(), ""));
    wFolderName.setText(Const.NVL(dataSet.getFolderName(), ""));
    wBaseFilename.setText(Const.NVL(dataSet.getBaseFilename(), ""));
    for (int i = 0; i < dataSet.getFields().size(); i++) {
        DataSetField field = dataSet.getFields().get(i);
        int colNr = 1;
        wFieldMapping.setText(Const.NVL(field.getFieldName(), ""), colNr++, i);
        wFieldMapping.setText(ValueMetaFactory.getValueMetaName(field.getType()), colNr++, i);
        wFieldMapping.setText(Const.NVL(field.getFormat(), ""), colNr++, i);
        wFieldMapping.setText(field.getLength() >= 0 ? Integer.toString(field.getLength()) : "", colNr++, i);
        wFieldMapping.setText(field.getPrecision() >= 0 ? Integer.toString(field.getPrecision()) : "", colNr++, i);
        wFieldMapping.setText(Const.NVL(field.getComment(), ""), colNr++, i);
    }
}