Examples with RowGeneratorMeta org.apache.hop.pipeline.transforms.rowgenerator.RowGeneratorMeta used on opensource projects

Example 1 with RowGeneratorMeta

use of org.apache.hop.pipeline.transforms.rowgenerator.RowGeneratorMeta in project hop by apache.

the class UserDefinedJavaClassDialog method test.

private boolean test() {
    PluginRegistry registry = PluginRegistry.getInstance();
    String scriptTransformName = wTransformName.getText();
    if (!checkForTransformClass()) {
        return false;
    }
    // Create a transform with the information in this dialog
    UserDefinedJavaClassMeta udjcMeta = new UserDefinedJavaClassMeta();
    getInfo(udjcMeta);
    try {
        // First, before we get into the trial run, just see if the classes
        // all compile.
        udjcMeta.cookClasses();
        if (udjcMeta.cookErrors.size() == 1) {
            Exception e = udjcMeta.cookErrors.get(0);
            new ErrorDialog(shell, "Error during class compilation", e.toString(), e);
            return false;
        } else if (udjcMeta.cookErrors.size() > 1) {
            Exception e = udjcMeta.cookErrors.get(0);
            new ErrorDialog(shell, "Errors during class compilation", String.format("Multiple errors during class compilation. First error:\n%s", e.toString()), e);
            return false;
        }
        // What fields are coming into the transform?
        IRowMeta rowMeta = pipelineMeta.getPrevTransformFields(variables, transformName).clone();
        if (rowMeta != null) {
            // time
            if (genMeta == null) {
                genMeta = new RowGeneratorMeta();
                genMeta.setRowLimit("10");
                // CHECKSTYLE:Indentation:OFF
                for (int i = 0; i < rowMeta.size(); i++) {
                    IValueMeta valueMeta = rowMeta.getValueMeta(i);
                    if (valueMeta.isStorageBinaryString()) {
                        valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL);
                    }
                    GeneratorField field = new GeneratorField();
                    field.setName(valueMeta.getName());
                    field.setType(valueMeta.getTypeDesc());
                    field.setLength(valueMeta.getLength());
                    field.setPrecision(valueMeta.getPrecision());
                    field.setCurrency(valueMeta.getCurrencySymbol());
                    field.setDecimal(valueMeta.getDecimalSymbol());
                    field.setGroup(valueMeta.getGroupingSymbol());
                    String string = null;
                    switch(valueMeta.getType()) {
                        case IValueMeta.TYPE_DATE:
                            field.setFormat("yyyy/MM/dd HH:mm:ss");
                            valueMeta.setConversionMask(field.getFormat());
                            string = valueMeta.getString(new Date());
                            break;
                        case IValueMeta.TYPE_STRING:
                            string = "test value test value";
                            break;
                        case IValueMeta.TYPE_INTEGER:
                            field.setFormat("#");
                            valueMeta.setConversionMask(field.getFormat());
                            string = valueMeta.getString(Long.valueOf(0L));
                            break;
                        case IValueMeta.TYPE_NUMBER:
                            field.setFormat("#.#");
                            valueMeta.setConversionMask(field.getFormat());
                            string = valueMeta.getString(Double.valueOf(0.0D));
                            break;
                        case IValueMeta.TYPE_BIGNUMBER:
                            field.setFormat("#.#");
                            valueMeta.setConversionMask(field.getFormat());
                            string = valueMeta.getString(BigDecimal.ZERO);
                            break;
                        case IValueMeta.TYPE_BOOLEAN:
                            string = valueMeta.getString(Boolean.TRUE);
                            break;
                        case IValueMeta.TYPE_BINARY:
                            string = valueMeta.getString(new byte[] { 65, 66, 67, 68, 69, 70, 71, 72, 73, 74 });
                            break;
                        default:
                            break;
                    }
                    field.setValue(string);
                    genMeta.getFields().add(field);
                }
            }
            TransformMeta genTransform = new TransformMeta(registry.getPluginId(TransformPluginType.class, genMeta), "## TEST DATA ##", genMeta);
            genTransform.setLocation(50, 50);
            TransformMeta scriptTransform = new TransformMeta(registry.getPluginId(TransformPluginType.class, udjcMeta), Const.NVL(scriptTransformName, "## SCRIPT ##"), udjcMeta);
            scriptTransformName = scriptTransform.getName();
            scriptTransform.setLocation(150, 50);
            // Create a hop between both transforms...
            // 
            PipelineHopMeta hop = new PipelineHopMeta(genTransform, scriptTransform);
            // Generate a new test pipeline...
            // 
            PipelineMeta pipelineMeta = new PipelineMeta();
            pipelineMeta.setName(wTransformName.getText() + " - PREVIEW");
            pipelineMeta.addTransform(genTransform);
            pipelineMeta.addTransform(scriptTransform);
            pipelineMeta.addPipelineHop(hop);
            // OK, now we ask the user to edit this dialog...
            // 
            // Now run this pipeline and grab the results...
            // 
            PipelinePreviewProgressDialog progressDialog = new PipelinePreviewProgressDialog(shell, variables, pipelineMeta, new String[] { scriptTransformName }, new int[] { Const.toInt(genMeta.getRowLimit(), 10) });
            progressDialog.open();
            Pipeline pipeline = progressDialog.getPipeline();
            String loggingText = progressDialog.getLoggingText();
            if (!progressDialog.isCancelled()) {
                if (pipeline.getResult() != null && pipeline.getResult().getNrErrors() > 0) {
                    EnterTextDialog etd = new EnterTextDialog(shell, BaseMessages.getString("System.Dialog.PreviewError.Title"), BaseMessages.getString("System.Dialog.PreviewError.Message"), loggingText, true);
                    etd.setReadOnly();
                    etd.open();
                }
            }
            IRowMeta previewRowsMeta = progressDialog.getPreviewRowsMeta(wTransformName.getText());
            List<Object[]> previewRows = progressDialog.getPreviewRows(wTransformName.getText());
            if (previewRowsMeta != null && previewRows != null && previewRows.size() > 0) {
                PreviewRowsDialog prd = new PreviewRowsDialog(shell, variables, SWT.NONE, wTransformName.getText(), previewRowsMeta, previewRows, loggingText);
                prd.open();
            }
            return true;
        } else {
            throw new HopException(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.Exception.CouldNotGetFields"));
        }
    } catch (Exception e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.TestFailed.DialogTitle"), BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.TestFailed.DialogMessage"), e);
        return false;
    }
}

Example 2 with RowGeneratorMeta

use of org.apache.hop.pipeline.transforms.rowgenerator.RowGeneratorMeta in project hop by apache.

the class BeamRowGeneratorTransformHandler method handleTransform.

@Override
public void handleTransform(ILogChannel log, IVariables variables, IBeamPipelineEngineRunConfiguration runConfiguration, IHopMetadataProvider metadataProvider, PipelineMeta pipelineMeta, List<String> transformPluginClasses, List<String> xpPluginClasses, TransformMeta transformMeta, Map<String, PCollection<HopRow>> transformCollectionMap, Pipeline pipeline, IRowMeta rowMeta, List<TransformMeta> previousTransforms, PCollection<HopRow> input) throws HopException {
    // Don't simply case but serialize/de-serialize the metadata to prevent classloader exceptions
    // 
    RowGeneratorMeta meta = new RowGeneratorMeta();
    loadTransformMetadata(meta, transformMeta, metadataProvider, pipelineMeta);
    List<ICheckResult> remarks = new ArrayList<>();
    final RowMetaAndData rowMetaAndData = RowGenerator.buildRow(meta, remarks, "");
    if (!remarks.isEmpty()) {
        String message = "There are " + remarks.size() + " remarks concerning the generated rows:" + Const.CR;
        for (ICheckResult remark : remarks) {
            message += remark.getText() + Const.CR;
        }
        throw new HopException(message);
    }
    String rowMetaJson = JsonRowMeta.toJson(rowMetaAndData.getRowMeta());
    String rowDataXml;
    try {
        rowDataXml = rowMetaAndData.getRowMeta().getDataXml(rowMetaAndData.getData());
    } catch (IOException e) {
        throw new HopException("Error encoding row as XML", e);
    }
    long intervalMs = Const.toLong(variables.resolve(meta.getIntervalInMs()), -1L);
    if (intervalMs < 0) {
        throw new HopException("The interval in milliseconds is expected to be >= 0, not '" + meta.getIntervalInMs() + "'");
    }
    PCollection<HopRow> afterInput;
    if (meta.isNeverEnding()) {
        SyntheticSourceOptions options;
        String json = "{" + "\"numRecords\" : " + Long.MAX_VALUE + ", \"delayDistribution\" : { \"type\" : \"const\", \"const\" : " + intervalMs + "}" + ", \"forceNumInitialBundles\" : " + transformMeta.getCopies(variables) + "}";
        try {
            options = SyntheticSourceOptions.fromJsonString(json, SyntheticSourceOptions.class);
        } catch (Exception e) {
            throw new HopException("Unable to parse options for the Beam unbounded synthetic source, JSON: " + json, e);
        }
        SyntheticUnboundedSource unboundedSource = new SyntheticUnboundedSource(options);
        Read.Unbounded<KV<byte[], byte[]>> unboundedReader = Read.from(unboundedSource);
        PCollection<KV<byte[], byte[]>> sourceInput = pipeline.apply(unboundedReader);
        String currentTimeField = variables.resolve(meta.getRowTimeField());
        int currentTimeFieldIndex = rowMeta.indexOfValue(currentTimeField);
        String previousTimeField = variables.resolve(meta.getLastTimeField());
        int previousTimeFieldIndex = rowMeta.indexOfValue(previousTimeField);
        afterInput = sourceInput.apply(ParDo.of(new StaticHopRowFn(transformMeta.getName(), rowMetaJson, rowDataXml, true, currentTimeFieldIndex, previousTimeFieldIndex, transformPluginClasses, xpPluginClasses)));
    } else {
        // A fixed number of records
        // 
        long numRecords = Const.toLong(variables.resolve(meta.getRowLimit()), -1L);
        if (numRecords < 0) {
            throw new HopException("Please specify a valid number of records to generate, not '" + meta.getRowLimit() + "'");
        }
        String json = "{" + "\"numRecords\" : " + numRecords + ", \"forceNumInitialBundles\" : " + transformMeta.getCopies(variables) + "}";
        SyntheticSourceOptions options;
        try {
            options = SyntheticSourceOptions.fromJsonString(json, SyntheticSourceOptions.class);
        } catch (Exception e) {
            throw new HopException("Unable to parse options for the Beam unbounded synthetic source, JSON: " + json, e);
        }
        SyntheticBoundedSource boundedSource = new SyntheticBoundedSource(options);
        Read.Bounded<KV<byte[], byte[]>> boundedReader = Read.from(boundedSource);
        PCollection<KV<byte[], byte[]>> sourceInput = pipeline.apply(boundedReader);
        afterInput = sourceInput.apply(ParDo.of(new StaticHopRowFn(transformMeta.getName(), rowMetaJson, rowDataXml, false, -1, -1, transformPluginClasses, xpPluginClasses)));
    }
    transformCollectionMap.put(transformMeta.getName(), afterInput);
    log.logBasic("Handled transform (ROW GENERATOR) : " + transformMeta.getName());
}

Example 3 with RowGeneratorMeta

use of org.apache.hop.pipeline.transforms.rowgenerator.RowGeneratorMeta in project hop by apache.

the class JsonOutputTest method createRowGeneratorTransform.

/**
 * Creates a row generator transform for this class..
 *
 * @param name
 * @param registry
 * @return
 */
private TransformMeta createRowGeneratorTransform(String name, PluginRegistry registry) {
    // Default the name if it is empty
    String testFileOutputName = (Utils.isEmpty(name) ? "generate rows" : name);
    // create the RowGenerator and Transform Meta
    RowGeneratorMeta rowGeneratorMeta = new RowGeneratorMeta();
    String rowGeneratorPid = registry.getPluginId(TransformPluginType.class, rowGeneratorMeta);
    TransformMeta generateRowsTransform = new TransformMeta(rowGeneratorPid, testFileOutputName, rowGeneratorMeta);
    // Set the field names, types and values
    rowGeneratorMeta.getFields().addAll(Arrays.asList(new GeneratorField("Id", "Integer", "", -1, -1, "", "", "", "1", false), new GeneratorField("State", "String", "", -1, -1, "", "", "", "Florida", false), new GeneratorField("City", "String", "", -1, -1, "", "", "", "Orlando", false)));
    rowGeneratorMeta.setRowLimit("10");
    // return the transform meta
    return generateRowsTransform;
}

Example 4 with RowGeneratorMeta

use of org.apache.hop.pipeline.transforms.rowgenerator.RowGeneratorMeta in project hop by apache.

the class ScriptValuesDialog method newTest.

private boolean newTest() {
    PluginRegistry registry = PluginRegistry.getInstance();
    String scriptTransformName = wTransformName.getText();
    try {
        // What fields are coming into the transform?
        // 
        IRowMeta rowMeta = pipelineMeta.getPrevTransformFields(variables, transformName).clone();
        if (rowMeta != null) {
            // 
            if (genMeta == null) {
                genMeta = new RowGeneratorMeta();
                genMeta.setRowLimit("10");
                // CHECKSTYLE:Indentation:OFF
                for (int i = 0; i < rowMeta.size(); i++) {
                    IValueMeta valueMeta = rowMeta.getValueMeta(i);
                    if (valueMeta.isStorageBinaryString()) {
                        valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL);
                    }
                    String string = null;
                    String format = valueMeta.getFormatMask();
                    boolean setValueMetaFormat = false;
                    switch(valueMeta.getType()) {
                        case IValueMeta.TYPE_DATE:
                            format = "yyyy/MM/dd HH:mm:ss";
                            setValueMetaFormat = true;
                            string = valueMeta.getString(new Date());
                            break;
                        case IValueMeta.TYPE_STRING:
                            string = "test value test value";
                            break;
                        case IValueMeta.TYPE_INTEGER:
                            format = "#";
                            setValueMetaFormat = true;
                            string = valueMeta.getString(0L);
                            break;
                        case IValueMeta.TYPE_NUMBER:
                            format = "#.#";
                            setValueMetaFormat = true;
                            string = valueMeta.getString(0.0D);
                            break;
                        case IValueMeta.TYPE_BIGNUMBER:
                            format = "#.#";
                            setValueMetaFormat = true;
                            string = valueMeta.getString(BigDecimal.ZERO);
                            break;
                        case IValueMeta.TYPE_BOOLEAN:
                            string = valueMeta.getString(Boolean.TRUE);
                            break;
                        case IValueMeta.TYPE_BINARY:
                            string = valueMeta.getString(new byte[] { 65, 66, 67, 68, 69, 70, 71, 72, 73, 74 });
                            break;
                        default:
                            break;
                    }
                    if (setValueMetaFormat) {
                        valueMeta.setConversionMask(format);
                    }
                    GeneratorField field = new GeneratorField(valueMeta.getName(), valueMeta.getTypeDesc(), format, valueMeta.getLength(), valueMeta.getPrecision(), valueMeta.getCurrencySymbol(), valueMeta.getDecimalSymbol(), valueMeta.getGroupingSymbol(), string, false);
                    genMeta.getFields().add(field);
                }
            }
            TransformMeta genTransform = new TransformMeta(registry.getPluginId(TransformPluginType.class, genMeta), "## TEST DATA ##", genMeta);
            genTransform.setLocation(50, 50);
            // Now create a JavaScript transform with the information in this dialog
            // 
            ScriptValuesMeta scriptMeta = new ScriptValuesMeta();
            getInfo(scriptMeta);
            TransformMeta scriptTransform = new TransformMeta(registry.getPluginId(TransformPluginType.class, scriptMeta), Const.NVL(scriptTransformName, "## SCRIPT ##"), scriptMeta);
            scriptTransformName = scriptTransform.getName();
            scriptTransform.setLocation(150, 50);
            // Create a hop between both transforms...
            // 
            PipelineHopMeta hop = new PipelineHopMeta(genTransform, scriptTransform);
            // Generate a new test pipeline...
            // 
            PipelineMeta pipelineMeta = new PipelineMeta();
            pipelineMeta.setName(wTransformName.getText() + " - PREVIEW");
            pipelineMeta.addTransform(genTransform);
            pipelineMeta.addTransform(scriptTransform);
            pipelineMeta.addPipelineHop(hop);
            // OK, now we ask the user to edit this dialog...
            // 
            // Now run this pipeline and grab the results...
            // 
            PipelinePreviewProgressDialog progressDialog = new PipelinePreviewProgressDialog(shell, variables, pipelineMeta, new String[] { scriptTransformName }, new int[] { Const.toInt(genMeta.getRowLimit(), 10) });
            progressDialog.open();
            Pipeline pipeline = progressDialog.getPipeline();
            String loggingText = progressDialog.getLoggingText();
            if (!progressDialog.isCancelled()) {
                if (pipeline.getResult() != null && pipeline.getResult().getNrErrors() > 0) {
                    EnterTextDialog etd = new EnterTextDialog(shell, BaseMessages.getString(PKG, "System.Dialog.PreviewError.Title"), BaseMessages.getString(PKG, "System.Dialog.PreviewError.Message"), loggingText, true);
                    etd.setReadOnly();
                    etd.open();
                }
            }
            IRowMeta previewRowsMeta = progressDialog.getPreviewRowsMeta(wTransformName.getText());
            List<Object[]> previewRows = progressDialog.getPreviewRows(wTransformName.getText());
            if (previewRowsMeta != null && previewRows != null && previewRows.size() > 0) {
                PreviewRowsDialog prd = new PreviewRowsDialog(shell, variables, SWT.NONE, wTransformName.getText(), previewRowsMeta, previewRows, loggingText);
                prd.open();
            }
        }
        return true;
    } catch (Exception e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "ScriptValuesDialogMod.TestFailed.DialogTitle"), BaseMessages.getString(PKG, "ScriptValuesDialogMod.TestFailed.DialogMessage"), e);
        return false;
    }
}