Examples with Pipeline org.apache.hop.pipeline.Pipeline used on opensource projects

Example 1 with Pipeline

use of org.apache.hop.pipeline.Pipeline in project hop by apache.

the class TransformCreatedXp method callExtensionPoint.

@Override
public void callExtensionPoint(ILogChannel log, IVariables variables, TransformInitThread initThread) throws HopException {
    ITransform transform = initThread.getCombi().transform;
    Pipeline pipeline = initThread.getPipeline();
    String connectionGroup = (String) pipeline.getExtensionDataMap().get(Const.CONNECTION_GROUP);
    if (connectionGroup != null) {
        // Pass the value down to the transform...
        // We do this before the transform initializes and perhaps asks for a new connection
        // 
        transform.getExtensionDataMap().put(Const.CONNECTION_GROUP, connectionGroup);
    }
}

Example 2 with Pipeline

use of org.apache.hop.pipeline.Pipeline in project hop by apache.

the class CypherDialog method preview.

private synchronized void preview() {
    CypherMeta oneMeta = new CypherMeta();
    this.getInfo(oneMeta);
    PipelineMeta previewMeta = PipelinePreviewFactory.generatePreviewPipeline(HopGui.getInstance().getMetadataProvider(), oneMeta, this.wTransformName.getText());
    EnterNumberDialog numberDialog = new EnterNumberDialog(this.shell, this.props.getDefaultPreviewSize(), BaseMessages.getString(PKG, "CypherDialog.PreviewSize.DialogTitle"), BaseMessages.getString(PKG, "CypherDialog.PreviewSize.DialogMessage"));
    int previewSize = numberDialog.open();
    if (previewSize > 0) {
        PipelinePreviewProgressDialog progressDialog = new PipelinePreviewProgressDialog(this.shell, variables, previewMeta, new String[] { this.wTransformName.getText() }, new int[] { previewSize });
        progressDialog.open();
        Pipeline pipeline = progressDialog.getPipeline();
        String loggingText = progressDialog.getLoggingText();
        if (!progressDialog.isCancelled() && pipeline.getResult() != null && pipeline.getResult().getNrErrors() > 0L) {
            EnterTextDialog etd = new EnterTextDialog(this.shell, BaseMessages.getString(PKG, "System.Dialog.PreviewError.Title", new String[0]), BaseMessages.getString(PKG, "System.Dialog.PreviewError.Message", new String[0]), loggingText, true);
            etd.setReadOnly();
            etd.open();
        }
        PreviewRowsDialog prd = new PreviewRowsDialog(this.shell, variables, 0, this.wTransformName.getText(), progressDialog.getPreviewRowsMeta(this.wTransformName.getText()), progressDialog.getPreviewRows(this.wTransformName.getText()), loggingText);
        prd.open();
    }
}

Example 3 with Pipeline

use of org.apache.hop.pipeline.Pipeline in project hop by apache.

the class PipelineLoggingExtensionPoint method logPipelineMetadata.

private void logPipelineMetadata(final ILogChannel log, final Session session, final NeoConnection connection, final IPipelineEngine<PipelineMeta> pipeline) throws HopException {
    log.logDetailed("Logging pipeline metadata to Neo4j connection : " + connection.getName());
    final PipelineMeta pipelineMeta = pipeline.getPipelineMeta();
    synchronized (session) {
        session.writeTransaction((TransactionWork<Void>) transaction -> {
            try {
                Map<String, Object> transPars = new HashMap<>();
                transPars.put("pipelineName", pipelineMeta.getName());
                transPars.put("description", pipelineMeta.getDescription());
                transPars.put("filename", pipelineMeta.getFilename());
                StringBuilder transCypher = new StringBuilder();
                transCypher.append("MERGE (pipeline:Pipeline { name : $pipelineName } ) ");
                transCypher.append("SET pipeline.filename = $filename, pipeline.description = $description ");
                transaction.run(transCypher.toString(), transPars);
                log.logDetailed("Pipeline cypher : " + transCypher);
                for (TransformMeta transformMeta : pipelineMeta.getTransforms()) {
                    Map<String, Object> transformPars = new HashMap<>();
                    transformPars.put("pipelineName", pipelineMeta.getName());
                    transformPars.put("transformName", transformMeta.getName());
                    transformPars.put("description", transformMeta.getDescription());
                    transformPars.put("pluginId", transformMeta.getPluginId());
                    transformPars.put("copies", transformMeta.getCopies(pipeline));
                    transformPars.put("locationX", transformMeta.getLocation().x);
                    transformPars.put("locationY", transformMeta.getLocation().y);
                    StringBuilder transformCypher = new StringBuilder();
                    transformCypher.append("MATCH (pipeline:Pipeline { name : $pipelineName } ) ");
                    transformCypher.append("MERGE (transform:Transform { pipelineName : $pipelineName, name : $transformName } ) ");
                    transformCypher.append("SET ");
                    transformCypher.append("  transform.description = $description ");
                    transformCypher.append(", transform.pluginId = $pluginId ");
                    transformCypher.append(", transform.copies = $copies ");
                    transformCypher.append(", transform.locationX = $locationX ");
                    transformCypher.append(", transform.locationY = $locationY ");
                    transformCypher.append("MERGE (transform)-[rel:TRANSFORM_OF_PIPELINE]->(pipeline) ");
                    log.logDetailed("Transform '" + transformMeta.getName() + "' cypher : " + transformCypher);
                    transaction.run(transformCypher.toString(), transformPars);
                }
                for (int i = 0; i < pipelineMeta.nrPipelineHops(); i++) {
                    PipelineHopMeta hopMeta = pipelineMeta.getPipelineHop(i);
                    Map<String, Object> hopPars = new HashMap<>();
                    hopPars.put("fromTransform", hopMeta.getFromTransform().getName());
                    hopPars.put("toTransform", hopMeta.getToTransform().getName());
                    hopPars.put("pipelineName", pipelineMeta.getName());
                    StringBuilder hopCypher = new StringBuilder();
                    hopCypher.append("MATCH (from:Transform { pipelineName : $pipelineName, name : $fromTransform }) ");
                    hopCypher.append("MATCH (to:Transform { pipelineName : $pipelineName, name : $toTransform }) ");
                    hopCypher.append("MERGE (from)-[rel:PRECEDES]->(to) ");
                    transaction.run(hopCypher.toString(), hopPars);
                }
                transaction.commit();
            } catch (Exception e) {
                transaction.rollback();
                log.logError("Error logging pipeline metadata", e);
            }
            return null;
        });
    }
}

Example 4 with Pipeline

use of org.apache.hop.pipeline.Pipeline in project hop by apache.

the class UserDefinedJavaClassDialog method test.

private boolean test() {
    PluginRegistry registry = PluginRegistry.getInstance();
    String scriptTransformName = wTransformName.getText();
    if (!checkForTransformClass()) {
        return false;
    }
    // Create a transform with the information in this dialog
    UserDefinedJavaClassMeta udjcMeta = new UserDefinedJavaClassMeta();
    getInfo(udjcMeta);
    try {
        // First, before we get into the trial run, just see if the classes
        // all compile.
        udjcMeta.cookClasses();
        if (udjcMeta.cookErrors.size() == 1) {
            Exception e = udjcMeta.cookErrors.get(0);
            new ErrorDialog(shell, "Error during class compilation", e.toString(), e);
            return false;
        } else if (udjcMeta.cookErrors.size() > 1) {
            Exception e = udjcMeta.cookErrors.get(0);
            new ErrorDialog(shell, "Errors during class compilation", String.format("Multiple errors during class compilation. First error:\n%s", e.toString()), e);
            return false;
        }
        // What fields are coming into the transform?
        IRowMeta rowMeta = pipelineMeta.getPrevTransformFields(variables, transformName).clone();
        if (rowMeta != null) {
            // time
            if (genMeta == null) {
                genMeta = new RowGeneratorMeta();
                genMeta.setRowLimit("10");
                // CHECKSTYLE:Indentation:OFF
                for (int i = 0; i < rowMeta.size(); i++) {
                    IValueMeta valueMeta = rowMeta.getValueMeta(i);
                    if (valueMeta.isStorageBinaryString()) {
                        valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL);
                    }
                    GeneratorField field = new GeneratorField();
                    field.setName(valueMeta.getName());
                    field.setType(valueMeta.getTypeDesc());
                    field.setLength(valueMeta.getLength());
                    field.setPrecision(valueMeta.getPrecision());
                    field.setCurrency(valueMeta.getCurrencySymbol());
                    field.setDecimal(valueMeta.getDecimalSymbol());
                    field.setGroup(valueMeta.getGroupingSymbol());
                    String string = null;
                    switch(valueMeta.getType()) {
                        case IValueMeta.TYPE_DATE:
                            field.setFormat("yyyy/MM/dd HH:mm:ss");
                            valueMeta.setConversionMask(field.getFormat());
                            string = valueMeta.getString(new Date());
                            break;
                        case IValueMeta.TYPE_STRING:
                            string = "test value test value";
                            break;
                        case IValueMeta.TYPE_INTEGER:
                            field.setFormat("#");
                            valueMeta.setConversionMask(field.getFormat());
                            string = valueMeta.getString(Long.valueOf(0L));
                            break;
                        case IValueMeta.TYPE_NUMBER:
                            field.setFormat("#.#");
                            valueMeta.setConversionMask(field.getFormat());
                            string = valueMeta.getString(Double.valueOf(0.0D));
                            break;
                        case IValueMeta.TYPE_BIGNUMBER:
                            field.setFormat("#.#");
                            valueMeta.setConversionMask(field.getFormat());
                            string = valueMeta.getString(BigDecimal.ZERO);
                            break;
                        case IValueMeta.TYPE_BOOLEAN:
                            string = valueMeta.getString(Boolean.TRUE);
                            break;
                        case IValueMeta.TYPE_BINARY:
                            string = valueMeta.getString(new byte[] { 65, 66, 67, 68, 69, 70, 71, 72, 73, 74 });
                            break;
                        default:
                            break;
                    }
                    field.setValue(string);
                    genMeta.getFields().add(field);
                }
            }
            TransformMeta genTransform = new TransformMeta(registry.getPluginId(TransformPluginType.class, genMeta), "## TEST DATA ##", genMeta);
            genTransform.setLocation(50, 50);
            TransformMeta scriptTransform = new TransformMeta(registry.getPluginId(TransformPluginType.class, udjcMeta), Const.NVL(scriptTransformName, "## SCRIPT ##"), udjcMeta);
            scriptTransformName = scriptTransform.getName();
            scriptTransform.setLocation(150, 50);
            // Create a hop between both transforms...
            // 
            PipelineHopMeta hop = new PipelineHopMeta(genTransform, scriptTransform);
            // Generate a new test pipeline...
            // 
            PipelineMeta pipelineMeta = new PipelineMeta();
            pipelineMeta.setName(wTransformName.getText() + " - PREVIEW");
            pipelineMeta.addTransform(genTransform);
            pipelineMeta.addTransform(scriptTransform);
            pipelineMeta.addPipelineHop(hop);
            // OK, now we ask the user to edit this dialog...
            // 
            // Now run this pipeline and grab the results...
            // 
            PipelinePreviewProgressDialog progressDialog = new PipelinePreviewProgressDialog(shell, variables, pipelineMeta, new String[] { scriptTransformName }, new int[] { Const.toInt(genMeta.getRowLimit(), 10) });
            progressDialog.open();
            Pipeline pipeline = progressDialog.getPipeline();
            String loggingText = progressDialog.getLoggingText();
            if (!progressDialog.isCancelled()) {
                if (pipeline.getResult() != null && pipeline.getResult().getNrErrors() > 0) {
                    EnterTextDialog etd = new EnterTextDialog(shell, BaseMessages.getString("System.Dialog.PreviewError.Title"), BaseMessages.getString("System.Dialog.PreviewError.Message"), loggingText, true);
                    etd.setReadOnly();
                    etd.open();
                }
            }
            IRowMeta previewRowsMeta = progressDialog.getPreviewRowsMeta(wTransformName.getText());
            List<Object[]> previewRows = progressDialog.getPreviewRows(wTransformName.getText());
            if (previewRowsMeta != null && previewRows != null && previewRows.size() > 0) {
                PreviewRowsDialog prd = new PreviewRowsDialog(shell, variables, SWT.NONE, wTransformName.getText(), previewRowsMeta, previewRows, loggingText);
                prd.open();
            }
            return true;
        } else {
            throw new HopException(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.Exception.CouldNotGetFields"));
        }
    } catch (Exception e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.TestFailed.DialogTitle"), BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.TestFailed.DialogMessage"), e);
        return false;
    }
}

Example 5 with Pipeline

use of org.apache.hop.pipeline.Pipeline in project hop by apache.

the class ScriptValueAddFunctions_SetVariableScopeTest method setSystemScopeVariable_NoParent.

@Test
public void setSystemScopeVariable_NoParent() {
    Pipeline pipeline = createPipeline();
    Assert.assertNull(System.getProperty(VARIABLE_NAME));
    try {
        ScriptValuesAddedFunctions.setSystemScopeVariable(pipeline, VARIABLE_NAME, VARIABLE_VALUE);
        Assert.assertEquals(System.getProperty(VARIABLE_NAME), VARIABLE_VALUE);
        verify(pipeline).setVariable(eq(VARIABLE_NAME), eq(VARIABLE_VALUE));
    } finally {
        System.clearProperty(VARIABLE_NAME);
    }
}