Examples with PipelineHopMeta org.apache.hop.pipeline.PipelineHopMeta used on opensource projects

Example 1 with PipelineHopMeta

use of org.apache.hop.pipeline.PipelineHopMeta in project hop by apache.

the class PipelineLoggingExtensionPoint method logPipelineMetadata.

private void logPipelineMetadata(final ILogChannel log, final Session session, final NeoConnection connection, final IPipelineEngine<PipelineMeta> pipeline) throws HopException {
    log.logDetailed("Logging pipeline metadata to Neo4j connection : " + connection.getName());
    final PipelineMeta pipelineMeta = pipeline.getPipelineMeta();
    synchronized (session) {
        session.writeTransaction((TransactionWork<Void>) transaction -> {
            try {
                Map<String, Object> transPars = new HashMap<>();
                transPars.put("pipelineName", pipelineMeta.getName());
                transPars.put("description", pipelineMeta.getDescription());
                transPars.put("filename", pipelineMeta.getFilename());
                StringBuilder transCypher = new StringBuilder();
                transCypher.append("MERGE (pipeline:Pipeline { name : $pipelineName } ) ");
                transCypher.append("SET pipeline.filename = $filename, pipeline.description = $description ");
                transaction.run(transCypher.toString(), transPars);
                log.logDetailed("Pipeline cypher : " + transCypher);
                for (TransformMeta transformMeta : pipelineMeta.getTransforms()) {
                    Map<String, Object> transformPars = new HashMap<>();
                    transformPars.put("pipelineName", pipelineMeta.getName());
                    transformPars.put("transformName", transformMeta.getName());
                    transformPars.put("description", transformMeta.getDescription());
                    transformPars.put("pluginId", transformMeta.getPluginId());
                    transformPars.put("copies", transformMeta.getCopies(pipeline));
                    transformPars.put("locationX", transformMeta.getLocation().x);
                    transformPars.put("locationY", transformMeta.getLocation().y);
                    StringBuilder transformCypher = new StringBuilder();
                    transformCypher.append("MATCH (pipeline:Pipeline { name : $pipelineName } ) ");
                    transformCypher.append("MERGE (transform:Transform { pipelineName : $pipelineName, name : $transformName } ) ");
                    transformCypher.append("SET ");
                    transformCypher.append("  transform.description = $description ");
                    transformCypher.append(", transform.pluginId = $pluginId ");
                    transformCypher.append(", transform.copies = $copies ");
                    transformCypher.append(", transform.locationX = $locationX ");
                    transformCypher.append(", transform.locationY = $locationY ");
                    transformCypher.append("MERGE (transform)-[rel:TRANSFORM_OF_PIPELINE]->(pipeline) ");
                    log.logDetailed("Transform '" + transformMeta.getName() + "' cypher : " + transformCypher);
                    transaction.run(transformCypher.toString(), transformPars);
                }
                for (int i = 0; i < pipelineMeta.nrPipelineHops(); i++) {
                    PipelineHopMeta hopMeta = pipelineMeta.getPipelineHop(i);
                    Map<String, Object> hopPars = new HashMap<>();
                    hopPars.put("fromTransform", hopMeta.getFromTransform().getName());
                    hopPars.put("toTransform", hopMeta.getToTransform().getName());
                    hopPars.put("pipelineName", pipelineMeta.getName());
                    StringBuilder hopCypher = new StringBuilder();
                    hopCypher.append("MATCH (from:Transform { pipelineName : $pipelineName, name : $fromTransform }) ");
                    hopCypher.append("MATCH (to:Transform { pipelineName : $pipelineName, name : $toTransform }) ");
                    hopCypher.append("MERGE (from)-[rel:PRECEDES]->(to) ");
                    transaction.run(hopCypher.toString(), hopPars);
                }
                transaction.commit();
            } catch (Exception e) {
                transaction.rollback();
                log.logError("Error logging pipeline metadata", e);
            }
            return null;
        });
    }
}

Example 2 with PipelineHopMeta

use of org.apache.hop.pipeline.PipelineHopMeta in project hop by apache.

the class UserDefinedJavaClassDialog method test.

private boolean test() {
    PluginRegistry registry = PluginRegistry.getInstance();
    String scriptTransformName = wTransformName.getText();
    if (!checkForTransformClass()) {
        return false;
    }
    // Create a transform with the information in this dialog
    UserDefinedJavaClassMeta udjcMeta = new UserDefinedJavaClassMeta();
    getInfo(udjcMeta);
    try {
        // First, before we get into the trial run, just see if the classes
        // all compile.
        udjcMeta.cookClasses();
        if (udjcMeta.cookErrors.size() == 1) {
            Exception e = udjcMeta.cookErrors.get(0);
            new ErrorDialog(shell, "Error during class compilation", e.toString(), e);
            return false;
        } else if (udjcMeta.cookErrors.size() > 1) {
            Exception e = udjcMeta.cookErrors.get(0);
            new ErrorDialog(shell, "Errors during class compilation", String.format("Multiple errors during class compilation. First error:\n%s", e.toString()), e);
            return false;
        }
        // What fields are coming into the transform?
        IRowMeta rowMeta = pipelineMeta.getPrevTransformFields(variables, transformName).clone();
        if (rowMeta != null) {
            // time
            if (genMeta == null) {
                genMeta = new RowGeneratorMeta();
                genMeta.setRowLimit("10");
                // CHECKSTYLE:Indentation:OFF
                for (int i = 0; i < rowMeta.size(); i++) {
                    IValueMeta valueMeta = rowMeta.getValueMeta(i);
                    if (valueMeta.isStorageBinaryString()) {
                        valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL);
                    }
                    GeneratorField field = new GeneratorField();
                    field.setName(valueMeta.getName());
                    field.setType(valueMeta.getTypeDesc());
                    field.setLength(valueMeta.getLength());
                    field.setPrecision(valueMeta.getPrecision());
                    field.setCurrency(valueMeta.getCurrencySymbol());
                    field.setDecimal(valueMeta.getDecimalSymbol());
                    field.setGroup(valueMeta.getGroupingSymbol());
                    String string = null;
                    switch(valueMeta.getType()) {
                        case IValueMeta.TYPE_DATE:
                            field.setFormat("yyyy/MM/dd HH:mm:ss");
                            valueMeta.setConversionMask(field.getFormat());
                            string = valueMeta.getString(new Date());
                            break;
                        case IValueMeta.TYPE_STRING:
                            string = "test value test value";
                            break;
                        case IValueMeta.TYPE_INTEGER:
                            field.setFormat("#");
                            valueMeta.setConversionMask(field.getFormat());
                            string = valueMeta.getString(Long.valueOf(0L));
                            break;
                        case IValueMeta.TYPE_NUMBER:
                            field.setFormat("#.#");
                            valueMeta.setConversionMask(field.getFormat());
                            string = valueMeta.getString(Double.valueOf(0.0D));
                            break;
                        case IValueMeta.TYPE_BIGNUMBER:
                            field.setFormat("#.#");
                            valueMeta.setConversionMask(field.getFormat());
                            string = valueMeta.getString(BigDecimal.ZERO);
                            break;
                        case IValueMeta.TYPE_BOOLEAN:
                            string = valueMeta.getString(Boolean.TRUE);
                            break;
                        case IValueMeta.TYPE_BINARY:
                            string = valueMeta.getString(new byte[] { 65, 66, 67, 68, 69, 70, 71, 72, 73, 74 });
                            break;
                        default:
                            break;
                    }
                    field.setValue(string);
                    genMeta.getFields().add(field);
                }
            }
            TransformMeta genTransform = new TransformMeta(registry.getPluginId(TransformPluginType.class, genMeta), "## TEST DATA ##", genMeta);
            genTransform.setLocation(50, 50);
            TransformMeta scriptTransform = new TransformMeta(registry.getPluginId(TransformPluginType.class, udjcMeta), Const.NVL(scriptTransformName, "## SCRIPT ##"), udjcMeta);
            scriptTransformName = scriptTransform.getName();
            scriptTransform.setLocation(150, 50);
            // Create a hop between both transforms...
            // 
            PipelineHopMeta hop = new PipelineHopMeta(genTransform, scriptTransform);
            // Generate a new test pipeline...
            // 
            PipelineMeta pipelineMeta = new PipelineMeta();
            pipelineMeta.setName(wTransformName.getText() + " - PREVIEW");
            pipelineMeta.addTransform(genTransform);
            pipelineMeta.addTransform(scriptTransform);
            pipelineMeta.addPipelineHop(hop);
            // OK, now we ask the user to edit this dialog...
            // 
            // Now run this pipeline and grab the results...
            // 
            PipelinePreviewProgressDialog progressDialog = new PipelinePreviewProgressDialog(shell, variables, pipelineMeta, new String[] { scriptTransformName }, new int[] { Const.toInt(genMeta.getRowLimit(), 10) });
            progressDialog.open();
            Pipeline pipeline = progressDialog.getPipeline();
            String loggingText = progressDialog.getLoggingText();
            if (!progressDialog.isCancelled()) {
                if (pipeline.getResult() != null && pipeline.getResult().getNrErrors() > 0) {
                    EnterTextDialog etd = new EnterTextDialog(shell, BaseMessages.getString("System.Dialog.PreviewError.Title"), BaseMessages.getString("System.Dialog.PreviewError.Message"), loggingText, true);
                    etd.setReadOnly();
                    etd.open();
                }
            }
            IRowMeta previewRowsMeta = progressDialog.getPreviewRowsMeta(wTransformName.getText());
            List<Object[]> previewRows = progressDialog.getPreviewRows(wTransformName.getText());
            if (previewRowsMeta != null && previewRows != null && previewRows.size() > 0) {
                PreviewRowsDialog prd = new PreviewRowsDialog(shell, variables, SWT.NONE, wTransformName.getText(), previewRowsMeta, previewRows, loggingText);
                prd.open();
            }
            return true;
        } else {
            throw new HopException(BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.Exception.CouldNotGetFields"));
        }
    } catch (Exception e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.TestFailed.DialogTitle"), BaseMessages.getString(PKG, "UserDefinedJavaClassDialog.TestFailed.DialogMessage"), e);
        return false;
    }
}

Example 3 with PipelineHopMeta

use of org.apache.hop.pipeline.PipelineHopMeta in project hop by apache.

the class PipelineLogEditor method createPipelineFile.

/**
 * Create a new pipeline file: ask the user for a name. Add a standard transform and a dummy to
 * show how it works.
 *
 * @param parent
 */
private void createPipelineFile(Composite parent) {
    try {
        PipelineMeta pipelineMeta = new PipelineMeta();
        // Add a Pipeline Logging transform...
        // 
        PipelineLoggingMeta pipelineLoggingMeta = new PipelineLoggingMeta();
        pipelineLoggingMeta.setLoggingTransforms(true);
        TransformMeta pipelineLogging = new TransformMeta("Pipeline Logging", pipelineLoggingMeta);
        pipelineLogging.setLocation(200, 150);
        pipelineMeta.addTransform(pipelineLogging);
        // Add a dummy
        // 
        DummyMeta dummyMeta = new DummyMeta();
        TransformMeta dummy = new TransformMeta("Save logging here", dummyMeta);
        dummy.setLocation(500, 150);
        pipelineMeta.addTransform(dummy);
        // Add a hop between both transforms...
        // 
        pipelineMeta.addPipelineHop(new PipelineHopMeta(pipelineLogging, dummy));
        // Save it...
        // 
        HopPipelineFileType<PipelineMeta> type = new HopPipelineFileType<>();
        String filename = BaseDialog.presentFileDialog(// save
        true, parent.getShell(), wFilename, manager.getVariables(), type.getFilterExtensions(), type.getFilterNames(), true);
        if (filename != null) {
            // User specified a pipeline filename
            // 
            String realFilename = manager.getVariables().resolve(filename);
            pipelineMeta.setFilename(realFilename);
            pipelineMeta.clearChanged();
            HopDataOrchestrationPerspective perspective = HopGui.getDataOrchestrationPerspective();
            // Switch to the perspective
            // 
            perspective.activate();
            // Open it in the Hop GUI
            // 
            HopGui.getDataOrchestrationPerspective().addPipeline(hopGui, pipelineMeta, type);
            // Save the file
            hopGui.fileDelegate.fileSave();
        }
    } catch (Exception e) {
        new ErrorDialog(parent.getShell(), "Error", "Error creating pipeline", e);
    }
}

Example 4 with PipelineHopMeta

use of org.apache.hop.pipeline.PipelineHopMeta in project hop by apache.

the class PipelineProbeEditor method createPipelineFile.

/**
 * Create a new pipeline file: ask the user for a name. Add a standard transform and a dummy to
 * show how it works.
 *
 * @param parent
 */
private void createPipelineFile(Composite parent) {
    try {
        PipelineMeta pipelineMeta = new PipelineMeta();
        // Add a Pipeline Data Probe transform...
        // 
        PipelineDataProbeMeta pipelineDataProbeMeta = new PipelineDataProbeMeta();
        pipelineDataProbeMeta.setLoggingTransforms(true);
        TransformMeta pipelineLogging = new TransformMeta("Pipeline Data Probe", pipelineDataProbeMeta);
        pipelineLogging.setLocation(200, 150);
        pipelineMeta.addTransform(pipelineLogging);
        // Add a dummy
        // 
        DummyMeta dummyMeta = new DummyMeta();
        TransformMeta dummy = new TransformMeta("Process values here", dummyMeta);
        dummy.setLocation(500, 150);
        pipelineMeta.addTransform(dummy);
        // Add a hop between both transforms...
        // 
        pipelineMeta.addPipelineHop(new PipelineHopMeta(pipelineLogging, dummy));
        // Save it...
        // 
        HopPipelineFileType<PipelineMeta> type = new HopPipelineFileType<>();
        String filename = BaseDialog.presentFileDialog(// save
        true, parent.getShell(), wFilename, manager.getVariables(), type.getFilterExtensions(), type.getFilterNames(), true);
        if (filename != null) {
            // User specified a pipeline filename
            // 
            String realFilename = manager.getVariables().resolve(filename);
            pipelineMeta.setFilename(realFilename);
            pipelineMeta.clearChanged();
            HopDataOrchestrationPerspective perspective = HopGui.getDataOrchestrationPerspective();
            // Switch to the perspective
            // 
            perspective.activate();
            // Open it in the Hop GUI
            // 
            HopGui.getDataOrchestrationPerspective().addPipeline(hopGui, pipelineMeta, type);
            // Save the file
            hopGui.fileDelegate.fileSave();
        }
    } catch (Exception e) {
        new ErrorDialog(parent.getShell(), "Error", "Error creating pipeline", e);
    }
}

Example 5 with PipelineHopMeta

use of org.apache.hop.pipeline.PipelineHopMeta in project hop by apache.

the class JsonOutputTest method test.

public String test(boolean compatibilityMode) throws Exception {
    HopEnvironment.init();
    // Create a new transformation...
    // 
    PipelineMeta pipelineMeta = new PipelineMeta();
    pipelineMeta.setName("testJsonOutput");
    PluginRegistry registry = PluginRegistry.getInstance();
    // create an injector transform
    String injectorTransformName = "injector transform";
    TransformMeta injectorTransform = TestUtilities.createInjectorTransform(injectorTransformName, registry);
    pipelineMeta.addTransform(injectorTransform);
    // create a row generator transform
    TransformMeta rowGeneratorTransform = createRowGeneratorTransform("Create rows for testJsonOutput1", registry);
    pipelineMeta.addTransform(rowGeneratorTransform);
    // create a PipelineHopMeta for injector and add it to the pipelineMeta
    PipelineHopMeta hop_injectoryRowGenerator = new PipelineHopMeta(injectorTransform, rowGeneratorTransform);
    pipelineMeta.addPipelineHop(hop_injectoryRowGenerator);
    // create the json output transform
    // but first lets get a filename
    String jsonFileName = TestUtilities.createEmptyTempFile("testJsonOutput1_");
    TransformMeta jsonOutputTransform = createJsonOutputTransform("json output transform", jsonFileName, registry);
    ((JsonOutputMeta) jsonOutputTransform.getTransform()).setCompatibilityMode(compatibilityMode);
    pipelineMeta.addTransform(jsonOutputTransform);
    // create a PipelineHopMeta for jsonOutputTransform and add it to the pipelineMeta
    PipelineHopMeta hop_RowGeneratorOutputTextFile = new PipelineHopMeta(rowGeneratorTransform, jsonOutputTransform);
    pipelineMeta.addPipelineHop(hop_RowGeneratorOutputTextFile);
    // Create a dummy transform and add it to the tranMeta
    String dummyTransformName = "dummy transform";
    TransformMeta dummyTransform = createDummyTransform(dummyTransformName, registry);
    pipelineMeta.addTransform(dummyTransform);
    // create a PipelineHopMeta for the
    PipelineHopMeta hopOutputJson_dummyTransform = new PipelineHopMeta(jsonOutputTransform, dummyTransform);
    pipelineMeta.addPipelineHop(hopOutputJson_dummyTransform);
    // Now execute the transformation...
    Pipeline pipeline = new LocalPipelineEngine(pipelineMeta);
    pipeline.prepareExecution();
    // Create a row collector and add it to the dummy transform interface
    IEngineComponent dummyITransform = pipeline.findComponent(dummyTransformName, 0);
    TransformRowsCollector dummyRowCollector = new TransformRowsCollector();
    dummyITransform.addRowListener(dummyRowCollector);
    // RowProducer rowProducer = pipeline.addRowProducer(injectorTransformName, 0);
    pipeline.startThreads();
    pipeline.waitUntilFinished();
    // get the results and return it
    File outputFile = new File(jsonFileName + ".js");
    String jsonStructure = FileUtils.readFileToString(outputFile);
    return jsonStructure;
}