Examples with NeoConnection org.apache.hop.neo4j.shared.NeoConnection used on opensource projects

Example 1 with NeoConnection

use of org.apache.hop.neo4j.shared.NeoConnection in project hop by apache.

the class HopNeo4jPerspective method refreshResults.

private void refreshResults() {
    // See if logging is enabled
    // 
    ILogChannel log = hopGui.getLog();
    String searchName = wExecutions.getText();
    int amount = Const.toInt(wAmount.getText(), 50);
    boolean onlyRoot = wOnlyRoot.getSelection();
    try {
        final NeoConnection connection = findLoggingConnection();
        if (connection == null) {
            wUsedConnection.setText("");
            return;
        }
        wUsedConnection.setText(Const.NVL(connection.getName(), ""));
        log.logDetailed("Logging workflow information to Neo4j connection : " + connection.getName());
        Map<String, Object> resultsParameters = new HashMap<>();
        final StringBuilder resultsCypher = new StringBuilder();
        resultsCypher.append("MATCH(e:Execution) ");
        resultsCypher.append("WHERE e.type in [ 'PIPELINE', 'WORKFLOW' ] ");
        if (StringUtils.isNotEmpty(searchName)) {
            resultsCypher.append("AND e.name = $name ");
            resultsParameters.put("name", searchName);
        }
        if (onlyRoot) {
            resultsCypher.append("AND e.root = true ");
        }
        resultsCypher.append("RETURN e.id, e.name, e.type, e.linesRead, e.linesWritten, e.linesInput, e.linesOutput, e.linesRejected, e.errors,  e.executionStart, e.durationMs ");
        resultsCypher.append("ORDER BY e.executionStart desc ");
        resultsCypher.append("LIMIT " + amount);
        wResults.clearAll(false);
        try (Driver driver = connection.getDriver(log, hopGui.getVariables())) {
            try (Session session = connection.getSession(log, driver, hopGui.getVariables())) {
                session.readTransaction(tx -> {
                    Result result = tx.run(resultsCypher.toString(), resultsParameters);
                    while (result.hasNext()) {
                        Record record = result.next();
                        TableItem item = new TableItem(wResults.table, SWT.NONE);
                        int pos = 0;
                        Value vId = record.get(pos++);
                        item.setText(pos, Const.NVL(vId.asString(), ""));
                        Value vName = record.get(pos++);
                        item.setText(pos, Const.NVL(vName.asString(), ""));
                        Value vType = record.get(pos++);
                        item.setText(pos, Const.NVL(vType.asString(), ""));
                        Value vLinesRead = record.get(pos++);
                        item.setText(pos, Long.toString(vLinesRead.asLong(0)));
                        Value vLinesWritten = record.get(pos++);
                        item.setText(pos, Long.toString(vLinesWritten.asLong(0)));
                        Value vLinesInput = record.get(pos++);
                        item.setText(pos, Long.toString(vLinesInput.asLong(0)));
                        Value vLinesOutput = record.get(pos++);
                        item.setText(pos, Long.toString(vLinesOutput.asLong(0)));
                        Value vLinesRejected = record.get(pos++);
                        item.setText(pos, Long.toString(vLinesRejected.asLong(0)));
                        Value vErrors = record.get(pos++);
                        long errors = vErrors.asLong(0);
                        item.setText(pos, Long.toString(vErrors.asLong(0)));
                        Value vExecutionStart = record.get(pos++);
                        item.setText(pos, Const.NVL(vExecutionStart.asString(), "").replace("T", " "));
                        Value vDurationMs = record.get(pos++);
                        String durationHMS = LoggingCore.getFancyDurationFromMs(Long.valueOf(vDurationMs.asLong(0)));
                        item.setText(pos, durationHMS);
                        if (errors != 0) {
                            item.setBackground(errorLineBackground);
                        }
                    }
                    wResults.removeEmptyRows();
                    wResults.setRowNums();
                    wResults.optWidth(true);
                    return null;
                });
                // Also populate the executions combo box for pipelines and workflows
                // 
                String execCypher = "match(e:Execution) where e.type in ['PIPELINE', 'WORKFLOW'] return distinct e.name order by e.name";
                session.readTransaction(tx -> {
                    List<String> list = new ArrayList<>();
                    Result result = tx.run(execCypher);
                    while (result.hasNext()) {
                        Record record = result.next();
                        Value value = record.get(0);
                        list.add(value.asString());
                    }
                    wExecutions.setItems(list.toArray(new String[0]));
                    return null;
                });
            }
        } finally {
            wExecutions.setText(Const.NVL(searchName, ""));
        }
    } catch (Throwable e) {
        new ErrorDialog(hopGui.getShell(), BaseMessages.getString(PKG, "Neo4jPerspectiveDialog.ErrorSearching.Dialog.Header"), BaseMessages.getString(PKG, "Neo4jPerspectiveDialog.ErrorSearching.Dialog.Message"), e);
    }
}

Example 2 with NeoConnection

use of org.apache.hop.neo4j.shared.NeoConnection in project hop by apache.

the class HopNeo4jPerspective method analyze.

/**
 * Analyze a log record entry when a user clicks on it.
 *
 * @param event
 */
private void analyze(Event event) {
    ILogChannel log = hopGui.getLog();
    System.out.println("Analyze");
    if (!(event.item instanceof TableItem)) {
        return;
    }
    TableItem item = (TableItem) event.item;
    String id = item.getText(1);
    String name = item.getText(2);
    String type = item.getText(3);
    int errors = Const.toInt(item.getText(9), -1);
    try {
        final NeoConnection connection = findLoggingConnection();
        if (connection == null) {
            return;
        }
        log.logDetailed("Logging workflow information to Neo4j connection : " + connection.getName());
        try (Driver driver = connection.getDriver(log, hopGui.getVariables())) {
            try (Session session = connection.getSession(log, driver, hopGui.getVariables())) {
                analyzeLogging(session, id, name, type);
                List<List<HistoryResult>> shortestPaths = analyzeErrorLineage(session, id, name, type, errors);
                analyzeCypherStatements(connection, session, id, name, type, errors, shortestPaths);
            }
        }
    } catch (Exception e) {
        new ErrorDialog(hopGui.getShell(), BaseMessages.getString(PKG, "Neo4jPerspectiveDialog.ErrorAnalyze.Dialog.Header"), BaseMessages.getString(PKG, "Neo4jPerspectiveDialog.ErrorAnalyze.Dialog.Message"), e);
    }
}

Example 3 with NeoConnection

use of org.apache.hop.neo4j.shared.NeoConnection in project hop by apache.

the class HopNeo4jPerspective method open.

private void open(Event event) {
    try {
        NeoConnection connection = findLoggingConnection();
        if (connection == null) {
            return;
        }
        // See if there is a selected line in the error tree...
        // 
        TreeItem[] treeSelection = wTree.getSelection();
        if (treeSelection != null || treeSelection.length > 0) {
            TreeItem treeItem = treeSelection[0];
            String id = treeItem.getText(1);
            if (StringUtils.isNotEmpty(id)) {
                openItem(treeItem);
                return;
            }
        }
        // Fallback is opening the selected root workflow or pipeline
        // 
        TableItem[] resultsSelection = wResults.table.getSelection();
        if (resultsSelection == null || resultsSelection.length == 0) {
            return;
        }
        TableItem tableItem = resultsSelection[0];
        String id = tableItem.getText(1);
        String name = tableItem.getText(2);
        String type = tableItem.getText(3);
        openItem(connection, id, name, type);
    } catch (Exception e) {
        new ErrorDialog(hopGui.getShell(), BaseMessages.getString(PKG, "Neo4jPerspectiveDialog.ErrorOpeningPipeline.Dialog.Header"), "Neo4jPerspectiveDialog.ErrorOpeningPipeline.Dialog.Message", e);
    }
}

Example 4 with NeoConnection

use of org.apache.hop.neo4j.shared.NeoConnection in project hop by apache.

the class PipelineLoggingExtensionPoint method logPipelineMetadata.

private void logPipelineMetadata(final ILogChannel log, final Session session, final NeoConnection connection, final IPipelineEngine<PipelineMeta> pipeline) throws HopException {
    log.logDetailed("Logging pipeline metadata to Neo4j connection : " + connection.getName());
    final PipelineMeta pipelineMeta = pipeline.getPipelineMeta();
    synchronized (session) {
        session.writeTransaction((TransactionWork<Void>) transaction -> {
            try {
                Map<String, Object> transPars = new HashMap<>();
                transPars.put("pipelineName", pipelineMeta.getName());
                transPars.put("description", pipelineMeta.getDescription());
                transPars.put("filename", pipelineMeta.getFilename());
                StringBuilder transCypher = new StringBuilder();
                transCypher.append("MERGE (pipeline:Pipeline { name : $pipelineName } ) ");
                transCypher.append("SET pipeline.filename = $filename, pipeline.description = $description ");
                transaction.run(transCypher.toString(), transPars);
                log.logDetailed("Pipeline cypher : " + transCypher);
                for (TransformMeta transformMeta : pipelineMeta.getTransforms()) {
                    Map<String, Object> transformPars = new HashMap<>();
                    transformPars.put("pipelineName", pipelineMeta.getName());
                    transformPars.put("transformName", transformMeta.getName());
                    transformPars.put("description", transformMeta.getDescription());
                    transformPars.put("pluginId", transformMeta.getPluginId());
                    transformPars.put("copies", transformMeta.getCopies(pipeline));
                    transformPars.put("locationX", transformMeta.getLocation().x);
                    transformPars.put("locationY", transformMeta.getLocation().y);
                    StringBuilder transformCypher = new StringBuilder();
                    transformCypher.append("MATCH (pipeline:Pipeline { name : $pipelineName } ) ");
                    transformCypher.append("MERGE (transform:Transform { pipelineName : $pipelineName, name : $transformName } ) ");
                    transformCypher.append("SET ");
                    transformCypher.append("  transform.description = $description ");
                    transformCypher.append(", transform.pluginId = $pluginId ");
                    transformCypher.append(", transform.copies = $copies ");
                    transformCypher.append(", transform.locationX = $locationX ");
                    transformCypher.append(", transform.locationY = $locationY ");
                    transformCypher.append("MERGE (transform)-[rel:TRANSFORM_OF_PIPELINE]->(pipeline) ");
                    log.logDetailed("Transform '" + transformMeta.getName() + "' cypher : " + transformCypher);
                    transaction.run(transformCypher.toString(), transformPars);
                }
                for (int i = 0; i < pipelineMeta.nrPipelineHops(); i++) {
                    PipelineHopMeta hopMeta = pipelineMeta.getPipelineHop(i);
                    Map<String, Object> hopPars = new HashMap<>();
                    hopPars.put("fromTransform", hopMeta.getFromTransform().getName());
                    hopPars.put("toTransform", hopMeta.getToTransform().getName());
                    hopPars.put("pipelineName", pipelineMeta.getName());
                    StringBuilder hopCypher = new StringBuilder();
                    hopCypher.append("MATCH (from:Transform { pipelineName : $pipelineName, name : $fromTransform }) ");
                    hopCypher.append("MATCH (to:Transform { pipelineName : $pipelineName, name : $toTransform }) ");
                    hopCypher.append("MERGE (from)-[rel:PRECEDES]->(to) ");
                    transaction.run(hopCypher.toString(), hopPars);
                }
                transaction.commit();
            } catch (Exception e) {
                transaction.rollback();
                log.logError("Error logging pipeline metadata", e);
            }
            return null;
        });
    }
}

Example 5 with NeoConnection

use of org.apache.hop.neo4j.shared.NeoConnection in project hop by apache.

the class GraphOutput method init.

@Override
public boolean init() {
    try {
        if (!meta.isReturningGraph()) {
            // 
            if (StringUtils.isEmpty(meta.getConnectionName())) {
                log.logError("You need to specify a Neo4j connection to use in this transform");
                return false;
            }
            IHopMetadataSerializer<NeoConnection> serializer = metadataProvider.getSerializer(NeoConnection.class);
            data.neoConnection = serializer.load(meta.getConnectionName());
            if (data.neoConnection == null) {
                log.logError("Connection '" + meta.getConnectionName() + "' could not be found in the metadata : " + metadataProvider.getDescription());
                return false;
            }
            try {
                data.driver = data.neoConnection.getDriver(log, this);
                data.session = data.neoConnection.getSession(log, data.driver, this);
                data.version4 = data.neoConnection.isVersion4();
            } catch (Exception e) {
                log.logError("Unable to get or create Neo4j database driver for database '" + data.neoConnection.getName() + "'", e);
                return false;
            }
            data.batchSize = Const.toLong(resolve(meta.getBatchSize()), 1);
        }
        if (StringUtils.isEmpty(meta.getModel())) {
            logError("No model name is specified");
            return false;
        }
        IHopMetadataSerializer<GraphModel> modelSerializer = metadataProvider.getSerializer(GraphModel.class);
        data.graphModel = modelSerializer.load(meta.getModel());
        if (data.graphModel == null) {
            logError("Model '" + meta.getModel() + "' could not be found!");
            return false;
        }
        // Perform a few sanity checks...
        // 
        data.graphModel.validateIntegrity();
        data.modelValidator = null;
        if (meta.isValidatingAgainstModel()) {
            // Validate the model...
            // 
            List<NodeProperty> usedNodeProperties = findUsedNodeProperties();
            data.modelValidator = new ModelValidator(data.graphModel, usedNodeProperties);
            int nrErrors = data.modelValidator.validateBeforeLoad(log, data.session);
            if (nrErrors > 0) {
                // There were validation errors, we can stop here...
                log.logError("Validation against graph model '" + data.graphModel.getName() + "' failed with " + nrErrors + " errors.");
                return false;
            } else {
                log.logBasic("Validation against graph model '" + data.graphModel.getName() + "' was successful.");
            }
        }
    } catch (HopException e) {
        log.logError("Could not find Neo4j connection'" + meta.getConnectionName() + "'", e);
        return false;
    }
    return super.init();
}