Examples with DatabaseMeta org.apache.hop.core.database.DatabaseMeta used on opensource projects

Example 1 with DatabaseMeta

use of org.apache.hop.core.database.DatabaseMeta in project hop by apache.

the class WorkflowActionEvalTableContentTest method setUp.

@Before
public void setUp() throws Exception {
    MockDriver.registerInstance();
    IWorkflowEngine<WorkflowMeta> workflow = new LocalWorkflowEngine(new WorkflowMeta());
    action = new ActionEvalTableContent();
    workflow.getWorkflowMeta().addAction(new ActionMeta(action));
    action.setParentWorkflow(workflow);
    workflow.setStopped(false);
    DatabaseMeta dbMeta = new DatabaseMeta();
    dbMeta.setDatabaseType("mock-db");
    action.setDatabase(dbMeta);
    action.setVariable(Const.HOP_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_WORKFLOW_ACTIONS, "N");
}

Example 2 with DatabaseMeta

use of org.apache.hop.core.database.DatabaseMeta in project hop by apache.

the class ActionEvalTableContentDialog method getSql.

private void getSql() {
    DatabaseMeta inf = getWorkflowMeta().findDatabase(wConnection.getText());
    if (inf != null) {
        DatabaseExplorerDialog std = new DatabaseExplorerDialog(shell, SWT.NONE, variables, inf, getWorkflowMeta().getDatabases());
        if (std.open()) {
            String sql = "SELECT *" + Const.CR + "FROM " + inf.getQuotedSchemaTableCombination(variables, std.getSchemaName(), std.getTableName()) + Const.CR;
            wSql.setText(sql);
            MessageBox yn = new MessageBox(shell, SWT.YES | SWT.NO | SWT.CANCEL | SWT.ICON_QUESTION);
            yn.setMessage(BaseMessages.getString(PKG, "ActionEvalTableContent.IncludeFieldNamesInSQL"));
            yn.setText(BaseMessages.getString(PKG, "ActionEvalTableContent.DialogCaptionQuestion"));
            int id = yn.open();
            switch(id) {
                case SWT.CANCEL:
                    break;
                case SWT.NO:
                    wSql.setText(sql);
                    break;
                case SWT.YES:
                    Database db = new Database(loggingObject, variables, inf);
                    try {
                        db.connect();
                        IRowMeta fields = db.getQueryFields(sql, false);
                        if (fields != null) {
                            sql = "SELECT" + Const.CR;
                            for (int i = 0; i < fields.size(); i++) {
                                IValueMeta field = fields.getValueMeta(i);
                                if (i == 0) {
                                    sql += "  ";
                                } else {
                                    sql += ", ";
                                }
                                sql += inf.quoteField(field.getName()) + Const.CR;
                            }
                            sql += "FROM " + inf.getQuotedSchemaTableCombination(variables, std.getSchemaName(), std.getTableName()) + Const.CR;
                            wSql.setText(sql);
                        } else {
                            MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
                            mb.setMessage(BaseMessages.getString(PKG, "ActionEvalTableContent.ERROR_CouldNotRetrieveFields") + Const.CR + BaseMessages.getString(PKG, "ActionEvalTableContent.PerhapsNoPermissions"));
                            mb.setText(BaseMessages.getString(PKG, "ActionEvalTableContent.DialogCaptionError2"));
                            mb.open();
                        }
                    } catch (HopException e) {
                        MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
                        mb.setText(BaseMessages.getString(PKG, "ActionEvalTableContent.DialogCaptionError3"));
                        mb.setMessage(BaseMessages.getString(PKG, "ActionEvalTableContent.AnErrorOccurred") + Const.CR + e.getMessage());
                        mb.open();
                    } finally {
                        db.disconnect();
                    }
                    break;
                default:
                    break;
            }
        }
    } else {
        MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
        mb.setMessage(BaseMessages.getString(PKG, "ActionEvalTableContent.ConnectionNoLongerAvailable"));
        mb.setText(BaseMessages.getString(PKG, "ActionEvalTableContent.DialogCaptionError4"));
        mb.open();
    }
}

Example 3 with DatabaseMeta

use of org.apache.hop.core.database.DatabaseMeta in project hop by apache.

the class PipelineMetaModifier method getTestPipeline.

public PipelineMeta getTestPipeline(ILogChannel log, IVariables variables, IHopMetadataProvider metadataProvider) throws HopException {
    // OK, so now replace an input transform with a data set attached with an Injector transform...
    // However, we don't want to have the user see this so we need to copy pipeline.pipelineMeta
    // first...
    // 
    // Clone seems to has problems so we'll take the long (XML) way around...
    // 
    InputStream stream;
    try {
        stream = new ByteArrayInputStream(pipelineMeta.getXml(variables).getBytes(Const.XML_ENCODING));
    } catch (UnsupportedEncodingException e) {
        throw new HopException("Encoding error", e);
    }
    PipelineMeta copyPipelineMeta = new PipelineMeta(stream, metadataProvider, true, variables);
    // Pass the metadata references...
    // 
    copyPipelineMeta.setMetadataProvider(pipelineMeta.getMetadataProvider());
    // 
    for (PipelineUnitTestDatabaseReplacement dbReplacement : unitTest.getDatabaseReplacements()) {
        String sourceDatabaseName = variables.resolve(dbReplacement.getOriginalDatabaseName());
        String replacementDatabaseName = variables.resolve(dbReplacement.getReplacementDatabaseName());
        DatabaseMeta sourceDatabaseMeta = copyPipelineMeta.findDatabase(sourceDatabaseName);
        DatabaseMeta replacementDatabaseMeta = copyPipelineMeta.findDatabase(replacementDatabaseName);
        if (sourceDatabaseMeta == null) {
            throw new HopException("Unable to find source database connection '" + sourceDatabaseName + "', can not be replaced");
        }
        if (replacementDatabaseMeta == null) {
            throw new HopException("Unable to find replacement database connection '" + replacementDatabaseName + "', can not be used to replace");
        }
        if (log.isDetailed()) {
            log.logDetailed("Replaced database connection '" + sourceDatabaseName + "' with connection '" + replacementDatabaseName + "'");
        }
        sourceDatabaseMeta.replaceMeta(replacementDatabaseMeta);
    }
    // Replace all transforms with an Input Data Set marker with an Injector
    // Replace all transforms with a Golden Data Set marker with a Dummy
    // Apply the tweaks to the transforms:
    // - Bypass : replace with Dummy
    // - Remove : remove transform and all connected hops.
    // 
    // Loop over the original pipeline to allow us to safely modify the copy
    // 
    List<TransformMeta> transforms = pipelineMeta.getTransforms();
    for (TransformMeta transform : transforms) {
        TransformMeta transformMeta = copyPipelineMeta.findTransform(transform.getName());
        PipelineUnitTestSetLocation inputLocation = unitTest.findInputLocation(transformMeta.getName());
        PipelineUnitTestSetLocation goldenLocation = unitTest.findGoldenLocation(transformMeta.getName());
        PipelineUnitTestTweak transformTweak = unitTest.findTweak(transformMeta.getName());
        // 
        if (inputLocation != null) {
            handleInputDataSet(log, inputLocation, unitTest, pipelineMeta, transformMeta, metadataProvider);
        }
        // 
        if (goldenLocation != null) {
            handleGoldenDataSet(log, goldenLocation, transformMeta, metadataProvider);
        }
        if (transformTweak != null && transformTweak.getTweak() != null) {
            switch(transformTweak.getTweak()) {
                case NONE:
                    break;
                case REMOVE_TRANSFORM:
                    handleTweakRemoveTransform(log, copyPipelineMeta, transformMeta);
                    break;
                case BYPASS_TRANSFORM:
                    handleTweakBypassTransform(log, transformMeta);
                    break;
                default:
                    break;
            }
        }
    }
    return copyPipelineMeta;
}

Example 4 with DatabaseMeta

use of org.apache.hop.core.database.DatabaseMeta in project hop by apache.

the class InsertUpdateMeta method getSqlStatements.

@Override
public SqlStatement getSqlStatements(IVariables variables, PipelineMeta pipelineMeta, TransformMeta transformMeta, IRowMeta prev, IHopMetadataProvider metadataProvider) throws HopTransformException {
    SqlStatement sqlStatement = // default: nothing to do!
    new SqlStatement(transformMeta.getName(), null, null);
    String connectionName = variables.resolve(connection);
    if (StringUtils.isEmpty(connectionName)) {
        sqlStatement.setError(BaseMessages.getString(PKG, "InsertUpdateMeta.ReturnValue.NoConnectionDefined"));
        return sqlStatement;
    }
    DatabaseMeta databaseMeta;
    try {
        databaseMeta = metadataProvider.getSerializer(DatabaseMeta.class).load(connectionName);
        if (databaseMeta == null) {
            sqlStatement.setError("Error finding database connection " + connectionName + " in the metadata");
            return sqlStatement;
        }
    } catch (Exception e) {
        sqlStatement.setError("Error loading database connection " + connectionName + " from Hop metadata: " + Const.getSimpleStackTrace(e));
        return sqlStatement;
    }
    if (prev != null && prev.size() > 0) {
        String[] keyLookup = null;
        String[] keyStream = null;
        String[] updateLookup = null;
        String[] updateStream = null;
        if (insertUpdateLookupField.getLookupKeys().size() > 0) {
            keyLookup = new String[insertUpdateLookupField.getLookupKeys().size()];
            for (int i = 0; i < insertUpdateLookupField.getLookupKeys().size(); i++) {
                keyLookup[i] = insertUpdateLookupField.getLookupKeys().get(i).getKeyLookup();
            }
        }
        if (insertUpdateLookupField.getLookupKeys().size() > 0) {
            keyStream = new String[insertUpdateLookupField.getLookupKeys().size()];
            for (int i = 0; i < insertUpdateLookupField.getLookupKeys().size(); i++) {
                keyStream[i] = insertUpdateLookupField.getLookupKeys().get(i).getKeyStream();
            }
        }
        if (insertUpdateLookupField.getValueFields().size() > 0) {
            updateLookup = new String[insertUpdateLookupField.getValueFields().size()];
            for (int i = 0; i < insertUpdateLookupField.getValueFields().size(); i++) {
                updateLookup[i] = insertUpdateLookupField.getValueFields().get(i).getUpdateLookup();
            }
        }
        if (insertUpdateLookupField.getValueFields().size() > 0) {
            updateStream = new String[insertUpdateLookupField.getValueFields().size()];
            for (int i = 0; i < insertUpdateLookupField.getValueFields().size(); i++) {
                updateStream[i] = insertUpdateLookupField.getValueFields().get(i).getUpdateStream();
            }
        }
        IRowMeta tableFields = RowMetaUtils.getRowMetaForUpdate(prev, keyLookup, keyStream, updateLookup, updateStream);
        if (!Utils.isEmpty(insertUpdateLookupField.getTableName())) {
            Database db = new Database(loggingObject, variables, databaseMeta);
            try {
                db.connect();
                String schemaTable = databaseMeta.getQuotedSchemaTableCombination(variables, variables.resolve(insertUpdateLookupField.getSchemaName()), variables.resolve(insertUpdateLookupField.getTableName()));
                String crTable = db.getDDL(schemaTable, tableFields, null, false, null, true);
                String crIndex = "";
                String[] idxFields = null;
                if (keyLookup != null && keyLookup.length > 0) {
                    idxFields = new String[keyLookup.length];
                    System.arraycopy(keyLookup, 0, idxFields, 0, keyLookup.length);
                } else {
                    sqlStatement.setError(BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.MissingKeyFields"));
                }
                // Key lookup dimensions...
                if (idxFields != null && !db.checkIndexExists(variables.resolve(insertUpdateLookupField.getSchemaName()), variables.resolve(insertUpdateLookupField.getTableName()), idxFields)) {
                    String indexName = "idx_" + variables.resolve(insertUpdateLookupField.getTableName()) + "_lookup";
                    crIndex = db.getCreateIndexStatement(schemaTable, indexName, idxFields, false, false, false, true);
                }
                String sql = crTable + Const.CR + crIndex;
                if (sql.length() == 0) {
                    sqlStatement.setSql(null);
                } else {
                    sqlStatement.setSql(sql);
                }
            } catch (HopException e) {
                sqlStatement.setError(BaseMessages.getString(PKG, "InsertUpdateMeta.ReturnValue.ErrorOccurred") + e.getMessage());
            }
        } else {
            sqlStatement.setError(BaseMessages.getString(PKG, "InsertUpdateMeta.ReturnValue.NoTableDefinedOnConnection"));
        }
    } else {
        sqlStatement.setError(BaseMessages.getString(PKG, "InsertUpdateMeta.ReturnValue.NotReceivingAnyFields"));
    }
    return sqlStatement;
}

Example 5 with DatabaseMeta

use of org.apache.hop.core.database.DatabaseMeta in project hop by apache.

the class InsertUpdateMeta method analyseImpact.

@Override
public void analyseImpact(IVariables variables, List<DatabaseImpact> impact, PipelineMeta pipelineMeta, TransformMeta transformMeta, IRowMeta prev, String[] input, String[] output, IRowMeta info, IHopMetadataProvider metadataProvider) throws HopTransformException {
    DatabaseMeta databaseMeta = null;
    try {
        databaseMeta = metadataProvider.getSerializer(DatabaseMeta.class).load(variables.resolve(connection));
        if (prev != null) {
            // Lookup: we do a lookup on the natural keys
            for (int i = 0; i < insertUpdateLookupField.getLookupKeys().size(); i++) {
                InsertUpdateKeyField keyField = insertUpdateLookupField.getLookupKeys().get(i);
                IValueMeta v = prev.searchValueMeta(keyField.getKeyStream());
                DatabaseImpact ii = new DatabaseImpact(DatabaseImpact.TYPE_IMPACT_READ, pipelineMeta.getName(), transformMeta.getName(), databaseMeta.getDatabaseName(), variables.resolve(insertUpdateLookupField.getTableName()), keyField.getKeyLookup(), keyField.getKeyStream(), v != null ? v.getOrigin() : "?", "", "Type = " + v.toStringMeta());
                impact.add(ii);
            }
            // Insert update fields : read/write
            for (int i = 0; i < insertUpdateLookupField.getValueFields().size(); i++) {
                InsertUpdateValue valueField = insertUpdateLookupField.getValueFields().get(i);
                IValueMeta v = prev.searchValueMeta(valueField.getUpdateStream());
                DatabaseImpact ii = new DatabaseImpact(DatabaseImpact.TYPE_IMPACT_READ_WRITE, pipelineMeta.getName(), transformMeta.getName(), databaseMeta.getDatabaseName(), variables.resolve(insertUpdateLookupField.getTableName()), valueField.getUpdateLookup(), valueField.getUpdateStream(), v != null ? v.getOrigin() : "?", "", "Type = " + v.toStringMeta());
                impact.add(ii);
            }
        }
    } catch (HopException e) {
        throw new HopTransformException("Unable to get databaseMeta for connection: " + Const.CR + variables.resolve(connection));
    }
}