Examples with Database org.pentaho.di.core.database.Database used on opensource projects

Example 11 with Database

use of org.pentaho.di.core.database.Database in project pentaho-kettle by pentaho.

the class Trans method writeStepPerformanceLogRecords.

/**
 * Write step performance log records.
 *
 * @param startSequenceNr the start sequence numberr
 * @param status          the logging status. If this is End, perform cleanup
 * @return the new sequence number
 * @throws KettleException if any errors occur during logging
 */
private int writeStepPerformanceLogRecords(int startSequenceNr, LogStatus status) throws KettleException {
    int lastSeqNr = 0;
    Database ldb = null;
    PerformanceLogTable performanceLogTable = transMeta.getPerformanceLogTable();
    if (!performanceLogTable.isDefined() || !transMeta.isCapturingStepPerformanceSnapShots() || stepPerformanceSnapShots == null || stepPerformanceSnapShots.isEmpty()) {
        // nothing to do here!
        return 0;
    }
    try {
        ldb = new Database(this, performanceLogTable.getDatabaseMeta());
        ldb.shareVariablesWith(this);
        ldb.connect();
        ldb.setCommit(logCommitSize);
        // Write to the step performance log table...
        // 
        RowMetaInterface rowMeta = performanceLogTable.getLogRecord(LogStatus.START, null, null).getRowMeta();
        ldb.prepareInsert(rowMeta, performanceLogTable.getActualSchemaName(), performanceLogTable.getActualTableName());
        synchronized (stepPerformanceSnapShots) {
            Iterator<List<StepPerformanceSnapShot>> iterator = stepPerformanceSnapShots.values().iterator();
            while (iterator.hasNext()) {
                List<StepPerformanceSnapShot> snapshots = iterator.next();
                synchronized (snapshots) {
                    Iterator<StepPerformanceSnapShot> snapshotsIterator = snapshots.iterator();
                    while (snapshotsIterator.hasNext()) {
                        StepPerformanceSnapShot snapshot = snapshotsIterator.next();
                        if (snapshot.getSeqNr() >= startSequenceNr && snapshot.getSeqNr() <= lastStepPerformanceSnapshotSeqNrAdded) {
                            RowMetaAndData row = performanceLogTable.getLogRecord(LogStatus.START, snapshot, null);
                            ldb.setValuesInsert(row.getRowMeta(), row.getData());
                            ldb.insertRow(true);
                        }
                        lastSeqNr = snapshot.getSeqNr();
                    }
                }
            }
        }
        ldb.insertFinished(true);
        // 
        if (status.equals(LogStatus.END)) {
            ldb.cleanupLogRecords(performanceLogTable, getName());
        }
    } catch (Exception e) {
        throw new KettleException(BaseMessages.getString(PKG, "Trans.Exception.ErrorWritingStepPerformanceLogRecordToTable"), e);
    } finally {
        if (ldb != null) {
            ldb.disconnect();
        }
    }
    return lastSeqNr + 1;
}

Example 12 with Database

use of org.pentaho.di.core.database.Database in project pentaho-kettle by pentaho.

the class Trans method closeUniqueDatabaseConnections.

/**
 * Close unique database connections. If there are errors in the Result, perform a rollback
 *
 * @param result the result of the transformation execution
 */
private void closeUniqueDatabaseConnections(Result result) {
    // 
    if (parentJob != null && transactionId != null && parentJob.getTransactionId() != null && transactionId.equals(parentJob.getTransactionId())) {
        return;
    }
    // 
    if (parentTrans != null && parentTrans.getTransMeta().isUsingUniqueConnections() && transactionId != null && parentTrans.getTransactionId() != null && transactionId.equals(parentTrans.getTransactionId())) {
        return;
    }
    // First we get all the database connections ...
    // 
    DatabaseConnectionMap map = DatabaseConnectionMap.getInstance();
    synchronized (map) {
        List<Database> databaseList = new ArrayList<>(map.getMap().values());
        for (Database database : databaseList) {
            if (database.getConnectionGroup().equals(getTransactionId())) {
                try {
                    // 
                    if (result.getNrErrors() > 0) {
                        try {
                            database.rollback(true);
                            log.logBasic(BaseMessages.getString(PKG, "Trans.Exception.TransactionsRolledBackOnConnection", database.toString()));
                        } catch (Exception e) {
                            throw new KettleDatabaseException(BaseMessages.getString(PKG, "Trans.Exception.ErrorRollingBackUniqueConnection", database.toString()), e);
                        }
                    } else {
                        try {
                            database.commit(true);
                            log.logBasic(BaseMessages.getString(PKG, "Trans.Exception.TransactionsCommittedOnConnection", database.toString()));
                        } catch (Exception e) {
                            throw new KettleDatabaseException(BaseMessages.getString(PKG, "Trans.Exception.ErrorCommittingUniqueConnection", database.toString()), e);
                        }
                    }
                } catch (Exception e) {
                    log.logError(BaseMessages.getString(PKG, "Trans.Exception.ErrorHandlingTransformationTransaction", database.toString()), e);
                    result.setNrErrors(result.getNrErrors() + 1);
                } finally {
                    try {
                        // This database connection belongs to this transformation.
                        database.closeConnectionOnly();
                    } catch (Exception e) {
                        log.logError(BaseMessages.getString(PKG, "Trans.Exception.ErrorHandlingTransformationTransaction", database.toString()), e);
                        result.setNrErrors(result.getNrErrors() + 1);
                    } finally {
                        // Remove the database from the list...
                        // 
                        map.removeConnection(database.getConnectionGroup(), database.getPartitionId(), database);
                    }
                }
            }
        }
        // Who else needs to be informed of the rollback or commit?
        // 
        List<DatabaseTransactionListener> transactionListeners = map.getTransactionListeners(getTransactionId());
        if (result.getNrErrors() > 0) {
            for (DatabaseTransactionListener listener : transactionListeners) {
                try {
                    listener.rollback();
                } catch (Exception e) {
                    log.logError(BaseMessages.getString(PKG, "Trans.Exception.ErrorHandlingTransactionListenerRollback"), e);
                    result.setNrErrors(result.getNrErrors() + 1);
                }
            }
        } else {
            for (DatabaseTransactionListener listener : transactionListeners) {
                try {
                    listener.commit();
                } catch (Exception e) {
                    log.logError(BaseMessages.getString(PKG, "Trans.Exception.ErrorHandlingTransactionListenerCommit"), e);
                    result.setNrErrors(result.getNrErrors() + 1);
                }
            }
        }
    }
}

Example 13 with Database

use of org.pentaho.di.core.database.Database in project pentaho-kettle by pentaho.

the class Trans method endProcessing.

/**
 * End processing. Also handle any logging operations associated with the end of a transformation
 *
 * @return true if all end processing is successful, false otherwise
 * @throws KettleException if any errors occur during processing
 */
private synchronized boolean endProcessing() throws KettleException {
    LogStatus status;
    if (isStopped()) {
        status = LogStatus.STOP;
    } else if (isFinished()) {
        status = LogStatus.END;
    } else if (isPaused()) {
        status = LogStatus.PAUSED;
    } else {
        status = LogStatus.RUNNING;
    }
    TransLogTable transLogTable = transMeta.getTransLogTable();
    int intervalInSeconds = Const.toInt(environmentSubstitute(transLogTable.getLogInterval()), -1);
    logDate = new Date();
    // OK, we have some logging to do...
    // 
    DatabaseMeta logcon = transMeta.getTransLogTable().getDatabaseMeta();
    String logTable = transMeta.getTransLogTable().getActualTableName();
    if (logcon != null) {
        Database ldb = null;
        try {
            // 
            if (transLogTableDatabaseConnection == null) {
                ldb = new Database(this, logcon);
                ldb.shareVariablesWith(this);
                ldb.connect();
                ldb.setCommit(logCommitSize);
                transLogTableDatabaseConnection = ldb;
            } else {
                ldb = transLogTableDatabaseConnection;
            }
            // 
            if (!Utils.isEmpty(logTable)) {
                ldb.writeLogRecord(transLogTable, status, this, null);
            }
            // 
            if (status.equals(LogStatus.END) || status.equals(LogStatus.STOP)) {
                ldb.cleanupLogRecords(transLogTable, getName());
            }
            // 
            if (!ldb.isAutoCommit()) {
                ldb.commitLog(true, transMeta.getTransLogTable());
            }
        } catch (KettleDatabaseException e) {
            // PDI-9790 error write to log db is transaction error
            log.logError(BaseMessages.getString(PKG, "Database.Error.WriteLogTable", logTable), e);
            errors.incrementAndGet();
        // end PDI-9790
        } catch (Exception e) {
            throw new KettleException(BaseMessages.getString(PKG, "Trans.Exception.ErrorWritingLogRecordToTable", transMeta.getTransLogTable().getActualTableName()), e);
        } finally {
            if (intervalInSeconds <= 0 || (status.equals(LogStatus.END) || status.equals(LogStatus.STOP))) {
                ldb.disconnect();
                // disconnected
                transLogTableDatabaseConnection = null;
            }
        }
    }
    return true;
}

Example 14 with Database

use of org.pentaho.di.core.database.Database in project pentaho-kettle by pentaho.

the class TableInput method init.

public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
    dbLock.lock();
    try {
        meta = (TableInputMeta) smi;
        data = (TableInputData) sdi;
        if (super.init(smi, sdi)) {
            // Verify some basic things first...
            // 
            boolean passed = true;
            if (Utils.isEmpty(meta.getSQL())) {
                logError(BaseMessages.getString(PKG, "TableInput.Exception.SQLIsNeeded"));
                passed = false;
            }
            if (meta.getDatabaseMeta() == null) {
                logError(BaseMessages.getString(PKG, "TableInput.Exception.DatabaseConnectionsIsNeeded"));
                passed = false;
            }
            if (!passed) {
                return false;
            }
            data.infoStream = meta.getStepIOMeta().getInfoStreams().get(0);
            if (meta.getDatabaseMeta() == null) {
                logError(BaseMessages.getString(PKG, "TableInput.Init.ConnectionMissing", getStepname()));
                return false;
            }
            data.db = new Database(this, meta.getDatabaseMeta());
            data.db.shareVariablesWith(this);
            data.db.setQueryLimit(Const.toInt(environmentSubstitute(meta.getRowLimit()), 0));
            try {
                if (getTransMeta().isUsingUniqueConnections()) {
                    synchronized (getTrans()) {
                        data.db.connect(getTrans().getTransactionId(), getPartitionID());
                    }
                } else {
                    data.db.connect(getPartitionID());
                }
                if (meta.getDatabaseMeta().isRequiringTransactionsOnQueries()) {
                    // needed for PGSQL it seems...
                    data.db.setCommit(100);
                }
                if (log.isDetailed()) {
                    logDetailed(BaseMessages.getString(PKG, "TableInput.Log.ConnectedToDatabase"));
                }
                return true;
            } catch (KettleException e) {
                logError(BaseMessages.getString(PKG, "TableInput.Log.ErrorOccurred", e.getMessage()));
                setErrors(1);
                stopAll();
            }
        }
        return false;
    } finally {
        dbLock.unlock();
    }
}

Example 15 with Database

use of org.pentaho.di.core.database.Database in project pentaho-kettle by pentaho.

the class TableInputMeta method getFields.

public void getFields(RowMetaInterface row, String origin, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException {
    if (databaseMeta == null) {
        // TODO: throw an exception here
        return;
    }
    if (cachedRowMetaActive) {
        row.addRowMeta(cachedRowMeta);
        return;
    }
    boolean param = false;
    Database db = getDatabase();
    // keep track of it for canceling purposes...
    super.databases = new Database[] { db };
    // First try without connecting to the database... (can be S L O W)
    String sNewSQL = sql;
    if (isVariableReplacementActive()) {
        sNewSQL = db.environmentSubstitute(sql);
        if (space != null) {
            sNewSQL = space.environmentSubstitute(sNewSQL);
        }
    }
    RowMetaInterface add = null;
    try {
        add = db.getQueryFields(sNewSQL, param);
    } catch (KettleDatabaseException dbe) {
        throw new KettleStepException("Unable to get queryfields for SQL: " + Const.CR + sNewSQL, dbe);
    }
    if (add != null) {
        attachOrigin(add, origin);
        row.addRowMeta(add);
    } else {
        try {
            db.connect();
            RowMetaInterface paramRowMeta = null;
            Object[] paramData = null;
            StreamInterface infoStream = getStepIOMeta().getInfoStreams().get(0);
            if (!Utils.isEmpty(infoStream.getStepname())) {
                param = true;
                if (info.length > 0 && info[0] != null) {
                    paramRowMeta = info[0];
                    paramData = RowDataUtil.allocateRowData(paramRowMeta.size());
                }
            }
            add = db.getQueryFields(sNewSQL, param, paramRowMeta, paramData);
            if (add == null) {
                return;
            }
            attachOrigin(add, origin);
            row.addRowMeta(add);
        } catch (KettleException ke) {
            throw new KettleStepException("Unable to get queryfields for SQL: " + Const.CR + sNewSQL, ke);
        } finally {
            db.disconnect();
        }
    }
    if (isLazyConversionActive()) {
        for (int i = 0; i < row.size(); i++) {
            ValueMetaInterface v = row.getValueMeta(i);
            try {
                if (v.getType() == ValueMetaInterface.TYPE_STRING) {
                    ValueMetaInterface storageMeta = ValueMetaFactory.cloneValueMeta(v);
                    storageMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
                    v.setStorageMetadata(storageMeta);
                    v.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
                }
            } catch (KettlePluginException e) {
                throw new KettleStepException("Unable to clone meta for lazy conversion: " + Const.CR + v, e);
            }
        }
    }
}