Examples with KettleTransException org.pentaho.di.core.exception.KettleTransException used on opensource projects

Example 1 with KettleTransException

use of org.pentaho.di.core.exception.KettleTransException in project pentaho-kettle by pentaho.

the class Trans method calculateBatchIdAndDateRange.

/**
 * Calculate the batch id and date range for the transformation.
 *
 * @throws KettleTransException if there are any errors during calculation
 */
public void calculateBatchIdAndDateRange() throws KettleTransException {
    TransLogTable transLogTable = transMeta.getTransLogTable();
    currentDate = new Date();
    logDate = new Date();
    startDate = Const.MIN_DATE;
    endDate = currentDate;
    DatabaseMeta logConnection = transLogTable.getDatabaseMeta();
    String logTable = environmentSubstitute(transLogTable.getActualTableName());
    String logSchema = environmentSubstitute(transLogTable.getActualSchemaName());
    try {
        if (logConnection != null) {
            String logSchemaAndTable = logConnection.getQuotedSchemaTableCombination(logSchema, logTable);
            if (Utils.isEmpty(logTable)) {
                // to log to.
                throw new KettleTransException(BaseMessages.getString(PKG, "Trans.Exception.NoLogTableDefined"));
            }
            if (Utils.isEmpty(transMeta.getName()) && logTable != null) {
                throw new KettleException(BaseMessages.getString(PKG, "Trans.Exception.NoTransnameAvailableForLogging"));
            }
            transLogTableDatabaseConnection = new Database(this, logConnection);
            transLogTableDatabaseConnection.shareVariablesWith(this);
            if (log.isDetailed()) {
                log.logDetailed(BaseMessages.getString(PKG, "Trans.Log.OpeningLogConnection", "" + logConnection));
            }
            transLogTableDatabaseConnection.connect();
            transLogTableDatabaseConnection.setCommit(logCommitSize);
            // 
            if (transLogTable.isBatchIdUsed()) {
                Long id_batch = logConnection.getNextBatchId(transLogTableDatabaseConnection, logSchema, logTable, transLogTable.getKeyField().getFieldName());
                setBatchId(id_batch.longValue());
            }
            // 
            // Get the date range from the logging table: from the last end_date to now. (currentDate)
            // 
            Object[] lastr = transLogTableDatabaseConnection.getLastLogDate(logSchemaAndTable, transMeta.getName(), false, LogStatus.END);
            if (lastr != null && lastr.length > 0) {
                startDate = (Date) lastr[0];
                if (log.isDetailed()) {
                    log.logDetailed(BaseMessages.getString(PKG, "Trans.Log.StartDateFound") + startDate);
                }
            }
            // 
            if (transMeta.getMaxDateConnection() != null && transMeta.getMaxDateTable() != null && transMeta.getMaxDateTable().length() > 0 && transMeta.getMaxDateField() != null && transMeta.getMaxDateField().length() > 0) {
                if (log.isDetailed()) {
                    log.logDetailed(BaseMessages.getString(PKG, "Trans.Log.LookingForMaxdateConnection", "" + transMeta.getMaxDateConnection()));
                }
                DatabaseMeta maxcon = transMeta.getMaxDateConnection();
                if (maxcon != null) {
                    Database maxdb = new Database(this, maxcon);
                    maxdb.shareVariablesWith(this);
                    try {
                        if (log.isDetailed()) {
                            log.logDetailed(BaseMessages.getString(PKG, "Trans.Log.OpeningMaximumDateConnection"));
                        }
                        maxdb.connect();
                        maxdb.setCommit(logCommitSize);
                        // 
                        // Determine the endDate by looking at a field in a table...
                        // 
                        String sql = "SELECT MAX(" + transMeta.getMaxDateField() + ") FROM " + transMeta.getMaxDateTable();
                        RowMetaAndData r1 = maxdb.getOneRow(sql);
                        if (r1 != null) {
                            // OK, we have a value, what's the offset?
                            Date maxvalue = r1.getRowMeta().getDate(r1.getData(), 0);
                            if (maxvalue != null) {
                                if (log.isDetailed()) {
                                    log.logDetailed(BaseMessages.getString(PKG, "Trans.Log.LastDateFoundOnTheMaxdateConnection") + r1);
                                }
                                endDate.setTime((long) (maxvalue.getTime() + (transMeta.getMaxDateOffset() * 1000)));
                            }
                        } else {
                            if (log.isDetailed()) {
                                log.logDetailed(BaseMessages.getString(PKG, "Trans.Log.NoLastDateFoundOnTheMaxdateConnection"));
                            }
                        }
                    } catch (KettleException e) {
                        throw new KettleTransException(BaseMessages.getString(PKG, "Trans.Exception.ErrorConnectingToDatabase", "" + transMeta.getMaxDateConnection()), e);
                    } finally {
                        maxdb.disconnect();
                    }
                } else {
                    throw new KettleTransException(BaseMessages.getString(PKG, "Trans.Exception.MaximumDateConnectionCouldNotBeFound", "" + transMeta.getMaxDateConnection()));
                }
            }
            // Get the maximum in depdate...
            if (transMeta.nrDependencies() > 0) {
                if (log.isDetailed()) {
                    log.logDetailed(BaseMessages.getString(PKG, "Trans.Log.CheckingForMaxDependencyDate"));
                }
                // 
                // Maybe one of the tables where this transformation is dependent on has changed?
                // If so we need to change the start-date!
                // 
                depDate = Const.MIN_DATE;
                Date maxdepdate = Const.MIN_DATE;
                if (lastr != null && lastr.length > 0) {
                    // #1: last depdate
                    Date dep = (Date) lastr[1];
                    if (dep != null) {
                        maxdepdate = dep;
                        depDate = dep;
                    }
                }
                for (int i = 0; i < transMeta.nrDependencies(); i++) {
                    TransDependency td = transMeta.getDependency(i);
                    DatabaseMeta depcon = td.getDatabase();
                    if (depcon != null) {
                        Database depdb = new Database(this, depcon);
                        try {
                            depdb.connect();
                            depdb.setCommit(logCommitSize);
                            String sql = "SELECT MAX(" + td.getFieldname() + ") FROM " + td.getTablename();
                            RowMetaAndData r1 = depdb.getOneRow(sql);
                            if (r1 != null) {
                                // OK, we have a row, get the result!
                                Date maxvalue = (Date) r1.getData()[0];
                                if (maxvalue != null) {
                                    if (log.isDetailed()) {
                                        log.logDetailed(BaseMessages.getString(PKG, "Trans.Log.FoundDateFromTable", td.getTablename(), "." + td.getFieldname(), " = " + maxvalue.toString()));
                                    }
                                    if (maxvalue.getTime() > maxdepdate.getTime()) {
                                        maxdepdate = maxvalue;
                                    }
                                } else {
                                    throw new KettleTransException(BaseMessages.getString(PKG, "Trans.Exception.UnableToGetDependencyInfoFromDB", td.getDatabase().getName() + ".", td.getTablename() + ".", td.getFieldname()));
                                }
                            } else {
                                throw new KettleTransException(BaseMessages.getString(PKG, "Trans.Exception.UnableToGetDependencyInfoFromDB", td.getDatabase().getName() + ".", td.getTablename() + ".", td.getFieldname()));
                            }
                        } catch (KettleException e) {
                            throw new KettleTransException(BaseMessages.getString(PKG, "Trans.Exception.ErrorInDatabase", "" + td.getDatabase()), e);
                        } finally {
                            depdb.disconnect();
                        }
                    } else {
                        throw new KettleTransException(BaseMessages.getString(PKG, "Trans.Exception.ConnectionCouldNotBeFound", "" + td.getDatabase()));
                    }
                    if (log.isDetailed()) {
                        log.logDetailed(BaseMessages.getString(PKG, "Trans.Log.Maxdepdate") + (XMLHandler.date2string(maxdepdate)));
                    }
                }
                // 
                if (maxdepdate.getTime() > depDate.getTime()) {
                    depDate = maxdepdate;
                    startDate = Const.MIN_DATE;
                }
            } else {
                depDate = currentDate;
            }
        }
        // OK, now we have a date-range. See if we need to set a maximum!
        if (// Do we have a difference specified?
        transMeta.getMaxDateDifference() > 0.0 && startDate.getTime() > Const.MIN_DATE.getTime()) {
            // Is the startdate > Minimum?
            // See if the end-date is larger then Start_date + DIFF?
            Date maxdesired = new Date(startDate.getTime() + ((long) transMeta.getMaxDateDifference() * 1000));
            // 
            if (endDate.compareTo(maxdesired) > 0) {
                endDate = maxdesired;
            }
        }
    } catch (KettleException e) {
        throw new KettleTransException(BaseMessages.getString(PKG, "Trans.Exception.ErrorCalculatingDateRange", logTable), e);
    }
// Be careful, We DO NOT close the trans log table database connection!!!
// It's closed later in beginProcessing() to prevent excessive connect/disconnect repetitions.
}

Example 2 with KettleTransException

use of org.pentaho.di.core.exception.KettleTransException in project pentaho-kettle by pentaho.

the class RepositoryFileProvider method isFileOpenedInFolder.

private void isFileOpenedInFolder(String path) throws KettleException {
    List<TransMeta> openedTransFiles = getSpoon().delegates.trans.getTransformationList();
    for (TransMeta t : openedTransFiles) {
        if (t.getRepositoryDirectory().getPath() != null && (t.getRepositoryDirectory().getPath() + "/").startsWith(path + "/")) {
            throw new KettleTransException();
        }
    }
    List<JobMeta> openedJobFiles = getSpoon().delegates.jobs.getJobList();
    for (JobMeta j : openedJobFiles) {
        if (j.getRepositoryDirectory().getPath() != null && (j.getRepositoryDirectory().getPath() + "/").startsWith(path + "/")) {
            throw new KettleJobException();
        }
    }
}

Example 3 with KettleTransException

use of org.pentaho.di.core.exception.KettleTransException in project pentaho-kettle by pentaho.

the class RepositoryBrowserController method rename.

public ObjectId rename(String id, String path, String newName, String type, String oldName) throws KettleException {
    RepositoryDirectoryInterface repositoryDirectoryInterface = findDirectory(path);
    ObjectId objectId = null;
    switch(type) {
        case JOB:
            if (getRepository().exists(newName, repositoryDirectoryInterface, RepositoryObjectType.JOB)) {
                throw new KettleObjectExistsException();
            }
            if (isJobOpened(id, path, oldName)) {
                throw new KettleJobException();
            }
            renameRecent(id, type, newName);
            objectId = getRepository().renameJob(() -> id, repositoryDirectoryInterface, newName);
            break;
        case TRANSFORMATION:
            if (getRepository().exists(newName, repositoryDirectoryInterface, RepositoryObjectType.TRANSFORMATION)) {
                throw new KettleObjectExistsException();
            }
            if (isTransOpened(id, path, oldName)) {
                throw new KettleTransException();
            }
            renameRecent(id, type, newName);
            objectId = getRepository().renameTransformation(() -> id, repositoryDirectoryInterface, newName);
            break;
        case FOLDER:
            isFileOpenedInFolder(path);
            RepositoryDirectoryInterface parent = findDirectory(path).getParent();
            if (parent == null) {
                parent = findDirectory(path);
            }
            RepositoryDirectoryInterface child = parent.findChild(newName);
            if (child != null) {
                throw new KettleObjectExistsException();
            }
            if (getRepository() instanceof RepositoryExtended) {
                objectId = ((RepositoryExtended) getRepository()).renameRepositoryDirectory(() -> id, null, newName, true);
            } else {
                objectId = getRepository().renameRepositoryDirectory(() -> id, null, newName);
            }
            break;
    }
    return objectId;
}

Example 4 with KettleTransException

use of org.pentaho.di.core.exception.KettleTransException in project pentaho-kettle by pentaho.

the class RepositoryBrowserController method remove.

public boolean remove(String id, String name, String path, String type) throws KettleException {
    try {
        switch(type) {
            case JOB:
                if (isJobOpened(id, path, name)) {
                    throw new KettleJobException();
                }
                getRepository().deleteJob(() -> id);
                break;
            case TRANSFORMATION:
                if (isTransOpened(id, path, name)) {
                    throw new KettleTransException();
                }
                getRepository().deleteTransformation(() -> id);
                break;
            case FOLDER:
                isFileOpenedInFolder(path);
                removeRecentsUsingPath(path);
                RepositoryDirectoryInterface repositoryDirectoryInterface = findDirectory(path);
                if (getRepository() instanceof RepositoryExtended) {
                    ((RepositoryExtended) getRepository()).deleteRepositoryDirectory(repositoryDirectoryInterface, true);
                } else {
                    getRepository().deleteRepositoryDirectory(repositoryDirectoryInterface);
                }
                break;
        }
        return true;
    } catch (KettleTransException | KettleJobException ke) {
        throw ke;
    } catch (Exception e) {
        return false;
    }
}

Example 5 with KettleTransException

use of org.pentaho.di.core.exception.KettleTransException in project pentaho-kettle by pentaho.

the class TransPartitioningTest method setUp.

@Before
public void setUp() throws Exception {
    MockitoAnnotations.initMocks(this);
    trans = new Trans() {

        @Override
        public void calculateBatchIdAndDateRange() throws KettleTransException {
        // avoid NPE if called
        }

        @Override
        public void beginProcessing() throws KettleTransException {
        // avoid NPE if called
        }
    };
    // prepare TransMeta complete mock:
    TransMeta meta = Mockito.mock(TransMeta.class);
    Mockito.when(meta.getName()).thenReturn("junit meta");
    Mockito.when(meta.getTransformationType()).thenReturn(TransformationType.Normal);
    Mockito.when(meta.getSizeRowset()).thenReturn(13);
    Mockito.when(meta.getTransHopSteps(Mockito.anyBoolean())).thenAnswer(new Answer<List<StepMeta>>() {

        @Override
        public List<StepMeta> answer(InvocationOnMock invocation) throws Throwable {
            return (new ArrayList<StepMeta>(chain));
        }
    });
    Mockito.when(meta.findNextSteps(Mockito.any(StepMeta.class))).then(new Answer<List<StepMeta>>() {

        @Override
        public List<StepMeta> answer(InvocationOnMock invocation) throws Throwable {
            Object obj = invocation.getArguments()[0];
            StepMeta findFor = StepMeta.class.cast(obj);
            List<StepMeta> ret = new ArrayList<StepMeta>();
            StepMeta nextStep = chain.higher(findFor);
            if (nextStep != null) {
                ret.add(nextStep);
            }
            return ret;
        }
    });
    Mockito.when(meta.findPreviousSteps(Mockito.any(StepMeta.class), Mockito.anyBoolean())).thenAnswer(new Answer<List<StepMeta>>() {

        @Override
        public List<StepMeta> answer(InvocationOnMock invocation) throws Throwable {
            Object obj = invocation.getArguments()[0];
            StepMeta findFor = StepMeta.class.cast(obj);
            List<StepMeta> ret = new ArrayList<StepMeta>();
            StepMeta prevStep = chain.lower(findFor);
            if (prevStep != null) {
                ret.add(prevStep);
            }
            return ret;
        }
    });
    Mockito.when(meta.findStep(Mockito.anyString())).thenAnswer(new Answer<StepMeta>() {

        @Override
        public StepMeta answer(InvocationOnMock invocation) throws Throwable {
            Object obj = invocation.getArguments()[0];
            String findFor = String.class.cast(obj);
            for (StepMeta item : chain) {
                if (item.getName().equals(findFor)) {
                    return item;
                }
            }
            return null;
        }
    });
    trans.setLog(log);
    trans.setTransMeta(meta);
}