Examples with NotePadMeta org.pentaho.di.core.NotePadMeta used on opensource projects

Example 36 with NotePadMeta

use of org.pentaho.di.core.NotePadMeta in project pentaho-kettle by pentaho.

the class TransMeta method getSelectedNoteLocations.

/**
 * Get an array of all the selected note locations.
 *
 * @return The selected note locations.
 */
public Point[] getSelectedNoteLocations() {
    List<Point> points = new ArrayList<>();
    for (NotePadMeta ni : getSelectedNotes()) {
        Point p = ni.getLocation();
        // explicit copy of location
        points.add(new Point(p.x, p.y));
    }
    return points.toArray(new Point[points.size()]);
}

Example 37 with NotePadMeta

use of org.pentaho.di.core.NotePadMeta in project pentaho-kettle by pentaho.

the class TransMeta method getStringList.

/**
 * Gets a list of all the strings used in this transformation. The parameters indicate which collections to search and
 * which to exclude.
 *
 * @param searchSteps
 *          true if steps should be searched, false otherwise
 * @param searchDatabases
 *          true if databases should be searched, false otherwise
 * @param searchNotes
 *          true if notes should be searched, false otherwise
 * @param includePasswords
 *          true if passwords should be searched, false otherwise
 * @return a list of search results for strings used in the transformation.
 */
public List<StringSearchResult> getStringList(boolean searchSteps, boolean searchDatabases, boolean searchNotes, boolean includePasswords) {
    List<StringSearchResult> stringList = new ArrayList<>();
    if (searchSteps) {
        // Loop over all steps in the transformation and see what the used vars are...
        for (int i = 0; i < nrSteps(); i++) {
            StepMeta stepMeta = getStep(i);
            stringList.add(new StringSearchResult(stepMeta.getName(), stepMeta, this, BaseMessages.getString(PKG, "TransMeta.SearchMetadata.StepName")));
            if (stepMeta.getDescription() != null) {
                stringList.add(new StringSearchResult(stepMeta.getDescription(), stepMeta, this, BaseMessages.getString(PKG, "TransMeta.SearchMetadata.StepDescription")));
            }
            StepMetaInterface metaInterface = stepMeta.getStepMetaInterface();
            StringSearcher.findMetaData(metaInterface, 1, stringList, stepMeta, this);
        }
    }
    // Loop over all steps in the transformation and see what the used vars are...
    if (searchDatabases) {
        for (int i = 0; i < nrDatabases(); i++) {
            DatabaseMeta meta = getDatabase(i);
            stringList.add(new StringSearchResult(meta.getName(), meta, this, BaseMessages.getString(PKG, "TransMeta.SearchMetadata.DatabaseConnectionName")));
            if (meta.getHostname() != null) {
                stringList.add(new StringSearchResult(meta.getHostname(), meta, this, BaseMessages.getString(PKG, "TransMeta.SearchMetadata.DatabaseHostName")));
            }
            if (meta.getDatabaseName() != null) {
                stringList.add(new StringSearchResult(meta.getDatabaseName(), meta, this, BaseMessages.getString(PKG, "TransMeta.SearchMetadata.DatabaseName")));
            }
            if (meta.getUsername() != null) {
                stringList.add(new StringSearchResult(meta.getUsername(), meta, this, BaseMessages.getString(PKG, "TransMeta.SearchMetadata.DatabaseUsername")));
            }
            if (meta.getPluginId() != null) {
                stringList.add(new StringSearchResult(meta.getPluginId(), meta, this, BaseMessages.getString(PKG, "TransMeta.SearchMetadata.DatabaseTypeDescription")));
            }
            if (meta.getDatabasePortNumberString() != null) {
                stringList.add(new StringSearchResult(meta.getDatabasePortNumberString(), meta, this, BaseMessages.getString(PKG, "TransMeta.SearchMetadata.DatabasePort")));
            }
            if (meta.getServername() != null) {
                stringList.add(new StringSearchResult(meta.getServername(), meta, this, BaseMessages.getString(PKG, "TransMeta.SearchMetadata.DatabaseServer")));
            }
            if (includePasswords) {
                if (meta.getPassword() != null) {
                    stringList.add(new StringSearchResult(meta.getPassword(), meta, this, BaseMessages.getString(PKG, "TransMeta.SearchMetadata.DatabasePassword")));
                }
            }
        }
    }
    // Loop over all steps in the transformation and see what the used vars are...
    if (searchNotes) {
        for (int i = 0; i < nrNotes(); i++) {
            NotePadMeta meta = getNote(i);
            if (meta.getNote() != null) {
                stringList.add(new StringSearchResult(meta.getNote(), meta, this, BaseMessages.getString(PKG, "TransMeta.SearchMetadata.NotepadText")));
            }
        }
    }
    return stringList;
}

Example 38 with NotePadMeta

use of org.pentaho.di.core.NotePadMeta in project pentaho-kettle by pentaho.

the class TransSplitter method getOriginalCopy.

private TransMeta getOriginalCopy(boolean isSlaveTrans, ClusterSchema clusterSchema, SlaveServer slaveServer) throws KettleException {
    TransMeta transMeta = new TransMeta();
    transMeta.setSlaveTransformation(true);
    if (isSlaveTrans) {
        transMeta.setName(getSlaveTransName(originalTransformation.getName(), clusterSchema, slaveServer));
        NotePadMeta slaveNote = new NotePadMeta("This is a generated slave transformation.\nIt will be run on slave server: " + slaveServer, 0, 0, -1, -1);
        transMeta.addNote(slaveNote);
        // add the slave partitioning schema's here.
        for (int i = 0; i < referenceSteps.length; i++) {
            StepMeta stepMeta = referenceSteps[i];
            verifySlavePartitioningConfiguration(transMeta, stepMeta, clusterSchema, slaveServer);
        }
    } else {
        transMeta.setName(originalTransformation.getName() + " (master)");
        NotePadMeta masterNote = new NotePadMeta("This is a generated master transformation.\nIt will be run on server: " + getMasterServer(), 0, 0, -1, -1);
        transMeta.addNote(masterNote);
    }
    // 
    for (ClusterSchema schema : originalTransformation.getClusterSchemas()) {
        transMeta.getClusterSchemas().add(schema.clone());
    }
    transMeta.setDatabases(originalTransformation.getDatabases());
    // Feedback
    transMeta.setFeedbackShown(originalTransformation.isFeedbackShown());
    transMeta.setFeedbackSize(originalTransformation.getFeedbackSize());
    // Priority management
    transMeta.setUsingThreadPriorityManagment(originalTransformation.isUsingThreadPriorityManagment());
    // Unique connections
    transMeta.setUsingUniqueConnections(originalTransformation.isUsingUniqueConnections());
    // Repository
    transMeta.setRepository(originalTransformation.getRepository());
    transMeta.setRepositoryDirectory(originalTransformation.getRepositoryDirectory());
    // Also set the logging details...
    transMeta.setTransLogTable((TransLogTable) originalTransformation.getTransLogTable().clone());
    // Rowset size
    transMeta.setSizeRowset(originalTransformation.getSizeRowset());
    return transMeta;
}

Example 39 with NotePadMeta

use of org.pentaho.di.core.NotePadMeta in project pentaho-kettle by pentaho.

the class KettleDatabaseRepositoryTransDelegate method loadTransformation.

/**
 * Read a transformation with a certain name from a repository
 *
 * @param rep
 *          The repository to read from.
 * @param transname
 *          The name of the transformation.
 * @param repdir
 *          the path to the repository directory
 * @param monitor
 *          The progress monitor to display the progress of the file-open operation in a dialog
 * @param setInternalVariables
 *          true if you want to set the internal variables based on this transformation information
 */
public TransMeta loadTransformation(TransMeta transMeta, String transname, RepositoryDirectoryInterface repdir, ProgressMonitorListener monitor, boolean setInternalVariables) throws KettleException {
    transMeta.setRepository(repository);
    transMeta.setMetaStore(repository.metaStore);
    synchronized (repository) {
        try {
            String pathAndName = repdir.isRoot() ? repdir + transname : repdir + RepositoryDirectory.DIRECTORY_SEPARATOR + transname;
            transMeta.setName(transname);
            transMeta.setRepositoryDirectory(repdir);
            // Get the transformation id
            if (log.isDetailed()) {
                log.logDetailed(BaseMessages.getString(PKG, "TransMeta.Log.LookingForTransformation", transname, repdir.getPath()));
            }
            if (monitor != null) {
                monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingTransformationInfoTask.Title"));
            }
            transMeta.setObjectId(getTransformationID(transname, repdir.getObjectId()));
            if (monitor != null) {
                monitor.worked(1);
            }
            // If no valid id is available in the database, then give error...
            if (transMeta.getObjectId() != null) {
                ObjectId[] noteids = repository.getTransNoteIDs(transMeta.getObjectId());
                ObjectId[] stepids = repository.getStepIDs(transMeta.getObjectId());
                ObjectId[] hopids = getTransHopIDs(transMeta.getObjectId());
                int nrWork = 3 + noteids.length + stepids.length + hopids.length;
                if (monitor != null) {
                    monitor.beginTask(BaseMessages.getString(PKG, "TransMeta.Monitor.LoadingTransformationTask.Title") + pathAndName, nrWork);
                }
                if (log.isDetailed()) {
                    log.logDetailed(BaseMessages.getString(PKG, "TransMeta.Log.LoadingTransformation", transMeta.getName()));
                }
                // Load the common database connections
                if (monitor != null) {
                    monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingTheAvailableSharedObjectsTask.Title"));
                }
                try {
                    transMeta.setSharedObjects(readTransSharedObjects(transMeta));
                } catch (Exception e) {
                    log.logError(BaseMessages.getString(PKG, "TransMeta.ErrorReadingSharedObjects.Message", e.toString()));
                    log.logError(Const.getStackTracker(e));
                }
                if (monitor != null) {
                    monitor.worked(1);
                }
                // Load the notes...
                if (monitor != null) {
                    monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingNoteTask.Title"));
                }
                for (int i = 0; i < noteids.length; i++) {
                    NotePadMeta ni = repository.notePadDelegate.loadNotePadMeta(noteids[i]);
                    if (transMeta.indexOfNote(ni) < 0) {
                        transMeta.addNote(ni);
                    }
                    if (monitor != null) {
                        monitor.worked(1);
                    }
                }
                if (monitor != null) {
                    monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingStepsTask.Title"));
                }
                // read all the attributes
                repository.connectionDelegate.fillStepAttributesBuffer(transMeta.getObjectId());
                // on one go!
                for (int i = 0; i < stepids.length; i++) {
                    if (log.isDetailed()) {
                        log.logDetailed(BaseMessages.getString(PKG, "TransMeta.Log.LoadingStepWithID") + stepids[i]);
                    }
                    if (monitor != null) {
                        monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingStepTask.Title") + (i + 1) + "/" + (stepids.length));
                    }
                    StepMeta stepMeta = repository.stepDelegate.loadStepMeta(stepids[i], transMeta.getDatabases(), transMeta.getPartitionSchemas());
                    if (stepMeta.isMissing()) {
                        transMeta.addMissingTrans((MissingTrans) stepMeta.getStepMetaInterface());
                    }
                    // In this case, we just add or replace the shared steps.
                    // The repository is considered "more central"
                    transMeta.addOrReplaceStep(stepMeta);
                    if (monitor != null) {
                        monitor.worked(1);
                    }
                }
                if (monitor != null) {
                    monitor.worked(1);
                }
                // clear the buffer (should be empty anyway)
                repository.connectionDelegate.setStepAttributesBuffer(null);
                // Have all StreamValueLookups, etc. reference the correct source steps...
                for (int i = 0; i < transMeta.nrSteps(); i++) {
                    StepMetaInterface sii = transMeta.getStep(i).getStepMetaInterface();
                    sii.searchInfoAndTargetSteps(transMeta.getSteps());
                }
                if (monitor != null) {
                    monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.LoadingTransformationDetailsTask.Title"));
                }
                loadRepTrans(transMeta);
                if (monitor != null) {
                    monitor.worked(1);
                }
                if (monitor != null) {
                    monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingHopTask.Title"));
                }
                for (int i = 0; i < hopids.length; i++) {
                    TransHopMeta hi = loadTransHopMeta(hopids[i], transMeta.getSteps());
                    if (hi != null) {
                        transMeta.addTransHop(hi);
                    }
                    if (monitor != null) {
                        monitor.worked(1);
                    }
                }
                // 
                for (int i = 0; i < transMeta.nrSteps(); i++) {
                    StepPartitioningMeta stepPartitioningMeta = transMeta.getStep(i).getStepPartitioningMeta();
                    if (stepPartitioningMeta != null) {
                        stepPartitioningMeta.setPartitionSchemaAfterLoading(transMeta.getPartitionSchemas());
                    }
                }
                // 
                for (int i = 0; i < transMeta.nrSteps(); i++) {
                    transMeta.getStep(i).setClusterSchemaAfterLoading(transMeta.getClusterSchemas());
                }
                if (monitor != null) {
                    monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingTheDependenciesTask.Title"));
                }
                ObjectId[] depids = repository.getTransDependencyIDs(transMeta.getObjectId());
                for (int i = 0; i < depids.length; i++) {
                    TransDependency td = loadTransDependency(depids[i], transMeta.getDatabases());
                    transMeta.addDependency(td);
                }
                if (monitor != null) {
                    monitor.worked(1);
                }
                // Load the group attributes map
                // 
                transMeta.setAttributesMap(loadTransAttributesMap(transMeta.getObjectId()));
                // 
                for (int i = 0; i < transMeta.nrSteps(); i++) {
                    StepMeta stepMeta = transMeta.getStep(i);
                    String sourceStep = repository.getStepAttributeString(stepMeta.getObjectId(), "step_error_handling_source_step");
                    if (sourceStep != null) {
                        StepErrorMeta stepErrorMeta = repository.stepDelegate.loadStepErrorMeta(transMeta, stepMeta, transMeta.getSteps());
                        // a bit of a trick, I know.
                        stepErrorMeta.getSourceStep().setStepErrorMeta(stepErrorMeta);
                    }
                }
                // Load all the log tables for the transformation...
                // 
                RepositoryAttributeInterface attributeInterface = new KettleDatabaseRepositoryTransAttribute(repository.connectionDelegate, transMeta.getObjectId());
                for (LogTableInterface logTable : transMeta.getLogTables()) {
                    logTable.loadFromRepository(attributeInterface);
                }
                if (monitor != null) {
                    monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.SortingStepsTask.Title"));
                }
                transMeta.sortSteps();
                if (monitor != null) {
                    monitor.worked(1);
                }
                if (monitor != null) {
                    monitor.done();
                }
            } else {
                throw new KettleException(BaseMessages.getString(PKG, "TransMeta.Exception.TransformationDoesNotExist") + transMeta.getName());
            }
            if (log.isDetailed()) {
                log.logDetailed(BaseMessages.getString(PKG, "TransMeta.Log.LoadedTransformation2", transname, String.valueOf(transMeta.getRepositoryDirectory() == null)));
                log.logDetailed(BaseMessages.getString(PKG, "TransMeta.Log.LoadedTransformation", transname, transMeta.getRepositoryDirectory().getPath()));
            }
            // close prepared statements, minimize locking etc.
            // 
            repository.connectionDelegate.closeAttributeLookupPreparedStatements();
            return transMeta;
        } catch (KettleDatabaseException e) {
            log.logError(BaseMessages.getString(PKG, "TransMeta.Log.DatabaseErrorOccuredReadingTransformation") + Const.CR + e);
            throw new KettleException(BaseMessages.getString(PKG, "TransMeta.Exception.DatabaseErrorOccuredReadingTransformation"), e);
        } catch (Exception e) {
            log.logError(BaseMessages.getString(PKG, "TransMeta.Log.DatabaseErrorOccuredReadingTransformation") + Const.CR + e);
            throw new KettleException(BaseMessages.getString(PKG, "TransMeta.Exception.DatabaseErrorOccuredReadingTransformation2"), e);
        } finally {
            transMeta.initializeVariablesFrom(null);
            if (setInternalVariables) {
                transMeta.setInternalKettleVariables();
            }
        }
    }
}

Example 40 with NotePadMeta

use of org.pentaho.di.core.NotePadMeta in project pentaho-kettle by pentaho.

the class KettleDatabaseRepositoryJobDelegate method saveJob.

/**
 * Stored a job in the repository
 *
 * @param jobMeta
 *          The job to store
 * @param monitor
 *          the (optional) UI progress monitor
 * @param overwrite
 *          Overwrite existing object(s)?
 * @throws KettleException
 *           in case some IO error occurs.
 */
public void saveJob(JobMeta jobMeta, String versionComment, ProgressMonitorListener monitor, boolean overwrite) throws KettleException {
    try {
        // Before saving the job, see if it's not locked by someone else...
        // 
        int nrWorks = 2 + jobMeta.nrDatabases() + jobMeta.nrNotes() + jobMeta.nrJobEntries() + jobMeta.nrJobHops();
        if (monitor != null) {
            monitor.beginTask(BaseMessages.getString(PKG, "JobMeta.Monitor.SavingTransformation") + jobMeta.getRepositoryDirectory() + Const.FILE_SEPARATOR + jobMeta.getName(), nrWorks);
        }
        repository.insertLogEntry("save job '" + jobMeta.getName() + "'");
        // 
        if (monitor != null) {
            monitor.subTask(BaseMessages.getString(PKG, "JobMeta.Monitor.HandlingPreviousVersionOfJob"));
        }
        jobMeta.setObjectId(getJobID(jobMeta.getName(), jobMeta.getRepositoryDirectory().getObjectId()));
        // If no valid id is available in the database, assign one...
        if (jobMeta.getObjectId() == null) {
            jobMeta.setObjectId(repository.connectionDelegate.getNextJobID());
        } else {
            // If we have a valid ID, we need to make sure everything is
            // cleared out
            // of the database for this id_job, before we put it back in...
            repository.deleteJob(jobMeta.getObjectId());
        }
        if (monitor != null) {
            monitor.worked(1);
        }
        // 
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "JobMeta.Log.SavingDatabaseConnections"));
        }
        for (int i = 0; i < jobMeta.nrDatabases(); i++) {
            if (monitor != null) {
                monitor.subTask(BaseMessages.getString(PKG, "JobMeta.Monitor.SavingDatabaseTask.Title") + (i + 1) + "/" + jobMeta.nrDatabases());
            }
            DatabaseMeta databaseMeta = jobMeta.getDatabase(i);
            // nothing was saved in the repository)
            if (overwrite || databaseMeta.hasChanged() || databaseMeta.getObjectId() == null) {
                repository.save(databaseMeta, versionComment, monitor, overwrite);
            }
            if (monitor != null) {
                monitor.worked(1);
            }
        }
        // entries to the save job. (retry)
        if (monitor != null) {
            monitor.subTask(BaseMessages.getString(PKG, "JobMeta.Monitor.SavingJobDetails"));
        }
        if (log.isDetailed()) {
            log.logDetailed("Saving job info to repository...");
        }
        insertJob(jobMeta);
        if (monitor != null) {
            monitor.worked(1);
        }
        // Save the group attributes map
        // 
        saveJobAttributesMap(jobMeta.getObjectId(), jobMeta.getAttributesMap());
        // 
        for (int i = 0; i < jobMeta.getSlaveServers().size(); i++) {
            SlaveServer slaveServer = jobMeta.getSlaveServers().get(i);
            repository.save(slaveServer, versionComment, null, jobMeta.getObjectId(), false, overwrite);
        }
        // 
        if (log.isDetailed()) {
            log.logDetailed("Saving notes to repository...");
        }
        for (int i = 0; i < jobMeta.nrNotes(); i++) {
            if (monitor != null) {
                monitor.subTask(BaseMessages.getString(PKG, "JobMeta.Monitor.SavingNoteNr") + (i + 1) + "/" + jobMeta.nrNotes());
            }
            NotePadMeta ni = jobMeta.getNote(i);
            repository.saveNotePadMeta(ni, jobMeta.getObjectId());
            if (ni.getObjectId() != null) {
                repository.insertJobNote(jobMeta.getObjectId(), ni.getObjectId());
            }
            if (monitor != null) {
                monitor.worked(1);
            }
        }
        // 
        if (log.isDetailed()) {
            log.logDetailed("Saving " + jobMeta.nrJobEntries() + " Job enty copies to repository...");
        }
        repository.updateJobEntryTypes();
        for (int i = 0; i < jobMeta.nrJobEntries(); i++) {
            if (monitor != null) {
                monitor.subTask(BaseMessages.getString(PKG, "JobMeta.Monitor.SavingJobEntryNr") + (i + 1) + "/" + jobMeta.nrJobEntries());
            }
            JobEntryCopy cge = jobMeta.getJobEntry(i);
            repository.jobEntryDelegate.saveJobEntryCopy(cge, jobMeta.getObjectId(), repository.metaStore);
            if (monitor != null) {
                monitor.worked(1);
            }
        }
        if (log.isDetailed()) {
            log.logDetailed("Saving job hops to repository...");
        }
        for (int i = 0; i < jobMeta.nrJobHops(); i++) {
            if (monitor != null) {
                monitor.subTask("Saving job hop #" + (i + 1) + "/" + jobMeta.nrJobHops());
            }
            JobHopMeta hi = jobMeta.getJobHop(i);
            saveJobHopMeta(hi, jobMeta.getObjectId());
            if (monitor != null) {
                monitor.worked(1);
            }
        }
        saveJobParameters(jobMeta);
        // Commit this transaction!!
        repository.commit();
        jobMeta.clearChanged();
        if (monitor != null) {
            monitor.done();
        }
    } catch (KettleDatabaseException dbe) {
        repository.rollback();
        throw new KettleException(BaseMessages.getString(PKG, "JobMeta.Exception.UnableToSaveJobInRepositoryRollbackPerformed"), dbe);
    }
}