Examples with StepMeta org.pentaho.di.trans.step.StepMeta used on opensource projects

Example 76 with StepMeta

use of org.pentaho.di.trans.step.StepMeta in project pentaho-kettle by pentaho.

the class TransMeta method getAlternativeStepname.

/**
 * Proposes an alternative stepname when the original already exists.
 *
 * @param stepname
 *          The stepname to find an alternative for
 * @return The suggested alternative stepname.
 */
public String getAlternativeStepname(String stepname) {
    String newname = stepname;
    StepMeta stepMeta = findStep(newname);
    int nr = 1;
    while (stepMeta != null) {
        nr++;
        newname = stepname + " " + nr;
        stepMeta = findStep(newname);
    }
    return newname;
}

Example 77 with StepMeta

use of org.pentaho.di.trans.step.StepMeta in project pentaho-kettle by pentaho.

the class TransMeta method exportResources.

/**
 * Exports the specified objects to a flat-file system, adding content with filename keys to a set of definitions. The
 * supplied resource naming interface allows the object to name appropriately without worrying about those parts of
 * the implementation specific details.
 *
 * @param space
 *          the variable space to use
 * @param definitions
 * @param resourceNamingInterface
 * @param repository
 *          The repository to optionally load other resources from (to be converted to XML)
 * @param metaStore
 *          the metaStore in which non-kettle metadata could reside.
 *
 * @return the filename of the exported resource
 */
@Override
public String exportResources(VariableSpace space, Map<String, ResourceDefinition> definitions, ResourceNamingInterface resourceNamingInterface, Repository repository, IMetaStore metaStore) throws KettleException {
    try {
        // Handle naming for both repository and XML bases resources...
        // 
        String baseName;
        String originalPath;
        String fullname;
        String extension = "ktr";
        if (Utils.isEmpty(getFilename())) {
            // Assume repository...
            // 
            originalPath = directory.getPath();
            baseName = getName();
            fullname = directory.getPath() + (directory.getPath().endsWith(RepositoryDirectory.DIRECTORY_SEPARATOR) ? "" : RepositoryDirectory.DIRECTORY_SEPARATOR) + getName() + "." + // 
            extension;
        } else {
            // Assume file
            // 
            FileObject fileObject = KettleVFS.getFileObject(space.environmentSubstitute(getFilename()), space);
            originalPath = fileObject.getParent().getURL().toString();
            baseName = fileObject.getName().getBaseName();
            fullname = fileObject.getURL().toString();
        }
        String exportFileName = resourceNamingInterface.nameResource(baseName, originalPath, extension, ResourceNamingInterface.FileNamingType.TRANSFORMATION);
        ResourceDefinition definition = definitions.get(exportFileName);
        if (definition == null) {
            // If we do this once, it will be plenty :-)
            // 
            TransMeta transMeta = (TransMeta) this.realClone(false);
            // 
            for (StepMeta stepMeta : transMeta.getSteps()) {
                stepMeta.exportResources(space, definitions, resourceNamingInterface, repository, metaStore);
            }
            // Change the filename, calling this sets internal variables
            // inside of the transformation.
            // 
            transMeta.setFilename(exportFileName);
            // All objects get re-located to the root folder
            // 
            transMeta.setRepositoryDirectory(new RepositoryDirectory());
            // Set a number of parameters for all the data files referenced so far...
            // 
            Map<String, String> directoryMap = resourceNamingInterface.getDirectoryMap();
            if (directoryMap != null) {
                for (String directory : directoryMap.keySet()) {
                    String parameterName = directoryMap.get(directory);
                    transMeta.addParameterDefinition(parameterName, directory, "Data file path discovered during export");
                }
            }
            // At the end, add ourselves to the map...
            // 
            String transMetaContent = transMeta.getXML();
            definition = new ResourceDefinition(exportFileName, transMetaContent);
            // 
            if (Utils.isEmpty(this.getFilename())) {
                // Repository
                definition.setOrigin(fullname);
            } else {
                definition.setOrigin(this.getFilename());
            }
            definitions.put(fullname, definition);
        }
        return exportFileName;
    } catch (FileSystemException e) {
        throw new KettleException(BaseMessages.getString(PKG, "TransMeta.Exception.ErrorOpeningOrValidatingTheXMLFile", getFilename()), e);
    } catch (KettleFileException e) {
        throw new KettleException(BaseMessages.getString(PKG, "TransMeta.Exception.ErrorOpeningOrValidatingTheXMLFile", getFilename()), e);
    }
}

Example 78 with StepMeta

use of org.pentaho.di.trans.step.StepMeta in project pentaho-kettle by pentaho.

the class TransMeta method loadSharedObject.

@Override
public boolean loadSharedObject(SharedObjectInterface object) {
    if (!super.loadSharedObject(object)) {
        if (object instanceof StepMeta) {
            StepMeta stepMeta = (StepMeta) object;
            addOrReplaceStep(stepMeta);
        } else if (object instanceof PartitionSchema) {
            PartitionSchema partitionSchema = (PartitionSchema) object;
            addOrReplacePartitionSchema(partitionSchema);
        } else if (object instanceof ClusterSchema) {
            ClusterSchema clusterSchema = (ClusterSchema) object;
            clusterSchema.shareVariablesWith(this);
            addOrReplaceClusterSchema(clusterSchema);
        } else {
            return false;
        }
    }
    return true;
}

Example 79 with StepMeta

use of org.pentaho.di.trans.step.StepMeta in project pentaho-kettle by pentaho.

the class KettleDatabaseRepositoryStepDelegate method loadStepMeta.

/**
 * Create a new step by loading the metadata from the specified repository.
 *
 * @param rep
 * @param stepId
 * @param databases
 * @param counters
 * @param partitionSchemas
 * @throws KettleException
 */
public StepMeta loadStepMeta(ObjectId stepId, List<DatabaseMeta> databases, List<PartitionSchema> partitionSchemas) throws KettleException {
    StepMeta stepMeta = new StepMeta();
    PluginRegistry registry = PluginRegistry.getInstance();
    try {
        RowMetaAndData r = getStep(stepId);
        if (r != null) {
            stepMeta.setObjectId(stepId);
            stepMeta.setName(r.getString(KettleDatabaseRepository.FIELD_STEP_NAME, null));
            stepMeta.setDescription(r.getString(KettleDatabaseRepository.FIELD_STEP_DESCRIPTION, null));
            long id_step_type = r.getInteger(KettleDatabaseRepository.FIELD_STEP_ID_STEP_TYPE, -1L);
            RowMetaAndData steptyperow = getStepType(new LongObjectId(id_step_type));
            stepMeta.setStepID(steptyperow.getString(KettleDatabaseRepository.FIELD_STEP_TYPE_CODE, null));
            stepMeta.setDistributes(r.getBoolean(KettleDatabaseRepository.FIELD_STEP_DISTRIBUTE, true));
            int copies = (int) r.getInteger(KettleDatabaseRepository.FIELD_STEP_COPIES, 1);
            String copiesString = r.getString(KettleDatabaseRepository.FIELD_STEP_COPIES_STRING, null);
            if (!Utils.isEmpty(copiesString)) {
                stepMeta.setCopiesString(copiesString);
            } else {
                stepMeta.setCopies(copies);
            }
            int x = (int) r.getInteger(KettleDatabaseRepository.FIELD_STEP_GUI_LOCATION_X, 0);
            int y = (int) r.getInteger(KettleDatabaseRepository.FIELD_STEP_GUI_LOCATION_Y, 0);
            stepMeta.setLocation(new Point(x, y));
            stepMeta.setDraw(r.getBoolean(KettleDatabaseRepository.FIELD_STEP_GUI_DRAW, false));
            // Generate the appropriate class...
            PluginInterface sp = registry.findPluginWithId(StepPluginType.class, stepMeta.getStepID());
            if (sp == null) {
                stepMeta.setStepMetaInterface(new MissingTrans(stepMeta.getName(), stepMeta.getStepID()));
            } else {
                stepMeta.setStepMetaInterface((StepMetaInterface) registry.loadClass(sp));
            }
            if (stepMeta.getStepMetaInterface() != null) {
                // Read the step info from the repository!
                readRepCompatibleStepMeta(stepMeta.getStepMetaInterface(), repository, stepMeta.getObjectId(), databases);
                stepMeta.getStepMetaInterface().readRep(repository, repository.metaStore, stepMeta.getObjectId(), databases);
            }
            // Get the partitioning as well...
            // 
            stepMeta.setStepPartitioningMeta(loadStepPartitioningMeta(stepMeta.getObjectId()));
            stepMeta.getStepPartitioningMeta().setPartitionSchemaAfterLoading(partitionSchemas);
            // Get the cluster schema name
            // 
            stepMeta.setClusterSchemaName(repository.getStepAttributeString(stepId, "cluster_schema"));
            // Are we using a custom row distribution plugin?
            // 
            String rowDistributionCode = repository.getStepAttributeString(stepId, 0, "row_distribution_code");
            RowDistributionInterface rowDistribution = PluginRegistry.getInstance().loadClass(RowDistributionPluginType.class, rowDistributionCode, RowDistributionInterface.class);
            stepMeta.setRowDistribution(rowDistribution);
            // Load the attribute groups map
            // 
            stepMeta.setAttributesMap(loadStepAttributesMap(stepId));
            // 
            return stepMeta;
        } else {
            throw new KettleException(BaseMessages.getString(PKG, "StepMeta.Exception.StepInfoCouldNotBeFound", String.valueOf(stepId)));
        }
    } catch (KettleDatabaseException dbe) {
        throw new KettleException(BaseMessages.getString(PKG, "StepMeta.Exception.StepCouldNotBeLoaded", String.valueOf(stepMeta.getObjectId())), dbe);
    }
}

Example 80 with StepMeta

use of org.pentaho.di.trans.step.StepMeta in project pentaho-kettle by pentaho.

the class KettleDatabaseRepositoryTransDelegate method saveTransformation.

/**
 * Saves the transformation to a repository.
 *
 * @param transMeta
 *          the transformation metadata to store
 * @param monitor
 *          the way we report progress to the user, can be null if no UI is present
 * @param overwrite
 *          Overwrite existing object(s)?
 * @throws KettleException
 *           if an error occurs.
 */
public void saveTransformation(TransMeta transMeta, String versionComment, ProgressMonitorListener monitor, boolean overwriteAssociated) throws KettleException {
    try {
        if (monitor != null) {
            monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.LockingRepository"));
        }
        repository.insertLogEntry("save transformation '" + transMeta.getName() + "'");
        // Clear attribute id cache
        // force repository lookup.
        repository.connectionDelegate.clearNextIDCounters();
        // Do we have a valid directory?
        if (transMeta.getRepositoryDirectory().getObjectId() == null) {
            throw new KettleException(BaseMessages.getString(PKG, "TransMeta.Exception.PlsSelectAValidDirectoryBeforeSavingTheTransformation"));
        }
        int nrWorks = 2 + transMeta.nrDatabases() + transMeta.nrNotes() + transMeta.nrSteps() + transMeta.nrTransHops();
        if (monitor != null) {
            monitor.beginTask(BaseMessages.getString(PKG, "TransMeta.Monitor.SavingTransformationTask.Title") + transMeta.getPathAndName(), nrWorks);
        }
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "TransMeta.Log.SavingOfTransformationStarted"));
        }
        if (monitor != null && monitor.isCanceled()) {
            throw new KettleDatabaseException();
        }
        // 
        if (monitor != null) {
            monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.HandlingOldVersionTransformationTask.Title"));
        // transMeta.setObjectId(getTransformationID(transMeta.getName(),
        // transMeta.getRepositoryDirectory().getObjectId()));
        }
        // If no valid id is available in the database, assign one...
        if (transMeta.getObjectId() == null) {
            transMeta.setObjectId(repository.connectionDelegate.getNextTransformationID());
        } else {
            // of the database for this id_transformation, before we put it back in...
            if (monitor != null) {
                monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.DeletingOldVersionTransformationTask.Title"));
            }
            if (log.isDebug()) {
                log.logDebug(BaseMessages.getString(PKG, "TransMeta.Log.DeletingOldVersionTransformation"));
            }
            repository.deleteTransformation(transMeta.getObjectId());
            if (log.isDebug()) {
                log.logDebug(BaseMessages.getString(PKG, "TransMeta.Log.OldVersionOfTransformationRemoved"));
            }
        }
        if (monitor != null) {
            monitor.worked(1);
        }
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "TransMeta.Log.SavingNotes"));
        }
        for (int i = 0; i < transMeta.nrNotes(); i++) {
            if (monitor != null && monitor.isCanceled()) {
                throw new KettleDatabaseException(BaseMessages.getString(PKG, "TransMeta.Log.UserCancelledTransSave"));
            }
            // if (monitor != null) monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.SavingNoteTask.Title") +
            // (i + 1) + "/" + transMeta.nrNotes());
            NotePadMeta ni = transMeta.getNote(i);
            repository.saveNotePadMeta(ni, transMeta.getObjectId());
            if (ni.getObjectId() != null) {
                repository.insertTransNote(transMeta.getObjectId(), ni.getObjectId());
            }
            if (monitor != null) {
                monitor.worked(1);
            }
        }
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "TransMeta.Log.SavingDatabaseConnections"));
        }
        for (int i = 0; i < transMeta.nrDatabases(); i++) {
            if (monitor != null && monitor.isCanceled()) {
                throw new KettleDatabaseException(BaseMessages.getString(PKG, "TransMeta.Log.UserCancelledTransSave"));
            }
            // if (monitor != null) monitor.subTask(BaseMessages.getString(PKG,
            // "TransMeta.Monitor.SavingDatabaseTask.Title") + (i + 1) + "/" + transMeta.nrDatabases());
            DatabaseMeta databaseMeta = transMeta.getDatabase(i);
            // repository)
            if (overwriteAssociated || databaseMeta.hasChanged() || databaseMeta.getObjectId() == null) {
                repository.save(databaseMeta, versionComment, monitor, overwriteAssociated);
            }
            if (monitor != null) {
                monitor.worked(1);
            }
        }
        // It is possible that we received another step through a plugin.
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "TransMeta.Log.CheckingStepTypes"));
        }
        repository.updateStepTypes();
        repository.updateDatabaseTypes();
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "TransMeta.Log.SavingSteps"));
        }
        for (int i = 0; i < transMeta.nrSteps(); i++) {
            if (monitor != null && monitor.isCanceled()) {
                throw new KettleDatabaseException(BaseMessages.getString(PKG, "TransMeta.Log.UserCancelledTransSave"));
            }
            // if (monitor != null) monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.SavingStepTask.Title") +
            // (i + 1) + "/" + transMeta.nrSteps());
            StepMeta stepMeta = transMeta.getStep(i);
            repository.stepDelegate.saveStepMeta(stepMeta, transMeta.getObjectId());
            if (monitor != null) {
                monitor.worked(1);
            }
        }
        repository.connectionDelegate.closeStepAttributeInsertPreparedStatement();
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "TransMeta.Log.SavingHops"));
        }
        for (int i = 0; i < transMeta.nrTransHops(); i++) {
            if (monitor != null && monitor.isCanceled()) {
                throw new KettleDatabaseException(BaseMessages.getString(PKG, "TransMeta.Log.UserCancelledTransSave"));
            }
            // if (monitor != null) monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.SavingHopTask.Title") +
            // (i + 1) + "/" + transMeta.nrTransHops());
            TransHopMeta hi = transMeta.getTransHop(i);
            saveTransHopMeta(hi, transMeta.getObjectId());
            if (monitor != null) {
                monitor.worked(1);
            }
        }
        // if (monitor != null) monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.FinishingTask.Title"));
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "TransMeta.Log.SavingTransformationInfo"));
        }
        // save the top level information for the transformation
        insertTransformation(transMeta);
        saveTransParameters(transMeta);
        repository.connectionDelegate.closeTransAttributeInsertPreparedStatement();
        // 
        for (int i = 0; i < transMeta.getPartitionSchemas().size(); i++) {
            if (monitor != null && monitor.isCanceled()) {
                throw new KettleDatabaseException(BaseMessages.getString(PKG, "TransMeta.Log.UserCancelledTransSave"));
            }
            PartitionSchema partitionSchema = transMeta.getPartitionSchemas().get(i);
            // See if this transformation really is a consumer of this object
            // It might be simply loaded as a shared object from the repository
            // 
            boolean isUsedByTransformation = transMeta.isUsingPartitionSchema(partitionSchema);
            repository.save(partitionSchema, versionComment, null, transMeta.getObjectId(), isUsedByTransformation, overwriteAssociated);
        }
        // 
        for (int i = 0; i < transMeta.getSlaveServers().size(); i++) {
            if (monitor != null && monitor.isCanceled()) {
                throw new KettleDatabaseException(BaseMessages.getString(PKG, "TransMeta.Log.UserCancelledTransSave"));
            }
            SlaveServer slaveServer = transMeta.getSlaveServers().get(i);
            boolean isUsedByTransformation = transMeta.isUsingSlaveServer(slaveServer);
            repository.save(slaveServer, versionComment, null, transMeta.getObjectId(), isUsedByTransformation, overwriteAssociated);
        }
        // Save the clustering schemas
        for (int i = 0; i < transMeta.getClusterSchemas().size(); i++) {
            if (monitor != null && monitor.isCanceled()) {
                throw new KettleDatabaseException(BaseMessages.getString(PKG, "TransMeta.Log.UserCancelledTransSave"));
            }
            ClusterSchema clusterSchema = transMeta.getClusterSchemas().get(i);
            boolean isUsedByTransformation = transMeta.isUsingClusterSchema(clusterSchema);
            repository.save(clusterSchema, versionComment, null, transMeta.getObjectId(), isUsedByTransformation, overwriteAssociated);
        }
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "TransMeta.Log.SavingDependencies"));
        }
        for (int i = 0; i < transMeta.nrDependencies(); i++) {
            if (monitor != null && monitor.isCanceled()) {
                throw new KettleDatabaseException(BaseMessages.getString(PKG, "TransMeta.Log.UserCancelledTransSave"));
            }
            TransDependency td = transMeta.getDependency(i);
            saveTransDependency(td, transMeta.getObjectId());
        }
        saveTransAttributesMap(transMeta.getObjectId(), transMeta.getAttributesMap());
        // Save the step error handling information as well!
        for (int i = 0; i < transMeta.nrSteps(); i++) {
            StepMeta stepMeta = transMeta.getStep(i);
            StepErrorMeta stepErrorMeta = stepMeta.getStepErrorMeta();
            if (stepErrorMeta != null) {
                repository.stepDelegate.saveStepErrorMeta(stepErrorMeta, transMeta.getObjectId(), stepMeta.getObjectId());
            }
        }
        repository.connectionDelegate.closeStepAttributeInsertPreparedStatement();
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "TransMeta.Log.SavingFinished"));
        }
        if (monitor != null) {
            monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.UnlockingRepository"));
        }
        repository.unlockRepository();
        // Perform a commit!
        repository.commit();
        transMeta.clearChanged();
        if (monitor != null) {
            monitor.worked(1);
        }
        if (monitor != null) {
            monitor.done();
        }
    } catch (KettleDatabaseException dbe) {
        // Oops, roll back!
        repository.rollback();
        log.logError(BaseMessages.getString(PKG, "TransMeta.Log.ErrorSavingTransformationToRepository") + Const.CR + dbe.getMessage());
        throw new KettleException(BaseMessages.getString(PKG, "TransMeta.Log.ErrorSavingTransformationToRepository"), dbe);
    }
}