use of org.pentaho.di.trans.steps.missing.MissingTrans in project pentaho-kettle by pentaho.
the class TransMeta method removeStep.
/**
* Removes a step from the transformation on a certain location (i.e. the specified index). Also marks that the
* transformation's steps have changed.
*
* @param i
* The index
*/
public void removeStep(int i) {
if (i < 0 || i >= steps.size()) {
return;
}
StepMeta removeStep = steps.get(i);
StepMetaInterface iface = removeStep.getStepMetaInterface();
if (iface instanceof StepMetaChangeListenerInterface) {
removeStepChangeListener((StepMetaChangeListenerInterface) iface);
}
steps.remove(i);
if (removeStep.getStepMetaInterface() instanceof MissingTrans) {
removeMissingTrans((MissingTrans) removeStep.getStepMetaInterface());
}
changed_steps = true;
clearCaches();
}
use of org.pentaho.di.trans.steps.missing.MissingTrans in project pentaho-kettle by pentaho.
the class KettleDatabaseRepositoryStepDelegate method loadStepMeta.
/**
* Create a new step by loading the metadata from the specified repository.
*
* @param rep
* @param stepId
* @param databases
* @param counters
* @param partitionSchemas
* @throws KettleException
*/
public StepMeta loadStepMeta(ObjectId stepId, List<DatabaseMeta> databases, List<PartitionSchema> partitionSchemas) throws KettleException {
StepMeta stepMeta = new StepMeta();
PluginRegistry registry = PluginRegistry.getInstance();
try {
RowMetaAndData r = getStep(stepId);
if (r != null) {
stepMeta.setObjectId(stepId);
stepMeta.setName(r.getString(KettleDatabaseRepository.FIELD_STEP_NAME, null));
stepMeta.setDescription(r.getString(KettleDatabaseRepository.FIELD_STEP_DESCRIPTION, null));
long id_step_type = r.getInteger(KettleDatabaseRepository.FIELD_STEP_ID_STEP_TYPE, -1L);
RowMetaAndData steptyperow = getStepType(new LongObjectId(id_step_type));
stepMeta.setStepID(steptyperow.getString(KettleDatabaseRepository.FIELD_STEP_TYPE_CODE, null));
stepMeta.setDistributes(r.getBoolean(KettleDatabaseRepository.FIELD_STEP_DISTRIBUTE, true));
int copies = (int) r.getInteger(KettleDatabaseRepository.FIELD_STEP_COPIES, 1);
String copiesString = r.getString(KettleDatabaseRepository.FIELD_STEP_COPIES_STRING, null);
if (!Utils.isEmpty(copiesString)) {
stepMeta.setCopiesString(copiesString);
} else {
stepMeta.setCopies(copies);
}
int x = (int) r.getInteger(KettleDatabaseRepository.FIELD_STEP_GUI_LOCATION_X, 0);
int y = (int) r.getInteger(KettleDatabaseRepository.FIELD_STEP_GUI_LOCATION_Y, 0);
stepMeta.setLocation(new Point(x, y));
stepMeta.setDraw(r.getBoolean(KettleDatabaseRepository.FIELD_STEP_GUI_DRAW, false));
// Generate the appropriate class...
PluginInterface sp = registry.findPluginWithId(StepPluginType.class, stepMeta.getStepID());
if (sp == null) {
stepMeta.setStepMetaInterface(new MissingTrans(stepMeta.getName(), stepMeta.getStepID()));
} else {
stepMeta.setStepMetaInterface((StepMetaInterface) registry.loadClass(sp));
}
if (stepMeta.getStepMetaInterface() != null) {
// Read the step info from the repository!
readRepCompatibleStepMeta(stepMeta.getStepMetaInterface(), repository, stepMeta.getObjectId(), databases);
stepMeta.getStepMetaInterface().readRep(repository, repository.metaStore, stepMeta.getObjectId(), databases);
}
// Get the partitioning as well...
//
stepMeta.setStepPartitioningMeta(loadStepPartitioningMeta(stepMeta.getObjectId()));
stepMeta.getStepPartitioningMeta().setPartitionSchemaAfterLoading(partitionSchemas);
// Get the cluster schema name
//
stepMeta.setClusterSchemaName(repository.getStepAttributeString(stepId, "cluster_schema"));
// Are we using a custom row distribution plugin?
//
String rowDistributionCode = repository.getStepAttributeString(stepId, 0, "row_distribution_code");
RowDistributionInterface rowDistribution = PluginRegistry.getInstance().loadClass(RowDistributionPluginType.class, rowDistributionCode, RowDistributionInterface.class);
stepMeta.setRowDistribution(rowDistribution);
// Load the attribute groups map
//
stepMeta.setAttributesMap(loadStepAttributesMap(stepId));
//
return stepMeta;
} else {
throw new KettleException(BaseMessages.getString(PKG, "StepMeta.Exception.StepInfoCouldNotBeFound", String.valueOf(stepId)));
}
} catch (KettleDatabaseException dbe) {
throw new KettleException(BaseMessages.getString(PKG, "StepMeta.Exception.StepCouldNotBeLoaded", String.valueOf(stepMeta.getObjectId())), dbe);
}
}
use of org.pentaho.di.trans.steps.missing.MissingTrans in project pentaho-kettle by pentaho.
the class TransDelegate method dataNodeToElement.
public void dataNodeToElement(final DataNode rootNode, final RepositoryElementInterface element) throws KettleException {
TransMeta transMeta = (TransMeta) element;
Set<String> privateDatabases = null;
// read the private databases
DataNode privateDbsNode = rootNode.getNode(NODE_TRANS_PRIVATE_DATABASES);
// BACKLOG-6635
if (privateDbsNode != null) {
privateDatabases = new HashSet<String>();
if (privateDbsNode.hasProperty(PROP_TRANS_PRIVATE_DATABASE_NAMES)) {
for (String privateDatabaseName : getString(privateDbsNode, PROP_TRANS_PRIVATE_DATABASE_NAMES).split(TRANS_PRIVATE_DATABASE_DELIMITER)) {
if (!privateDatabaseName.isEmpty()) {
privateDatabases.add(privateDatabaseName);
}
}
} else {
for (DataNode privateDatabase : privateDbsNode.getNodes()) {
privateDatabases.add(privateDatabase.getName());
}
}
}
transMeta.setPrivateDatabases(privateDatabases);
// read the steps...
//
DataNode stepsNode = rootNode.getNode(NODE_STEPS);
for (DataNode stepNode : stepsNode.getNodes()) {
StepMeta stepMeta = new StepMeta(new StringObjectId(stepNode.getId().toString()));
// for tracing, retain hierarchy
stepMeta.setParentTransMeta(transMeta);
// Read the basics
//
stepMeta.setName(getString(stepNode, PROP_NAME));
if (stepNode.hasProperty(PROP_DESCRIPTION)) {
stepMeta.setDescription(getString(stepNode, PROP_DESCRIPTION));
}
stepMeta.setDistributes(stepNode.getProperty(PROP_STEP_DISTRIBUTE).getBoolean());
DataProperty rowDistributionProperty = stepNode.getProperty(PROP_STEP_ROW_DISTRIBUTION);
String rowDistributionCode = rowDistributionProperty == null ? null : rowDistributionProperty.getString();
RowDistributionInterface rowDistribution = PluginRegistry.getInstance().loadClass(RowDistributionPluginType.class, rowDistributionCode, RowDistributionInterface.class);
stepMeta.setRowDistribution(rowDistribution);
stepMeta.setDraw(stepNode.getProperty(PROP_STEP_GUI_DRAW).getBoolean());
int copies = (int) stepNode.getProperty(PROP_STEP_COPIES).getLong();
String copiesString = stepNode.getProperty(PROP_STEP_COPIES_STRING) != null ? stepNode.getProperty(PROP_STEP_COPIES_STRING).getString() : StringUtils.EMPTY;
if (!Utils.isEmpty(copiesString)) {
stepMeta.setCopiesString(copiesString);
} else {
// for backward compatibility
stepMeta.setCopies(copies);
}
int x = (int) stepNode.getProperty(PROP_STEP_GUI_LOCATION_X).getLong();
int y = (int) stepNode.getProperty(PROP_STEP_GUI_LOCATION_Y).getLong();
stepMeta.setLocation(x, y);
// Load the group attributes map
//
AttributesMapUtil.loadAttributesMap(stepNode, stepMeta);
String stepType = getString(stepNode, PROP_STEP_TYPE);
// Create a new StepMetaInterface object...
//
PluginRegistry registry = PluginRegistry.getInstance();
PluginInterface stepPlugin = registry.findPluginWithId(StepPluginType.class, stepType);
StepMetaInterface stepMetaInterface = null;
if (stepPlugin != null) {
stepMetaInterface = (StepMetaInterface) registry.loadClass(stepPlugin);
// revert to the default in case we loaded an alternate version
stepType = stepPlugin.getIds()[0];
} else {
stepMeta.setStepMetaInterface((StepMetaInterface) new MissingTrans(stepMeta.getName(), stepType));
transMeta.addMissingTrans((MissingTrans) stepMeta.getStepMetaInterface());
}
stepMeta.setStepID(stepType);
// Read the metadata from the repository too...
//
RepositoryProxy proxy = new RepositoryProxy(stepNode.getNode(NODE_STEP_CUSTOM));
if (!stepMeta.isMissing()) {
readRepCompatibleStepMeta(stepMetaInterface, proxy, null, transMeta.getDatabases());
stepMetaInterface.readRep(proxy, transMeta.getMetaStore(), null, transMeta.getDatabases());
stepMeta.setStepMetaInterface(stepMetaInterface);
}
// Get the partitioning as well...
StepPartitioningMeta stepPartitioningMeta = new StepPartitioningMeta();
if (stepNode.hasProperty(PROP_PARTITIONING_SCHEMA)) {
String partSchemaId = stepNode.getProperty(PROP_PARTITIONING_SCHEMA).getRef().getId().toString();
String schemaName = repo.loadPartitionSchema(new StringObjectId(partSchemaId), null).getName();
stepPartitioningMeta.setPartitionSchemaName(schemaName);
String methodCode = getString(stepNode, PROP_PARTITIONING_METHOD);
stepPartitioningMeta.setMethod(StepPartitioningMeta.getMethod(methodCode));
if (stepPartitioningMeta.getPartitioner() != null) {
proxy = new RepositoryProxy(stepNode.getNode(NODE_PARTITIONER_CUSTOM));
stepPartitioningMeta.getPartitioner().loadRep(proxy, null);
}
stepPartitioningMeta.hasChanged(true);
}
stepMeta.setStepPartitioningMeta(stepPartitioningMeta);
stepMeta.getStepPartitioningMeta().setPartitionSchemaAfterLoading(transMeta.getPartitionSchemas());
// Get the cluster schema name
String clusterSchemaName = getString(stepNode, PROP_CLUSTER_SCHEMA);
stepMeta.setClusterSchemaName(clusterSchemaName);
if (clusterSchemaName != null && transMeta.getClusterSchemas() != null) {
// Get the cluster schema from the given name
for (ClusterSchema clusterSchema : transMeta.getClusterSchemas()) {
if (clusterSchema.getName().equals(clusterSchemaName)) {
stepMeta.setClusterSchema(clusterSchema);
break;
}
}
}
transMeta.addStep(stepMeta);
}
for (DataNode stepNode : stepsNode.getNodes()) {
ObjectId stepObjectId = new StringObjectId(stepNode.getId().toString());
StepMeta stepMeta = StepMeta.findStep(transMeta.getSteps(), stepObjectId);
//
if (stepNode.hasProperty(PROP_STEP_ERROR_HANDLING_SOURCE_STEP)) {
StepErrorMeta meta = new StepErrorMeta(transMeta, stepMeta);
meta.setTargetStep(StepMeta.findStep(transMeta.getSteps(), stepNode.getProperty(PROP_STEP_ERROR_HANDLING_TARGET_STEP).getString()));
meta.setEnabled(stepNode.getProperty(PROP_STEP_ERROR_HANDLING_IS_ENABLED).getBoolean());
meta.setNrErrorsValuename(getString(stepNode, PROP_STEP_ERROR_HANDLING_NR_VALUENAME));
meta.setErrorDescriptionsValuename(getString(stepNode, PROP_STEP_ERROR_HANDLING_DESCRIPTIONS_VALUENAME));
meta.setErrorFieldsValuename(getString(stepNode, PROP_STEP_ERROR_HANDLING_FIELDS_VALUENAME));
meta.setErrorCodesValuename(getString(stepNode, PROP_STEP_ERROR_HANDLING_CODES_VALUENAME));
meta.setMaxErrors(getString(stepNode, PROP_STEP_ERROR_HANDLING_MAX_ERRORS));
meta.setMaxPercentErrors(getString(stepNode, PROP_STEP_ERROR_HANDLING_MAX_PCT_ERRORS));
meta.setMinPercentRows(getString(stepNode, PROP_STEP_ERROR_HANDLING_MIN_PCT_ROWS));
// a bit of a trick, I know.
meta.getSourceStep().setStepErrorMeta(meta);
}
}
//
for (int i = 0; i < transMeta.nrSteps(); i++) {
StepMeta stepMeta = transMeta.getStep(i);
StepMetaInterface sii = stepMeta.getStepMetaInterface();
if (sii != null) {
sii.searchInfoAndTargetSteps(transMeta.getSteps());
}
}
// Read the notes...
//
DataNode notesNode = rootNode.getNode(NODE_NOTES);
int nrNotes = (int) notesNode.getProperty(PROP_NR_NOTES).getLong();
for (DataNode noteNode : notesNode.getNodes()) {
String xml = getString(noteNode, PROP_XML);
transMeta.addNote(new NotePadMeta(XMLHandler.getSubNode(XMLHandler.loadXMLString(xml), NotePadMeta.XML_TAG)));
}
if (transMeta.nrNotes() != nrNotes) {
throw new KettleException("The number of notes read [" + transMeta.nrNotes() + "] was not the number we expected [" + nrNotes + "]");
}
// Read the hops...
//
DataNode hopsNode = rootNode.getNode(NODE_HOPS);
int nrHops = (int) hopsNode.getProperty(PROP_NR_HOPS).getLong();
for (DataNode hopNode : hopsNode.getNodes()) {
String stepFromName = getString(hopNode, TRANS_HOP_FROM);
String stepToName = getString(hopNode, TRANS_HOP_TO);
boolean enabled = true;
if (hopNode.hasProperty(TRANS_HOP_ENABLED)) {
enabled = hopNode.getProperty(TRANS_HOP_ENABLED).getBoolean();
}
StepMeta stepFrom = StepMeta.findStep(transMeta.getSteps(), stepFromName);
StepMeta stepTo = StepMeta.findStep(transMeta.getSteps(), stepToName);
//
if (stepFrom != null && stepTo != null) {
transMeta.addTransHop(new TransHopMeta(stepFrom, stepTo, enabled));
}
}
if (transMeta.nrTransHops() != nrHops) {
throw new KettleException("The number of hops read [" + transMeta.nrTransHops() + "] was not the number we expected [" + nrHops + "]");
}
// Load the details at the end, to make sure we reference the databases correctly, etc.
//
loadTransformationDetails(rootNode, transMeta);
transMeta.eraseParameters();
DataNode paramsNode = rootNode.getNode(NODE_PARAMETERS);
int count = (int) paramsNode.getProperty(PROP_NR_PARAMETERS).getLong();
for (int idx = 0; idx < count; idx++) {
DataNode paramNode = paramsNode.getNode(TRANS_PARAM_PREFIX + idx);
String key = getString(paramNode, PARAM_KEY);
String def = getString(paramNode, PARAM_DEFAULT);
String desc = getString(paramNode, PARAM_DESC);
transMeta.addParameterDefinition(key, def, desc);
}
transMeta.activateParameters();
}
use of org.pentaho.di.trans.steps.missing.MissingTrans in project pentaho-kettle by pentaho.
the class MissingTransDialog method getErrorMessage.
private String getErrorMessage(List<MissingTrans> missingTrans, int mode) {
String message = "";
if (mode == MISSING_TRANS_STEPS) {
StringBuilder entries = new StringBuilder();
for (MissingTrans entry : missingTrans) {
if (missingTrans.indexOf(entry) == missingTrans.size() - 1) {
entries.append("- " + entry.getStepName() + " - " + entry.getMissingPluginId() + "\n\n");
} else {
entries.append("- " + entry.getStepName() + " - " + entry.getMissingPluginId() + "\n");
}
}
message = BaseMessages.getString(PKG, "MissingTransDialog.MissingTransSteps", entries.toString());
}
if (mode == MISSING_TRANS_STEP_ID) {
message = BaseMessages.getString(PKG, "MissingTransDialog.MissingTransStepId", stepname + " - " + ((MissingTrans) baseStepMeta).getMissingPluginId());
}
return message.toString();
}
use of org.pentaho.di.trans.steps.missing.MissingTrans in project pentaho-kettle by pentaho.
the class StepMetaTest method stepMetaXmlConsistency.
@Test
public void stepMetaXmlConsistency() throws Exception {
StepMeta meta = new StepMeta("id", "name", null);
StepMetaInterface smi = new MissingTrans(meta.getName(), meta.getStepID());
meta.setStepMetaInterface(smi);
StepMeta fromXml = StepMeta.fromXml(meta.getXML());
assertThat(meta.getXML(), is(fromXml.getXML()));
}
Aggregations