Example 16 with StepPartitioningMeta
use of org.pentaho.di.trans.step.StepPartitioningMeta in project pentaho-kettle by pentaho.
the class TransPainter method drawStep.
private void drawStep(StepMeta stepMeta) {
if (stepMeta == null) {
return;
}
int alpha = gc.getAlpha();
StepIOMetaInterface ioMeta = stepMeta.getStepMetaInterface().getStepIOMeta();
Point pt = stepMeta.getLocation();
if (pt == null) {
pt = new Point(50, 50);
}
Point screen = real2screen(pt.x, pt.y);
int x = screen.x;
int y = screen.y;
boolean stepError = false;
if (stepLogMap != null && !stepLogMap.isEmpty()) {
String log = stepLogMap.get(stepMeta);
if (!Utils.isEmpty(log)) {
stepError = true;
}
}
//
if (!stepMeta.getRemoteInputSteps().isEmpty()) {
gc.setLineWidth(1);
gc.setForeground(EColor.GRAY);
gc.setBackground(EColor.BACKGROUND);
gc.setFont(EFont.GRAPH);
String nrInput = Integer.toString(stepMeta.getRemoteInputSteps().size());
Point textExtent = gc.textExtent(nrInput);
// add a tiny listartHopStepttle bit of a margin
textExtent.x += 2;
textExtent.y += 2;
// Draw it an icon above the step icon.
// Draw it an icon and a half to the left
//
Point point = new Point(x - iconsize - iconsize / 2, y - iconsize);
gc.drawRectangle(point.x, point.y, textExtent.x, textExtent.y);
gc.drawText(nrInput, point.x + 1, point.y + 1);
// Now we draw an arrow from the cube to the step...
//
gc.drawLine(point.x + textExtent.x, point.y + textExtent.y / 2, x - iconsize / 2, point.y + textExtent.y / 2);
drawArrow(EImage.ARROW_DISABLED, x - iconsize / 2, point.y + textExtent.y / 2, x + iconsize / 3, y, Math.toRadians(15), 15, 1.8, null, null, null);
// Add to the list of areas...
if (!shadow) {
areaOwners.add(new AreaOwner(AreaType.REMOTE_INPUT_STEP, point.x, point.y, textExtent.x, textExtent.y, offset, stepMeta, STRING_REMOTE_INPUT_STEPS));
}
}
//
if (!stepMeta.getRemoteOutputSteps().isEmpty()) {
gc.setLineWidth(1);
gc.setForeground(EColor.GRAY);
gc.setBackground(EColor.BACKGROUND);
gc.setFont(EFont.GRAPH);
String nrOutput = Integer.toString(stepMeta.getRemoteOutputSteps().size());
Point textExtent = gc.textExtent(nrOutput);
// add a tiny little bit of a margin
textExtent.x += 2;
textExtent.y += 2;
// Draw it an icon above the step icon.
// Draw it an icon and a half to the right
//
Point point = new Point(x + 2 * iconsize + iconsize / 2 - textExtent.x, y - iconsize);
gc.drawRectangle(point.x, point.y, textExtent.x, textExtent.y);
gc.drawText(nrOutput, point.x + 1, point.y + 1);
// Now we draw an arrow from the cube to the step...
// This time, we start at the left side...
//
gc.drawLine(point.x, point.y + textExtent.y / 2, x + iconsize + iconsize / 2, point.y + textExtent.y / 2);
drawArrow(EImage.ARROW_DISABLED, x + 2 * iconsize / 3, y, x + iconsize + iconsize / 2, point.y + textExtent.y / 2, Math.toRadians(15), 15, 1.8, null, null, null);
// Add to the list of areas...
if (!shadow) {
areaOwners.add(new AreaOwner(AreaType.REMOTE_OUTPUT_STEP, point.x, point.y, textExtent.x, textExtent.y, offset, stepMeta, STRING_REMOTE_OUTPUT_STEPS));
}
}
//
if (stepMeta.isPartitioned()) {
gc.setLineWidth(1);
gc.setForeground(EColor.RED);
gc.setBackground(EColor.BACKGROUND);
gc.setFont(EFont.GRAPH);
PartitionSchema partitionSchema = stepMeta.getStepPartitioningMeta().getPartitionSchema();
if (partitionSchema != null) {
String nrInput;
if (partitionSchema.isDynamicallyDefined()) {
nrInput = "Dx" + partitionSchema.getNumberOfPartitionsPerSlave();
} else {
nrInput = "Px" + Integer.toString(partitionSchema.getPartitionIDs().size());
}
Point textExtent = gc.textExtent(nrInput);
// add a tiny little bit of a margin
textExtent.x += 2;
textExtent.y += 2;
// Draw it a 2 icons above the step icon.
// Draw it an icon and a half to the left
//
Point point = new Point(x - iconsize - iconsize / 2, y - iconsize - iconsize);
gc.drawRectangle(point.x, point.y, textExtent.x, textExtent.y);
gc.drawText(nrInput, point.x + 1, point.y + 1);
// Now we draw an arrow from the cube to the step...
//
gc.drawLine(point.x + textExtent.x, point.y + textExtent.y / 2, x - iconsize / 2, point.y + textExtent.y / 2);
gc.drawLine(x - iconsize / 2, point.y + textExtent.y / 2, x + iconsize / 3, y);
// Also draw the name of the partition schema below the box
//
gc.setForeground(EColor.GRAY);
gc.drawText(Const.NVL(partitionSchema.getName(), "<no partition name>"), point.x, point.y + textExtent.y + 3, true);
//
if (!shadow) {
areaOwners.add(new AreaOwner(AreaType.STEP_PARTITIONING, point.x, point.y, textExtent.x, textExtent.y, offset, stepMeta, STRING_PARTITIONING_CURRENT_STEP));
}
}
}
String name = stepMeta.getName();
if (stepMeta.isSelected()) {
gc.setLineWidth(linewidth + 2);
} else {
gc.setLineWidth(linewidth);
}
// Add to the list of areas...
if (!shadow) {
areaOwners.add(new AreaOwner(AreaType.STEP_ICON, x, y, iconsize, iconsize, offset, transMeta, stepMeta));
}
gc.setBackground(EColor.BACKGROUND);
gc.fillRoundRectangle(x - 1, y - 1, iconsize + 1, iconsize + 1, 8, 8);
gc.drawStepIcon(x, y, stepMeta, magnification);
if (stepError || stepMeta.isMissing()) {
gc.setForeground(EColor.RED);
} else {
gc.setForeground(EColor.CRYSTAL);
}
if (stepMeta.isSelected()) {
gc.setForeground(0, 93, 166);
}
gc.drawRoundRectangle(x - 1, y - 1, iconsize + 1, iconsize + 1, 8, 8);
Point namePosition = getNamePosition(name, screen, iconsize);
if (stepMeta.isSelected()) {
int tmpAlpha = gc.getAlpha();
gc.setAlpha(192);
gc.setBackground(216, 230, 241);
gc.fillRoundRectangle(namePosition.x - 8, namePosition.y - 2, gc.textExtent(name).x + 15, 25, BasePainter.CORNER_RADIUS_5 + 15, BasePainter.CORNER_RADIUS_5 + 15);
gc.setAlpha(tmpAlpha);
}
gc.setForeground(EColor.BLACK);
gc.setFont(EFont.GRAPH);
gc.drawText(name, namePosition.x, namePosition.y + 2, true);
boolean partitioned = false;
StepPartitioningMeta meta = stepMeta.getStepPartitioningMeta();
if (stepMeta.isPartitioned() && meta != null) {
partitioned = true;
}
String clusterMessage = "";
if (stepMeta.getClusterSchema() != null) {
clusterMessage = "C";
if (stepMeta.getClusterSchema().isDynamic()) {
clusterMessage += "xN";
} else {
clusterMessage += "x" + stepMeta.getClusterSchema().findNrSlaves();
}
Point textExtent = gc.textExtent(clusterMessage);
gc.setBackground(EColor.BACKGROUND);
gc.setForeground(EColor.BLACK);
gc.drawText(clusterMessage, x - textExtent.x + 1, y - textExtent.y + 1);
}
if (stepMeta.getCopies() != 1 && !partitioned) {
gc.setBackground(EColor.BACKGROUND);
gc.setForeground(EColor.BLACK);
String copies = "x" + stepMeta.getCopiesString();
Point textExtent = gc.textExtent(copies);
if (stepMeta.getClusterSchema() != null) {
Point clusterTextExtent = gc.textExtent(clusterMessage);
gc.drawText(copies, x - textExtent.x + 1, y - textExtent.y - clusterTextExtent.y + 1, false);
areaOwners.add(new AreaOwner(AreaType.STEP_COPIES_TEXT, x - textExtent.x + 1, y - textExtent.y - clusterTextExtent.y + 1, textExtent.x, textExtent.y, offset, transMeta, stepMeta));
} else {
gc.drawText(copies, x - textExtent.x + 1, y - textExtent.y + 1, false);
areaOwners.add(new AreaOwner(AreaType.STEP_COPIES_TEXT, x - textExtent.x + 1, y - textExtent.y + 1, textExtent.x, textExtent.y, offset, transMeta, stepMeta));
}
}
//
if (stepError) {
String log = stepLogMap.get(stepMeta);
// Show an error lines icon in the upper right corner of the step...
//
int xError = (x + iconsize) - (MINI_ICON_SIZE / 2) + 4;
int yError = y - (MINI_ICON_SIZE / 2) - 1;
Point ib = gc.getImageBounds(EImage.STEP_ERROR_RED);
gc.drawImage(EImage.STEP_ERROR_RED, xError, yError, magnification);
if (!shadow) {
areaOwners.add(new AreaOwner(AreaType.STEP_ERROR_RED_ICON, pt.x + iconsize - 3, pt.y - 8, ib.x, ib.y, offset, log, STRING_STEP_ERROR_LOG));
}
}
//
if (mouseOverSteps.contains(stepMeta)) {
gc.setTransform(translationX, translationY, 0, BasePainter.FACTOR_1_TO_1);
StepMetaInterface stepMetaInterface = stepMeta.getStepMetaInterface();
boolean mdiSupport = stepMetaInterface.getStepMetaInjectionInterface() != null || BeanInjectionInfo.isInjectionSupported(stepMetaInterface.getClass());
EImage[] miniIcons;
if (mdiSupport) {
miniIcons = new EImage[] { EImage.INPUT, EImage.EDIT, EImage.CONTEXT_MENU, EImage.OUTPUT, EImage.INJECT };
} else {
miniIcons = new EImage[] { EImage.INPUT, EImage.EDIT, EImage.CONTEXT_MENU, EImage.OUTPUT };
}
int totalHeight = 0;
int totalIconsWidth = 0;
int totalWidth = 2 * MINI_ICON_MARGIN;
for (EImage miniIcon : miniIcons) {
Point bounds = gc.getImageBounds(miniIcon);
totalWidth += bounds.x + MINI_ICON_MARGIN;
totalIconsWidth += bounds.x + MINI_ICON_MARGIN;
if (bounds.y > totalHeight) {
totalHeight = bounds.y;
}
}
totalHeight += 2 * MINI_ICON_MARGIN;
gc.setFont(EFont.SMALL);
String trimmedName = stepMeta.getName().length() < 30 ? stepMeta.getName() : stepMeta.getName().substring(0, 30);
Point nameExtent = gc.textExtent(trimmedName);
nameExtent.y += 2 * MINI_ICON_MARGIN;
nameExtent.x += 3 * MINI_ICON_MARGIN;
totalHeight += nameExtent.y;
if (nameExtent.x > totalWidth) {
totalWidth = nameExtent.x;
}
int areaX = translateToCurrentScale(x) + translateToCurrentScale(iconsize) / 2 - totalWidth / 2 + MINI_ICON_SKEW;
int areaY = translateToCurrentScale(y) + translateToCurrentScale(iconsize) + MINI_ICON_DISTANCE + BasePainter.CONTENT_MENU_INDENT;
gc.setForeground(EColor.CRYSTAL);
gc.setBackground(EColor.CRYSTAL);
gc.setLineWidth(1);
gc.fillRoundRectangle(areaX, areaY, totalWidth, totalHeight, BasePainter.CORNER_RADIUS_5, BasePainter.CORNER_RADIUS_5);
gc.setBackground(EColor.WHITE);
gc.fillRoundRectangle(areaX, areaY + nameExtent.y, totalWidth, (totalHeight - nameExtent.y), BasePainter.CORNER_RADIUS_5, BasePainter.CORNER_RADIUS_5);
gc.fillRectangle(areaX, areaY + nameExtent.y, totalWidth, (totalHeight - nameExtent.y) / 2);
gc.drawRoundRectangle(areaX, areaY, totalWidth, totalHeight, BasePainter.CORNER_RADIUS_5, BasePainter.CORNER_RADIUS_5);
gc.setForeground(EColor.WHITE);
gc.drawText(trimmedName, areaX + (totalWidth - nameExtent.x) / 2 + MINI_ICON_MARGIN, areaY + MINI_ICON_MARGIN, true);
gc.setForeground(EColor.CRYSTAL);
gc.setBackground(EColor.CRYSTAL);
gc.setFont(EFont.GRAPH);
areaOwners.add(new AreaOwner(AreaType.MINI_ICONS_BALLOON, translateTo1To1(areaX), translateTo1To1(areaY), translateTo1To1(totalWidth), translateTo1To1(totalHeight), offset, stepMeta, ioMeta));
gc.fillPolygon(new int[] { areaX + totalWidth / 2 - MINI_ICON_TRIANGLE_BASE / 2 + 1, areaY + 2, areaX + totalWidth / 2 + MINI_ICON_TRIANGLE_BASE / 2, areaY + 2, areaX + totalWidth / 2 - MINI_ICON_SKEW, areaY - MINI_ICON_DISTANCE - 3 });
gc.setBackground(EColor.WHITE);
// Put on the icons...
//
int xIcon = areaX + (totalWidth - totalIconsWidth) / 2 + MINI_ICON_MARGIN;
int yIcon = areaY + 5 + nameExtent.y;
for (int i = 0; i < miniIcons.length; i++) {
EImage miniIcon = miniIcons[i];
Point bounds = gc.getImageBounds(miniIcon);
boolean enabled = false;
switch(i) {
case // INPUT
0:
enabled = ioMeta.isInputAcceptor() || ioMeta.isInputDynamic();
areaOwners.add(new AreaOwner(AreaType.STEP_INPUT_HOP_ICON, translateTo1To1(xIcon), translateTo1To1(yIcon), translateTo1To1(bounds.x), translateTo1To1(bounds.y), offset, stepMeta, ioMeta));
break;
case // EDIT
1:
enabled = true;
areaOwners.add(new AreaOwner(AreaType.STEP_EDIT_ICON, translateTo1To1(xIcon), translateTo1To1(yIcon), translateTo1To1(bounds.x), translateTo1To1(bounds.y), offset, stepMeta, ioMeta));
break;
case // STEP_MENU
2:
enabled = true;
areaOwners.add(new AreaOwner(AreaType.STEP_MENU_ICON, translateTo1To1(xIcon), translateTo1To1(yIcon), translateTo1To1(bounds.x), translateTo1To1(bounds.y), offset, stepMeta, ioMeta));
break;
case // OUTPUT
3:
enabled = ioMeta.isOutputProducer() || ioMeta.isOutputDynamic();
areaOwners.add(new AreaOwner(AreaType.STEP_OUTPUT_HOP_ICON, translateTo1To1(xIcon), translateTo1To1(yIcon), translateTo1To1(bounds.x), translateTo1To1(bounds.y), offset, stepMeta, ioMeta));
break;
case // INJECT
4:
enabled = mdiSupport;
StepMetaInterface mdiObject = mdiSupport ? stepMetaInterface : null;
areaOwners.add(new AreaOwner(AreaType.STEP_INJECT_ICON, translateTo1To1(xIcon), translateTo1To1(yIcon), translateTo1To1(bounds.x), translateTo1To1(bounds.y), offset, stepMeta, mdiObject));
break;
default:
break;
}
if (enabled) {
gc.setAlpha(255);
} else {
gc.setAlpha(100);
}
gc.drawImage(miniIcon, xIcon, yIcon, BasePainter.FACTOR_1_TO_1);
xIcon += bounds.x + 5;
}
//
if (showTargetStreamsStep != null) {
ioMeta = showTargetStreamsStep.getStepMetaInterface().getStepIOMeta();
List<StreamInterface> targetStreams = ioMeta.getTargetStreams();
int targetsWidth = 0;
int targetsHeight = 0;
for (int i = 0; i < targetStreams.size(); i++) {
String description = targetStreams.get(i).getDescription();
Point extent = gc.textExtent(description);
if (extent.x > targetsWidth) {
targetsWidth = extent.x;
}
targetsHeight += extent.y + MINI_ICON_MARGIN;
}
targetsWidth += MINI_ICON_MARGIN;
gc.setBackground(EColor.LIGHTGRAY);
gc.fillRoundRectangle(areaX, areaY + totalHeight + 2, targetsWidth, targetsHeight, 7, 7);
gc.drawRoundRectangle(areaX, areaY + totalHeight + 2, targetsWidth, targetsHeight, 7, 7);
int targetY = areaY + totalHeight + MINI_ICON_MARGIN;
for (int i = 0; i < targetStreams.size(); i++) {
String description = targetStreams.get(i).getDescription();
Point extent = gc.textExtent(description);
gc.drawText(description, areaX + MINI_ICON_MARGIN, targetY, true);
if (i < targetStreams.size() - 1) {
gc.drawLine(areaX + MINI_ICON_MARGIN / 2, targetY + extent.y + 3, areaX + targetsWidth - MINI_ICON_MARGIN / 2, targetY + extent.y + 2);
}
areaOwners.add(new AreaOwner(AreaType.STEP_TARGET_HOP_ICON_OPTION, areaX, targetY, targetsWidth, extent.y + MINI_ICON_MARGIN, offset, stepMeta, targetStreams.get(i)));
targetY += extent.y + MINI_ICON_MARGIN;
}
gc.setBackground(EColor.BACKGROUND);
}
gc.setTransform(translationX, translationY, 0, magnification);
}
TransPainterExtension extension = new TransPainterExtension(gc, shadow, areaOwners, transMeta, stepMeta, null, x, y, 0, 0, 0, 0, offset, iconsize);
try {
ExtensionPointHandler.callExtensionPoint(LogChannel.GENERAL, KettleExtensionPoint.TransPainterStep.id, extension);
} catch (Exception e) {
LogChannel.GENERAL.logError("Error calling extension point(s) for the transformation painter step", e);
}
// Restore the previous alpha value
//
gc.setAlpha(alpha);
}
Also used :
EImage(org.pentaho.di.core.gui.PrimitiveGCInterface.EImage)
PartitionSchema(org.pentaho.di.partition.PartitionSchema)
StepMetaInterface(org.pentaho.di.trans.step.StepMetaInterface)
StepIOMetaInterface(org.pentaho.di.trans.step.StepIOMetaInterface)
Point(org.pentaho.di.core.gui.Point)
KettleExtensionPoint(org.pentaho.di.core.extension.KettleExtensionPoint)
StepPartitioningMeta(org.pentaho.di.trans.step.StepPartitioningMeta)
Point(org.pentaho.di.core.gui.Point)
KettleExtensionPoint(org.pentaho.di.core.extension.KettleExtensionPoint)
KettleException(org.pentaho.di.core.exception.KettleException)
AreaOwner(org.pentaho.di.core.gui.AreaOwner)
StreamInterface(org.pentaho.di.trans.step.errorhandling.StreamInterface)
Example 17 with StepPartitioningMeta
use of org.pentaho.di.trans.step.StepPartitioningMeta in project pentaho-kettle by pentaho.
the class TransSplitter method splitOriginalTransformation.
public void splitOriginalTransformation() throws KettleException {
clear();
// Mixing clusters is not supported at the moment
// Perform some basic checks on the cluster configuration.
//
findUsedOriginalSteps();
checkClusterConfiguration();
generateSlavePartitionSchemas();
try {
SlaveServer masterSlaveServer = getMasterServer();
masterTransMeta = getOriginalCopy(false, null, null);
ClusterSchema clusterSchema = originalTransformation.findFirstUsedClusterSchema();
List<SlaveServer> slaveServers = clusterSchema.getSlaveServers();
int nrSlavesNodes = clusterSchema.findNrSlaves();
boolean encrypt = false;
byte[] transformationKey = null;
PublicKey pubK = null;
if (encrypt) {
KeyPair pair = CertificateGenEncryptUtil.generateKeyPair();
pubK = pair.getPublic();
PrivateKey privK = pair.getPrivate();
Key key1 = CertificateGenEncryptUtil.generateSingleKey();
try {
transformationKey = CertificateGenEncryptUtil.encodeKeyForTransmission(privK, key1);
} catch (InvalidKeyException ex) {
masterTransMeta.getLogChannel().logError("Invalid key was used for encoding", ex);
} catch (IllegalBlockSizeException ex) {
masterTransMeta.getLogChannel().logError("Error happenned during key encoding", ex);
} catch (Exception ex) {
masterTransMeta.getLogChannel().logError("Error happenned during encryption initialization", ex);
}
}
for (int r = 0; r < referenceSteps.length; r++) {
StepMeta referenceStep = referenceSteps[r];
List<StepMeta> prevSteps = originalTransformation.findPreviousSteps(referenceStep);
int nrPreviousSteps = prevSteps.size();
for (int p = 0; p < nrPreviousSteps; p++) {
StepMeta previousStep = prevSteps.get(p);
if (!referenceStep.isClustered()) {
if (!previousStep.isClustered()) {
// No clustering involved here: just add the reference step to the master
//
StepMeta target = masterTransMeta.findStep(referenceStep.getName());
if (target == null) {
target = (StepMeta) referenceStep.clone();
masterTransMeta.addStep(target);
}
StepMeta source = masterTransMeta.findStep(previousStep.getName());
if (source == null) {
source = (StepMeta) previousStep.clone();
masterTransMeta.addStep(source);
}
// Add a hop too...
//
TransHopMeta masterHop = new TransHopMeta(source, target);
masterTransMeta.addTransHop(masterHop);
} else {
// reference step is NOT clustered
// Previous step is clustered
// --> We read from the slave server using socket readers.
// We need a reader for each slave server in the cluster
//
// Also add the reference step to the master. (cloned)
//
StepMeta masterStep = masterTransMeta.findStep(referenceStep.getName());
if (masterStep == null) {
masterStep = (StepMeta) referenceStep.clone();
masterStep.setLocation(masterStep.getLocation().x, masterStep.getLocation().y);
masterTransMeta.addStep(masterStep);
}
Queue<Integer> masterStepCopyNumbers = new LinkedList<Integer>();
for (int i = 0; i < masterStep.getCopies(); i++) {
masterStepCopyNumbers.add(i);
}
//
for (int slaveNr = 0; slaveNr < slaveServers.size(); slaveNr++) {
SlaveServer sourceSlaveServer = slaveServers.get(slaveNr);
if (!sourceSlaveServer.isMaster()) {
// MASTER: add remote input steps to the master step. That way it can receive data over sockets.
//
// SLAVE : add remote output steps to the previous step
//
TransMeta slave = getSlaveTransformation(clusterSchema, sourceSlaveServer);
// See if we can add a link to the previous using the Remote Steps concept.
//
StepMeta slaveStep = slave.findStep(previousStep.getName());
if (slaveStep == null) {
slaveStep = addSlaveCopy(slave, previousStep, sourceSlaveServer);
}
// Make sure the data finds its way back to the master.
//
// Verify the partitioning for this slave step.
// It's running in 1 or more copies depending on the number of partitions
// Get the number of target partitions...
//
StepPartitioningMeta previousStepPartitioningMeta = previousStep.getStepPartitioningMeta();
PartitionSchema previousPartitionSchema = previousStepPartitioningMeta.getPartitionSchema();
int nrOfSourceCopies = determineNrOfStepCopies(sourceSlaveServer, previousStep);
//
if (masterStep.getCopies() != 1 && masterStep.getCopies() != nrOfSourceCopies) {
// this case might be handled correctly later
String message = BaseMessages.getString(PKG, "TransSplitter.Clustering.CopyNumberStep", nrSlavesNodes, previousStep.getName(), masterStep.getName());
throw new KettleException(message);
}
//
for (int sourceCopyNr = 0; sourceCopyNr < nrOfSourceCopies; sourceCopyNr++) {
// The masterStepCopy number is increasing for each remote copy on each slave.
// This makes the master distribute to each copy of the slave properly.
// There is a check above to make sure that the master has either 1 copy or the same as slave*copies
Integer masterStepCopyNr = masterStepCopyNumbers.poll();
if (masterStepCopyNr == null) {
masterStepCopyNr = 0;
}
// We open a port on the various slave servers...
// So the source is the slave server, the target the master.
//
int port = getPort(clusterSchema, sourceSlaveServer, slaveStep.getName(), sourceCopyNr, masterSlaveServer, masterStep.getName(), masterStepCopyNr);
RemoteStep remoteMasterStep = new RemoteStep(sourceSlaveServer.getHostname(), masterSlaveServer.getHostname(), Integer.toString(port), slaveStep.getName(), sourceCopyNr, masterStep.getName(), masterStepCopyNr, sourceSlaveServer.getName(), masterSlaveServer.getName(), socketsBufferSize, compressingSocketStreams, originalTransformation.getStepFields(previousStep));
remoteMasterStep.setEncryptingStreams(encrypt);
remoteMasterStep.setKey(transformationKey);
masterStep.getRemoteInputSteps().add(remoteMasterStep);
RemoteStep remoteSlaveStep = new RemoteStep(sourceSlaveServer.getHostname(), masterSlaveServer.getHostname(), Integer.toString(port), slaveStep.getName(), sourceCopyNr, masterStep.getName(), masterStepCopyNr, sourceSlaveServer.getName(), masterSlaveServer.getName(), socketsBufferSize, compressingSocketStreams, originalTransformation.getStepFields(previousStep));
remoteSlaveStep.setEncryptingStreams(encrypt);
remoteSlaveStep.setKey(transformationKey);
slaveStep.getRemoteOutputSteps().add(remoteSlaveStep);
//
if (slaveStep.isPartitioned()) {
slaveStepCopyPartitionDistribution.addPartition(sourceSlaveServer.getName(), previousPartitionSchema.getName(), sourceCopyNr);
}
}
//
if (referenceStep.isPartitioned()) {
// Set the target partitioning schema for the source step (master)
//
StepPartitioningMeta stepPartitioningMeta = previousStepPartitioningMeta.clone();
PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema();
partitionSchema.setName(createTargetPartitionSchemaName(partitionSchema.getName()));
if (partitionSchema.isDynamicallyDefined()) {
// Expand the cluster definition to: nrOfSlaves*nrOfPartitionsPerSlave...
//
partitionSchema.expandPartitionsDynamically(clusterSchema.findNrSlaves(), originalTransformation);
}
masterStep.setTargetStepPartitioningMeta(stepPartitioningMeta);
masterTransMeta.addOrReplacePartitionSchema(partitionSchema);
// Now set the partitioning schema for the slave step...
// For the slave step, we only should those partition IDs that are interesting for the current
// slave...
//
stepPartitioningMeta = previousStepPartitioningMeta.clone();
partitionSchema = stepPartitioningMeta.getPartitionSchema();
partitionSchema.setName(createSlavePartitionSchemaName(partitionSchema.getName()));
if (partitionSchema.isDynamicallyDefined()) {
// Expand the cluster definition to: nrOfSlaves*nrOfPartitionsPerSlave...
//
partitionSchema.expandPartitionsDynamically(clusterSchema.findNrSlaves(), originalTransformation);
}
partitionSchema.retainPartitionsForSlaveServer(clusterSchema.findNrSlaves(), getSlaveServerNumber(clusterSchema, sourceSlaveServer));
slave.addOrReplacePartitionSchema(partitionSchema);
}
}
}
}
} else {
if (!previousStep.isClustered()) {
// reference step is clustered
// previous step is not clustered
// --> Add a socket writer for each slave server
//
// MASTER : add remote output step to the previous step
//
StepMeta sourceStep = masterTransMeta.findStep(previousStep.getName());
if (sourceStep == null) {
sourceStep = (StepMeta) previousStep.clone();
sourceStep.setLocation(previousStep.getLocation().x, previousStep.getLocation().y);
masterTransMeta.addStep(sourceStep);
}
Queue<Integer> masterStepCopyNumbers = new LinkedList<Integer>();
for (int i = 0; i < sourceStep.getCopies(); i++) {
masterStepCopyNumbers.add(i);
}
for (int s = 0; s < slaveServers.size(); s++) {
SlaveServer targetSlaveServer = slaveServers.get(s);
if (!targetSlaveServer.isMaster()) {
// SLAVE : add remote input step to the reference slave step...
//
TransMeta slaveTransMeta = getSlaveTransformation(clusterSchema, targetSlaveServer);
// also add the step itself.
StepMeta targetStep = slaveTransMeta.findStep(referenceStep.getName());
if (targetStep == null) {
targetStep = addSlaveCopy(slaveTransMeta, referenceStep, targetSlaveServer);
}
// Verify the partitioning for this slave step.
// It's running in 1 or more copies depending on the number of partitions
// Get the number of target partitions...
//
StepPartitioningMeta targetStepPartitioningMeta = referenceStep.getStepPartitioningMeta();
PartitionSchema targetPartitionSchema = targetStepPartitioningMeta.getPartitionSchema();
int nrOfTargetCopies = determineNrOfStepCopies(targetSlaveServer, referenceStep);
//
for (int targetCopyNr = 0; targetCopyNr < nrOfTargetCopies; targetCopyNr++) {
// The masterStepCopy number is increasing for each remote copy on each slave.
// This makes the master distribute to each copy of the slave properly.
// There is a check above to make sure that the master has either 1 copy or the same as slave*copies
Integer masterStepCopyNr = masterStepCopyNumbers.poll();
if (masterStepCopyNr == null) {
masterStepCopyNr = 0;
}
// The master step opens server socket ports
// So the IP address should be the same, in this case, the master...
//
int port = getPort(clusterSchema, masterSlaveServer, sourceStep.getName(), masterStepCopyNr, targetSlaveServer, referenceStep.getName(), targetCopyNr);
RemoteStep remoteMasterStep = new RemoteStep(masterSlaveServer.getHostname(), targetSlaveServer.getHostname(), Integer.toString(port), sourceStep.getName(), masterStepCopyNr, referenceStep.getName(), targetCopyNr, masterSlaveServer.getName(), targetSlaveServer.getName(), socketsBufferSize, compressingSocketStreams, originalTransformation.getStepFields(previousStep));
remoteMasterStep.setEncryptingStreams(encrypt);
remoteMasterStep.setKey(transformationKey);
sourceStep.getRemoteOutputSteps().add(remoteMasterStep);
RemoteStep remoteSlaveStep = new RemoteStep(masterSlaveServer.getHostname(), targetSlaveServer.getHostname(), Integer.toString(port), sourceStep.getName(), masterStepCopyNr, referenceStep.getName(), targetCopyNr, masterSlaveServer.getName(), targetSlaveServer.getName(), socketsBufferSize, compressingSocketStreams, originalTransformation.getStepFields(previousStep));
remoteSlaveStep.setEncryptingStreams(encrypt);
remoteSlaveStep.setKey(transformationKey);
targetStep.getRemoteInputSteps().add(remoteSlaveStep);
//
if (targetStep.isPartitioned()) {
slaveStepCopyPartitionDistribution.addPartition(targetSlaveServer.getName(), targetPartitionSchema.getName(), targetCopyNr);
}
}
//
if (targetStepPartitioningMeta.isPartitioned()) {
// Set the target partitioning schema for the source step (master)
//
StepPartitioningMeta stepPartitioningMeta = targetStepPartitioningMeta.clone();
PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema();
partitionSchema.setName(createTargetPartitionSchemaName(partitionSchema.getName()));
if (partitionSchema.isDynamicallyDefined()) {
// Expand the cluster definition to: nrOfSlaves*nrOfPartitionsPerSlave...
//
partitionSchema.expandPartitionsDynamically(clusterSchema.findNrSlaves(), originalTransformation);
}
sourceStep.setTargetStepPartitioningMeta(stepPartitioningMeta);
masterTransMeta.addOrReplacePartitionSchema(partitionSchema);
// Now set the partitioning schema for the slave step...
// For the slave step, we only should those partition IDs that are interesting for the current
// slave...
//
stepPartitioningMeta = targetStepPartitioningMeta.clone();
partitionSchema = stepPartitioningMeta.getPartitionSchema();
partitionSchema.setName(createSlavePartitionSchemaName(partitionSchema.getName()));
if (partitionSchema.isDynamicallyDefined()) {
// Expand the cluster definition to: nrOfSlaves*nrOfPartitionsPerSlave...
//
partitionSchema.expandPartitionsDynamically(clusterSchema.findNrSlaves(), originalTransformation);
}
partitionSchema.retainPartitionsForSlaveServer(clusterSchema.findNrSlaves(), getSlaveServerNumber(clusterSchema, targetSlaveServer));
slaveTransMeta.addOrReplacePartitionSchema(partitionSchema);
}
}
}
} else {
//
for (int slaveNr = 0; slaveNr < slaveServers.size(); slaveNr++) {
SlaveServer targetSlaveServer = slaveServers.get(slaveNr);
if (!targetSlaveServer.isMaster()) {
// SLAVE
TransMeta slaveTransMeta = getSlaveTransformation(clusterSchema, targetSlaveServer);
// This is the target step
//
StepMeta targetStep = slaveTransMeta.findStep(referenceStep.getName());
if (targetStep == null) {
targetStep = addSlaveCopy(slaveTransMeta, referenceStep, targetSlaveServer);
}
// This is the source step
//
StepMeta sourceStep = slaveTransMeta.findStep(previousStep.getName());
if (sourceStep == null) {
sourceStep = addSlaveCopy(slaveTransMeta, previousStep, targetSlaveServer);
}
// Add a hop between source and target
//
TransHopMeta slaveHop = new TransHopMeta(sourceStep, targetStep);
slaveTransMeta.addTransHop(slaveHop);
// Verify the partitioning
// That means is this case that it is possible that
//
// 1) the number of partitions is larger than the number of slaves
// 2) the partitioning method might change requiring the source step to do re-partitioning.
//
// We need to provide the source step with the information to re-partition correctly.
//
// Case 1: both source and target are partitioned on the same partition schema.
//
StepPartitioningMeta sourceStepPartitioningMeta = previousStep.getStepPartitioningMeta();
StepPartitioningMeta targetStepPartitioningMeta = referenceStep.getStepPartitioningMeta();
if (previousStep.isPartitioned() && referenceStep.isPartitioned() && sourceStepPartitioningMeta.equals(targetStepPartitioningMeta)) {
// Just divide the partitions over the available slaves...
// set the appropriate partition schema for both step...
//
StepPartitioningMeta stepPartitioningMeta = sourceStepPartitioningMeta.clone();
PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema();
partitionSchema.setName(createSlavePartitionSchemaName(partitionSchema.getName()));
if (partitionSchema.isDynamicallyDefined()) {
partitionSchema.expandPartitionsDynamically(clusterSchema.findNrSlaves(), originalTransformation);
}
partitionSchema.retainPartitionsForSlaveServer(clusterSchema.findNrSlaves(), getSlaveServerNumber(clusterSchema, targetSlaveServer));
sourceStep.setStepPartitioningMeta(stepPartitioningMeta);
targetStep.setStepPartitioningMeta(stepPartitioningMeta);
slaveTransMeta.addOrReplacePartitionSchema(partitionSchema);
} else if ((!previousStep.isPartitioned() && referenceStep.isPartitioned()) || (previousStep.isPartitioned() && referenceStep.isPartitioned() && !sourceStepPartitioningMeta.equals(targetStep.getStepPartitioningMeta()))) {
// Case 2: both source and target are partitioned on a different partition schema.
// Case 3: source is not partitioned, target is partitioned.
//
// --> This means that we're re-partitioning!!
//
PartitionSchema targetPartitionSchema = targetStepPartitioningMeta.getPartitionSchema();
PartitionSchema sourcePartitionSchema = sourceStepPartitioningMeta.getPartitionSchema();
//
for (int partSlaveNr = 0; partSlaveNr < slaveServers.size(); partSlaveNr++) {
SlaveServer sourceSlaveServer = slaveServers.get(partSlaveNr);
if (!sourceSlaveServer.isMaster()) {
// It's running in 1 or more copies depending on the number of partitions
// Get the number of target partitions...
//
Map<PartitionSchema, List<String>> partitionsMap = slaveServerPartitionsMap.get(sourceSlaveServer);
int nrOfTargetPartitions = 1;
if (targetStep.isPartitioned() && targetPartitionSchema != null) {
List<String> targetPartitionsList = partitionsMap.get(targetPartitionSchema);
nrOfTargetPartitions = targetPartitionsList.size();
} else if (targetStep.getCopies() > 1) {
nrOfTargetPartitions = targetStep.getCopies();
}
// Get the number of source partitions...
//
int nrOfSourcePartitions = 1;
if (sourceStep.isPartitioned() && sourcePartitionSchema != null) {
List<String> sourcePartitionsList = partitionsMap.get(sourcePartitionSchema);
nrOfSourcePartitions = sourcePartitionsList.size();
} else if (sourceStep.getCopies() > 1) {
nrOfSourcePartitions = sourceStep.getCopies();
}
//
for (int sourceCopyNr = 0; sourceCopyNr < nrOfSourcePartitions; sourceCopyNr++) {
for (int targetCopyNr = 0; targetCopyNr < nrOfTargetPartitions; targetCopyNr++) {
if (!targetSlaveServer.equals(sourceSlaveServer)) {
// We hit only get the remote steps, NOT the local ones.
// That's why it's OK to generate all combinations.
//
int outPort = getPort(clusterSchema, targetSlaveServer, sourceStep.getName(), sourceCopyNr, sourceSlaveServer, targetStep.getName(), targetCopyNr);
RemoteStep remoteOutputStep = new RemoteStep(targetSlaveServer.getHostname(), sourceSlaveServer.getHostname(), Integer.toString(outPort), sourceStep.getName(), sourceCopyNr, targetStep.getName(), targetCopyNr, targetSlaveServer.getName(), sourceSlaveServer.getName(), socketsBufferSize, compressingSocketStreams, originalTransformation.getStepFields(previousStep));
remoteOutputStep.setEncryptingStreams(encrypt);
remoteOutputStep.setKey(transformationKey);
sourceStep.getRemoteOutputSteps().add(remoteOutputStep);
// OK, so the source step is sending rows out on the reserved ports
// What we need to do now is link all the OTHER slaves up to them.
//
int inPort = getPort(clusterSchema, sourceSlaveServer, sourceStep.getName(), sourceCopyNr, targetSlaveServer, targetStep.getName(), targetCopyNr);
RemoteStep remoteInputStep = new RemoteStep(sourceSlaveServer.getHostname(), targetSlaveServer.getHostname(), Integer.toString(inPort), sourceStep.getName(), sourceCopyNr, targetStep.getName(), targetCopyNr, sourceSlaveServer.getName(), targetSlaveServer.getName(), socketsBufferSize, compressingSocketStreams, originalTransformation.getStepFields(previousStep));
remoteInputStep.setEncryptingStreams(encrypt);
remoteInputStep.setKey(transformationKey);
targetStep.getRemoteInputSteps().add(remoteInputStep);
}
// OK, save the partition number for the target step in the partition distribution...
//
slaveStepCopyPartitionDistribution.addPartition(sourceSlaveServer.getName(), targetPartitionSchema.getName(), targetCopyNr);
}
}
if (sourceStepPartitioningMeta.isPartitioned()) {
// Set the correct partitioning schema for the source step.
//
// Set the target partitioning schema for the target step (slave)
//
StepPartitioningMeta stepPartitioningMeta = sourceStepPartitioningMeta.clone();
PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema();
partitionSchema.setName(createSlavePartitionSchemaName(partitionSchema.getName()));
if (partitionSchema.isDynamicallyDefined()) {
// Expand the cluster definition to: nrOfSlaves*nrOfPartitionsPerSlave...
//
partitionSchema.expandPartitionsDynamically(clusterSchema.findNrSlaves(), originalTransformation);
}
partitionSchema.retainPartitionsForSlaveServer(clusterSchema.findNrSlaves(), getSlaveServerNumber(clusterSchema, targetSlaveServer));
sourceStep.setStepPartitioningMeta(stepPartitioningMeta);
slaveTransMeta.addOrReplacePartitionSchema(partitionSchema);
}
if (targetStepPartitioningMeta.isPartitioned()) {
// Set the target partitioning schema for the target step (slave)
//
StepPartitioningMeta stepPartitioningMeta = targetStepPartitioningMeta.clone();
PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema();
partitionSchema.setName(createSlavePartitionSchemaName(partitionSchema.getName()));
if (partitionSchema.isDynamicallyDefined()) {
partitionSchema.expandPartitionsDynamically(clusterSchema.findNrSlaves(), originalTransformation);
}
partitionSchema.retainPartitionsForSlaveServer(clusterSchema.findNrSlaves(), getSlaveServerNumber(clusterSchema, targetSlaveServer));
targetStep.setStepPartitioningMeta(stepPartitioningMeta);
slaveTransMeta.addOrReplacePartitionSchema(partitionSchema);
}
//
if (!sourceStepPartitioningMeta.isPartitioned() || !sourceStepPartitioningMeta.equals(targetStepPartitioningMeta)) {
// Not partitioned means the target is partitioned.
// Set the target partitioning on the source...
// Set the correct partitioning schema for the source step.
//
// Set the target partitioning schema for the target step (slave)
//
StepPartitioningMeta stepPartitioningMeta = targetStepPartitioningMeta.clone();
PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema();
partitionSchema.setName(createTargetPartitionSchemaName(partitionSchema.getName()));
if (partitionSchema.isDynamicallyDefined()) {
// Expand the cluster definition to: nrOfSlaves*nrOfPartitionsPerSlave...
//
partitionSchema.expandPartitionsDynamically(clusterSchema.findNrSlaves(), originalTransformation);
}
sourceStep.setTargetStepPartitioningMeta(stepPartitioningMeta);
slaveTransMeta.addOrReplacePartitionSchema(partitionSchema);
}
}
}
}
}
}
}
}
}
if (nrPreviousSteps == 0) {
if (!referenceStep.isClustered()) {
// Not clustered, simply add the step.
if (masterTransMeta.findStep(referenceStep.getName()) == null) {
masterTransMeta.addStep((StepMeta) referenceStep.clone());
}
} else {
for (int s = 0; s < slaveServers.size(); s++) {
SlaveServer slaveServer = slaveServers.get(s);
if (!slaveServer.isMaster()) {
// SLAVE
TransMeta slave = getSlaveTransformation(clusterSchema, slaveServer);
if (slave.findStep(referenceStep.getName()) == null) {
addSlaveCopy(slave, referenceStep, slaveServer);
}
}
}
}
}
}
//
for (int i = 0; i < referenceSteps.length; i++) {
StepMeta originalStep = referenceSteps[i];
// Also take care of the info steps...
// For example: StreamLookup, Table Input, etc.
//
StepMeta[] infoSteps = originalTransformation.getInfoStep(originalStep);
for (int p = 0; infoSteps != null && p < infoSteps.length; p++) {
StepMeta infoStep = infoSteps[p];
if (infoStep != null) {
if (!originalStep.isClustered()) {
if (!infoStep.isClustered()) {
// No clustering involved here: just add a link between the reference step and the infostep
//
StepMeta target = masterTransMeta.findStep(originalStep.getName());
StepMeta source = masterTransMeta.findStep(infoStep.getName());
// Add a hop too...
TransHopMeta masterHop = new TransHopMeta(source, target);
masterTransMeta.addTransHop(masterHop);
} else {
// reference step is NOT clustered
// Previous step is clustered
// --> We read from the slave server using socket readers.
// We need a reader for each slave server in the cluster
// On top of that we need to merge the data from all these steps using a dummy. (to make sure)
// That dummy needs to feed into Merge Join
//
int nrSlaves = clusterSchema.getSlaveServers().size();
for (int s = 0; s < nrSlaves; s++) {
SlaveServer sourceSlaveServer = clusterSchema.getSlaveServers().get(s);
if (!sourceSlaveServer.isMaster()) {
// //////////////////////////////////////////////////////////////////////////////////////////
// On the SLAVES: add a socket writer...
//
TransMeta slave = getSlaveTransformation(clusterSchema, sourceSlaveServer);
SocketWriterMeta socketWriterMeta = new SocketWriterMeta();
int port = getPort(clusterSchema, sourceSlaveServer, infoStep.getName(), 0, masterSlaveServer, originalStep.getName(), 0);
socketWriterMeta.setPort("" + port);
socketWriterMeta.setBufferSize(clusterSchema.getSocketsBufferSize());
socketWriterMeta.setFlushInterval(clusterSchema.getSocketsFlushInterval());
socketWriterMeta.setCompressed(clusterSchema.isSocketsCompressed());
StepMeta writerStep = new StepMeta(getWriterName(clusterSchema, sourceSlaveServer, infoStep.getName(), 0, masterSlaveServer, originalStep.getName(), 0), socketWriterMeta);
writerStep.setLocation(infoStep.getLocation().x + 50, infoStep.getLocation().y + 50);
writerStep.setDraw(true);
slave.addStep(writerStep);
// We also need to add a hop between infoStep and the new writer step
//
TransHopMeta slaveHop = new TransHopMeta(infoStep, writerStep);
if (slave.findTransHop(slaveHop) == null) {
slave.addTransHop(slaveHop);
}
// //////////////////////////////////////////////////////////////////////////////////////////
// On the MASTER : add a socket reader and a dummy step to merge the data...
//
SocketReaderMeta socketReaderMeta = new SocketReaderMeta();
socketReaderMeta.setPort("" + port);
socketReaderMeta.setBufferSize(clusterSchema.getSocketsBufferSize());
socketReaderMeta.setCompressed(clusterSchema.isSocketsCompressed());
StepMeta readerStep = new StepMeta(getReaderName(clusterSchema, sourceSlaveServer, infoStep.getName(), 0, masterSlaveServer, originalStep.getName(), 0), socketReaderMeta);
readerStep.setLocation(infoStep.getLocation().x, infoStep.getLocation().y + (s * FANOUT * 2) - (nrSlaves * FANOUT / 2));
readerStep.setDraw(true);
masterTransMeta.addStep(readerStep);
// Also add a single dummy step in the master that will merge the data from the slave
// transformations.
//
String dummyName = infoStep.getName();
StepMeta dummyStep = masterTransMeta.findStep(dummyName);
if (dummyStep == null) {
DummyTransMeta dummy = new DummyTransMeta();
dummyStep = new StepMeta(dummyName, dummy);
dummyStep.setLocation(infoStep.getLocation().x + (SPLIT / 2), infoStep.getLocation().y);
dummyStep.setDraw(true);
dummyStep.setDescription("This step merges the data from the various data streams coming " + "from the slave transformations.\nIt does that right before it hits the step that " + "reads from a specific (info) step.");
masterTransMeta.addStep(dummyStep);
// Now we need a hop from the dummy merge step to the actual target step (original step)
//
StepMeta masterTargetStep = masterTransMeta.findStep(originalStep.getName());
TransHopMeta targetHop = new TransHopMeta(dummyStep, masterTargetStep);
masterTransMeta.addTransHop(targetHop);
// Set the master target step as an info step... (use the cloned copy)
//
String[] infoStepNames = masterTargetStep.getStepMetaInterface().getStepIOMeta().getInfoStepnames();
if (infoStepNames != null) {
StepMeta[] is = new StepMeta[infoStepNames.length];
for (int n = 0; n < infoStepNames.length; n++) {
// OK, info steps moved to the slave steps
is[n] = slave.findStep(infoStepNames[n]);
if (infoStepNames[n].equals(infoStep.getName())) {
// We want to replace this one with the reader step: that's where we source from now
infoSteps[n] = readerStep;
}
}
masterTargetStep.getStepMetaInterface().getStepIOMeta().setInfoSteps(infoSteps);
}
}
// Add a hop between the reader step and the dummy
//
TransHopMeta mergeHop = new TransHopMeta(readerStep, dummyStep);
if (masterTransMeta.findTransHop(mergeHop) == null) {
masterTransMeta.addTransHop(mergeHop);
}
}
}
}
} else {
if (!infoStep.isClustered()) {
//
for (int s = 0; s < slaveServers.size(); s++) {
SlaveServer targetSlaveServer = slaveServers.get(s);
if (!targetSlaveServer.isMaster()) {
// MASTER
SocketWriterMeta socketWriterMeta = new SocketWriterMeta();
socketWriterMeta.setPort("" + getPort(clusterSchema, masterSlaveServer, infoStep.getName(), 0, targetSlaveServer, originalStep.getName(), 0));
socketWriterMeta.setBufferSize(clusterSchema.getSocketsBufferSize());
socketWriterMeta.setFlushInterval(clusterSchema.getSocketsFlushInterval());
socketWriterMeta.setCompressed(clusterSchema.isSocketsCompressed());
StepMeta writerStep = new StepMeta(getWriterName(clusterSchema, masterSlaveServer, infoStep.getName(), 0, targetSlaveServer, originalStep.getName(), 0), socketWriterMeta);
writerStep.setLocation(originalStep.getLocation().x, originalStep.getLocation().y + (s * FANOUT * 2) - (nrSlavesNodes * FANOUT / 2));
writerStep.setDraw(originalStep.isDrawn());
masterTransMeta.addStep(writerStep);
// The previous step: add a hop to it.
// It still has the original name as it is not clustered.
//
StepMeta previous = masterTransMeta.findStep(infoStep.getName());
if (previous == null) {
previous = (StepMeta) infoStep.clone();
masterTransMeta.addStep(previous);
}
TransHopMeta masterHop = new TransHopMeta(previous, writerStep);
masterTransMeta.addTransHop(masterHop);
// SLAVE
TransMeta slave = getSlaveTransformation(clusterSchema, targetSlaveServer);
SocketReaderMeta socketReaderMeta = new SocketReaderMeta();
socketReaderMeta.setHostname(masterSlaveServer.getHostname());
socketReaderMeta.setPort("" + getPort(clusterSchema, masterSlaveServer, infoStep.getName(), 0, targetSlaveServer, originalStep.getName(), 0));
socketReaderMeta.setBufferSize(clusterSchema.getSocketsBufferSize());
socketReaderMeta.setCompressed(clusterSchema.isSocketsCompressed());
StepMeta readerStep = new StepMeta(getReaderName(clusterSchema, masterSlaveServer, infoStep.getName(), 0, targetSlaveServer, originalStep.getName(), 0), socketReaderMeta);
readerStep.setLocation(originalStep.getLocation().x - (SPLIT / 2), originalStep.getLocation().y);
readerStep.setDraw(originalStep.isDrawn());
slave.addStep(readerStep);
// also add the step itself.
StepMeta slaveStep = slave.findStep(originalStep.getName());
if (slaveStep == null) {
slaveStep = addSlaveCopy(slave, originalStep, targetSlaveServer);
}
// And a hop from the
TransHopMeta slaveHop = new TransHopMeta(readerStep, slaveStep);
slave.addTransHop(slaveHop);
//
// Now we have to explain to the slaveStep that it has to source from previous
//
String[] infoStepNames = slaveStep.getStepMetaInterface().getStepIOMeta().getInfoStepnames();
if (infoStepNames != null) {
StepMeta[] is = new StepMeta[infoStepNames.length];
for (int n = 0; n < infoStepNames.length; n++) {
// OK, info steps moved to the slave steps
is[n] = slave.findStep(infoStepNames[n]);
if (infoStepNames[n].equals(infoStep.getName())) {
// We want to replace this one with the reader step: that's where we source from now
infoSteps[n] = readerStep;
}
}
slaveStep.getStepMetaInterface().getStepIOMeta().setInfoSteps(infoSteps);
}
}
}
} else {
//
for (int s = 0; s < slaveServers.size(); s++) {
SlaveServer slaveServer = slaveServers.get(s);
if (!slaveServer.isMaster()) {
TransMeta slave = getSlaveTransformation(clusterSchema, slaveServer);
StepMeta slaveStep = slave.findStep(originalStep.getName());
String[] infoStepNames = slaveStep.getStepMetaInterface().getStepIOMeta().getInfoStepnames();
if (infoStepNames != null) {
StepMeta[] is = new StepMeta[infoStepNames.length];
for (int n = 0; n < infoStepNames.length; n++) {
// OK, info steps moved to the slave steps
is[n] = slave.findStep(infoStepNames[n]);
// Hang on... is there a hop to the previous step?
if (slave.findTransHop(is[n], slaveStep) == null) {
TransHopMeta infoHop = new TransHopMeta(is[n], slaveStep);
slave.addTransHop(infoHop);
}
}
slaveStep.getStepMetaInterface().getStepIOMeta().setInfoSteps(infoSteps);
}
}
}
}
}
}
}
}
// Also add the original list of partition schemas to the slave step copy partition distribution...
//
slaveStepCopyPartitionDistribution.setOriginalPartitionSchemas(originalTransformation.getPartitionSchemas());
//
for (TransMeta transMeta : slaveTransMap.values()) {
transMeta.setSlaveStepCopyPartitionDistribution(slaveStepCopyPartitionDistribution);
if (encrypt) {
transMeta.setKey(pubK.getEncoded());
transMeta.setPrivateKey(false);
}
transMeta.clearChanged();
}
// do not erase partitioning schema for master transformation
// if some of steps is expected to run on master partitioned, that is the case
// when partition schema should exists as 'local' partition schema instead of slave's remote one
// see PDI-12766
masterTransMeta.setPartitionSchemas(originalTransformation.getPartitionSchemas());
masterTransMeta.setSlaveStepCopyPartitionDistribution(slaveStepCopyPartitionDistribution);
if (encrypt) {
masterTransMeta.setKey(pubK.getEncoded());
masterTransMeta.setPrivateKey(!false);
}
masterTransMeta.clearChanged();
// We're absolutely done here...
} catch (Exception e) {
throw new KettleException("Unexpected problem while generating master transformation", e);
}
}
Also used :
KettleException(org.pentaho.di.core.exception.KettleException)
PrivateKey(java.security.PrivateKey)
SocketWriterMeta(org.pentaho.di.trans.steps.socketwriter.SocketWriterMeta)
TransMeta(org.pentaho.di.trans.TransMeta)
DummyTransMeta(org.pentaho.di.trans.steps.dummytrans.DummyTransMeta)
IllegalBlockSizeException(javax.crypto.IllegalBlockSizeException)
SlaveServer(org.pentaho.di.cluster.SlaveServer)
StepPartitioningMeta(org.pentaho.di.trans.step.StepPartitioningMeta)
DummyTransMeta(org.pentaho.di.trans.steps.dummytrans.DummyTransMeta)
ArrayList(java.util.ArrayList)
LinkedList(java.util.LinkedList)
List(java.util.List)
KeyPair(java.security.KeyPair)
SocketReaderMeta(org.pentaho.di.trans.steps.socketreader.SocketReaderMeta)
PartitionSchema(org.pentaho.di.partition.PartitionSchema)
PublicKey(java.security.PublicKey)
InvalidKeyException(java.security.InvalidKeyException)
StepMeta(org.pentaho.di.trans.step.StepMeta)
KettleException(org.pentaho.di.core.exception.KettleException)
IllegalBlockSizeException(javax.crypto.IllegalBlockSizeException)
InvalidKeyException(java.security.InvalidKeyException)
LinkedList(java.util.LinkedList)
RemoteStep(org.pentaho.di.trans.step.RemoteStep)
TransHopMeta(org.pentaho.di.trans.TransHopMeta)
HashMap(java.util.HashMap)
Map(java.util.Map)
ClusterSchema(org.pentaho.di.cluster.ClusterSchema)
PublicKey(java.security.PublicKey)
Key(java.security.Key)
PrivateKey(java.security.PrivateKey)
Example 18 with StepPartitioningMeta
use of org.pentaho.di.trans.step.StepPartitioningMeta in project pentaho-kettle by pentaho.
the class TransSplitter method determineNrOfStepCopies.
/**
* Calculate the number of step copies in a step.<br>
* If a step is not running clustered, it's simply returning getCopies().<br>
* If a step is clustered and not doing any partitioning, it's simply returning getCopies().<br>
* If a step is clustered and partitioned, we need to look in the partitioning map for the specified slave server.<br>
* That is because the number of copies can vary over the slaves. (5 partitions over 3 slaves for example)
*
* @param slaveServer
* the slave server
* @param step
* the reference step
* @return the number of step copies that we run.
*/
private int determineNrOfStepCopies(SlaveServer slaveServer, StepMeta step) {
if (!step.isClustered()) {
return step.getCopies();
}
if (!step.isPartitioned()) {
return step.getCopies();
}
if (slaveServer.isMaster()) {
return step.getCopies();
}
// Partitioned and clustered...
//
StepPartitioningMeta stepPartitioningMeta = step.getStepPartitioningMeta();
PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema();
Map<PartitionSchema, List<String>> partitionMap = slaveServerPartitionsMap.get(slaveServer);
List<String> partitionList = partitionMap.get(partitionSchema);
return partitionList.size();
}
Also used :
PartitionSchema(org.pentaho.di.partition.PartitionSchema)
ArrayList(java.util.ArrayList)
LinkedList(java.util.LinkedList)
List(java.util.List)
StepPartitioningMeta(org.pentaho.di.trans.step.StepPartitioningMeta)
Example 19 with StepPartitioningMeta
use of org.pentaho.di.trans.step.StepPartitioningMeta in project pentaho-kettle by pentaho.
the class KettleDatabaseRepositoryTransDelegate method loadTransformation.
/**
* Read a transformation with a certain name from a repository
*
* @param rep
* The repository to read from.
* @param transname
* The name of the transformation.
* @param repdir
* the path to the repository directory
* @param monitor
* The progress monitor to display the progress of the file-open operation in a dialog
* @param setInternalVariables
* true if you want to set the internal variables based on this transformation information
*/
public TransMeta loadTransformation(TransMeta transMeta, String transname, RepositoryDirectoryInterface repdir, ProgressMonitorListener monitor, boolean setInternalVariables) throws KettleException {
transMeta.setRepository(repository);
transMeta.setMetaStore(repository.metaStore);
synchronized (repository) {
try {
String pathAndName = repdir.isRoot() ? repdir + transname : repdir + RepositoryDirectory.DIRECTORY_SEPARATOR + transname;
transMeta.setName(transname);
transMeta.setRepositoryDirectory(repdir);
// Get the transformation id
if (log.isDetailed()) {
log.logDetailed(BaseMessages.getString(PKG, "TransMeta.Log.LookingForTransformation", transname, repdir.getPath()));
}
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingTransformationInfoTask.Title"));
}
transMeta.setObjectId(getTransformationID(transname, repdir.getObjectId()));
if (monitor != null) {
monitor.worked(1);
}
// If no valid id is available in the database, then give error...
if (transMeta.getObjectId() != null) {
ObjectId[] noteids = repository.getTransNoteIDs(transMeta.getObjectId());
ObjectId[] stepids = repository.getStepIDs(transMeta.getObjectId());
ObjectId[] hopids = getTransHopIDs(transMeta.getObjectId());
int nrWork = 3 + noteids.length + stepids.length + hopids.length;
if (monitor != null) {
monitor.beginTask(BaseMessages.getString(PKG, "TransMeta.Monitor.LoadingTransformationTask.Title") + pathAndName, nrWork);
}
if (log.isDetailed()) {
log.logDetailed(BaseMessages.getString(PKG, "TransMeta.Log.LoadingTransformation", transMeta.getName()));
}
// Load the common database connections
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingTheAvailableSharedObjectsTask.Title"));
}
try {
transMeta.setSharedObjects(readTransSharedObjects(transMeta));
} catch (Exception e) {
log.logError(BaseMessages.getString(PKG, "TransMeta.ErrorReadingSharedObjects.Message", e.toString()));
log.logError(Const.getStackTracker(e));
}
if (monitor != null) {
monitor.worked(1);
}
// Load the notes...
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingNoteTask.Title"));
}
for (int i = 0; i < noteids.length; i++) {
NotePadMeta ni = repository.notePadDelegate.loadNotePadMeta(noteids[i]);
if (transMeta.indexOfNote(ni) < 0) {
transMeta.addNote(ni);
}
if (monitor != null) {
monitor.worked(1);
}
}
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingStepsTask.Title"));
}
// read all the attributes
repository.connectionDelegate.fillStepAttributesBuffer(transMeta.getObjectId());
// on one go!
for (int i = 0; i < stepids.length; i++) {
if (log.isDetailed()) {
log.logDetailed(BaseMessages.getString(PKG, "TransMeta.Log.LoadingStepWithID") + stepids[i]);
}
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingStepTask.Title") + (i + 1) + "/" + (stepids.length));
}
StepMeta stepMeta = repository.stepDelegate.loadStepMeta(stepids[i], transMeta.getDatabases(), transMeta.getPartitionSchemas());
if (stepMeta.isMissing()) {
transMeta.addMissingTrans((MissingTrans) stepMeta.getStepMetaInterface());
}
// In this case, we just add or replace the shared steps.
// The repository is considered "more central"
transMeta.addOrReplaceStep(stepMeta);
if (monitor != null) {
monitor.worked(1);
}
}
if (monitor != null) {
monitor.worked(1);
}
// clear the buffer (should be empty anyway)
repository.connectionDelegate.setStepAttributesBuffer(null);
// Have all StreamValueLookups, etc. reference the correct source steps...
for (int i = 0; i < transMeta.nrSteps(); i++) {
StepMetaInterface sii = transMeta.getStep(i).getStepMetaInterface();
sii.searchInfoAndTargetSteps(transMeta.getSteps());
}
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.LoadingTransformationDetailsTask.Title"));
}
loadRepTrans(transMeta);
if (monitor != null) {
monitor.worked(1);
}
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingHopTask.Title"));
}
for (int i = 0; i < hopids.length; i++) {
TransHopMeta hi = loadTransHopMeta(hopids[i], transMeta.getSteps());
if (hi != null) {
transMeta.addTransHop(hi);
}
if (monitor != null) {
monitor.worked(1);
}
}
//
for (int i = 0; i < transMeta.nrSteps(); i++) {
StepPartitioningMeta stepPartitioningMeta = transMeta.getStep(i).getStepPartitioningMeta();
if (stepPartitioningMeta != null) {
stepPartitioningMeta.setPartitionSchemaAfterLoading(transMeta.getPartitionSchemas());
}
}
//
for (int i = 0; i < transMeta.nrSteps(); i++) {
transMeta.getStep(i).setClusterSchemaAfterLoading(transMeta.getClusterSchemas());
}
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.ReadingTheDependenciesTask.Title"));
}
ObjectId[] depids = repository.getTransDependencyIDs(transMeta.getObjectId());
for (int i = 0; i < depids.length; i++) {
TransDependency td = loadTransDependency(depids[i], transMeta.getDatabases());
transMeta.addDependency(td);
}
if (monitor != null) {
monitor.worked(1);
}
// Load the group attributes map
//
transMeta.setAttributesMap(loadTransAttributesMap(transMeta.getObjectId()));
//
for (int i = 0; i < transMeta.nrSteps(); i++) {
StepMeta stepMeta = transMeta.getStep(i);
String sourceStep = repository.getStepAttributeString(stepMeta.getObjectId(), "step_error_handling_source_step");
if (sourceStep != null) {
StepErrorMeta stepErrorMeta = repository.stepDelegate.loadStepErrorMeta(transMeta, stepMeta, transMeta.getSteps());
// a bit of a trick, I know.
stepErrorMeta.getSourceStep().setStepErrorMeta(stepErrorMeta);
}
}
// Load all the log tables for the transformation...
//
RepositoryAttributeInterface attributeInterface = new KettleDatabaseRepositoryTransAttribute(repository.connectionDelegate, transMeta.getObjectId());
for (LogTableInterface logTable : transMeta.getLogTables()) {
logTable.loadFromRepository(attributeInterface);
}
if (monitor != null) {
monitor.subTask(BaseMessages.getString(PKG, "TransMeta.Monitor.SortingStepsTask.Title"));
}
transMeta.sortSteps();
if (monitor != null) {
monitor.worked(1);
}
if (monitor != null) {
monitor.done();
}
} else {
throw new KettleException(BaseMessages.getString(PKG, "TransMeta.Exception.TransformationDoesNotExist") + transMeta.getName());
}
if (log.isDetailed()) {
log.logDetailed(BaseMessages.getString(PKG, "TransMeta.Log.LoadedTransformation2", transname, String.valueOf(transMeta.getRepositoryDirectory() == null)));
log.logDetailed(BaseMessages.getString(PKG, "TransMeta.Log.LoadedTransformation", transname, transMeta.getRepositoryDirectory().getPath()));
}
// close prepared statements, minimize locking etc.
//
repository.connectionDelegate.closeAttributeLookupPreparedStatements();
return transMeta;
} catch (KettleDatabaseException e) {
log.logError(BaseMessages.getString(PKG, "TransMeta.Log.DatabaseErrorOccuredReadingTransformation") + Const.CR + e);
throw new KettleException(BaseMessages.getString(PKG, "TransMeta.Exception.DatabaseErrorOccuredReadingTransformation"), e);
} catch (Exception e) {
log.logError(BaseMessages.getString(PKG, "TransMeta.Log.DatabaseErrorOccuredReadingTransformation") + Const.CR + e);
throw new KettleException(BaseMessages.getString(PKG, "TransMeta.Exception.DatabaseErrorOccuredReadingTransformation2"), e);
} finally {
transMeta.initializeVariablesFrom(null);
if (setInternalVariables) {
transMeta.setInternalKettleVariables();
}
}
}
}
Also used :
KettleException(org.pentaho.di.core.exception.KettleException)
LongObjectId(org.pentaho.di.repository.LongObjectId)
ObjectId(org.pentaho.di.repository.ObjectId)
KettleDatabaseException(org.pentaho.di.core.exception.KettleDatabaseException)
StepMetaInterface(org.pentaho.di.trans.step.StepMetaInterface)
StepErrorMeta(org.pentaho.di.trans.step.StepErrorMeta)
TransDependency(org.pentaho.di.trans.TransDependency)
ValueMetaString(org.pentaho.di.core.row.value.ValueMetaString)
StepPartitioningMeta(org.pentaho.di.trans.step.StepPartitioningMeta)
StepMeta(org.pentaho.di.trans.step.StepMeta)
KettleException(org.pentaho.di.core.exception.KettleException)
KettleDatabaseException(org.pentaho.di.core.exception.KettleDatabaseException)
RepositoryAttributeInterface(org.pentaho.di.repository.RepositoryAttributeInterface)
LogTableInterface(org.pentaho.di.core.logging.LogTableInterface)
NotePadMeta(org.pentaho.di.core.NotePadMeta)
TransHopMeta(org.pentaho.di.trans.TransHopMeta)
Example 20 with StepPartitioningMeta
use of org.pentaho.di.trans.step.StepPartitioningMeta in project pentaho-kettle by pentaho.
the class TransPartitioningTest method prepareStepMetas_cl1_cl1.
/**
* This case simulates when we do have 2 step partitioned with one same partitioner We want to get a 'swim-lanes'
* transformation
*
* @throws KettlePluginException
*/
private void prepareStepMetas_cl1_cl1() throws KettlePluginException {
StepMeta dummy1 = new StepMeta(ONE, null);
StepMeta dummy2 = new StepMeta(TWO, null);
PartitionSchema schema = new PartitionSchema("p1", Arrays.asList(new String[] { PID1, PID2 }));
// for delayed binding StepPartitioning meta does not achieve
// schema name when using in constructor so we have to set it
// explicitly. See equals implementation for StepPartitioningMeta.
StepPartitioningMeta partMeta = new StepPartitioningMeta("Mirror to all partitions", schema);
// that is what I am talking about:
partMeta.setPartitionSchemaName(schema.getName());
dummy1.setStepPartitioningMeta(partMeta);
dummy2.setStepPartitioningMeta(partMeta);
chain.add(dummy1);
chain.add(dummy2);
for (StepMeta item : chain) {
item.setStepMetaInterface(new DummyTransMeta());
}
}
Also used :
PartitionSchema(org.pentaho.di.partition.PartitionSchema)
StepPartitioningMeta(org.pentaho.di.trans.step.StepPartitioningMeta)
StepMeta(org.pentaho.di.trans.step.StepMeta)
DummyTransMeta(org.pentaho.di.trans.steps.dummytrans.DummyTransMeta)
Aggregations
StepPartitioningMeta (org.pentaho.di.trans.step.StepPartitioningMeta)23
PartitionSchema (org.pentaho.di.partition.PartitionSchema)18
StepMeta (org.pentaho.di.trans.step.StepMeta)14
KettleException (org.pentaho.di.core.exception.KettleException)8
TransMeta (org.pentaho.di.trans.TransMeta)7
Test (org.junit.Test)6
StepMetaInterface (org.pentaho.di.trans.step.StepMetaInterface)5
DummyTransMeta (org.pentaho.di.trans.steps.dummytrans.DummyTransMeta)5
ArrayList (java.util.ArrayList)4
LinkedList (java.util.LinkedList)4
List (java.util.List)4
ClusterSchema (org.pentaho.di.cluster.ClusterSchema)4
NotePadMeta (org.pentaho.di.core.NotePadMeta)4
TransHopMeta (org.pentaho.di.trans.TransHopMeta)4
StepErrorMeta (org.pentaho.di.trans.step.StepErrorMeta)4
SlaveServer (org.pentaho.di.cluster.SlaveServer)3
KettleDatabaseException (org.pentaho.di.core.exception.KettleDatabaseException)3
KettleExtensionPoint (org.pentaho.di.core.extension.KettleExtensionPoint)3
ValueMetaString (org.pentaho.di.core.row.value.ValueMetaString)3
HashMap (java.util.HashMap)2
