use of org.pentaho.di.cluster.SlaveServer in project pentaho-kettle by pentaho.
the class Trans method sendToSlaveServer.
/**
* Send the transformation for execution to a Carte slave server.
*
* @param transMeta the transformation meta-data
* @param executionConfiguration the transformation execution configuration
* @param repository the repository
* @return The Carte object ID on the server.
* @throws KettleException if any errors occur during the dispatch to the slave server
*/
public static String sendToSlaveServer(TransMeta transMeta, TransExecutionConfiguration executionConfiguration, Repository repository, IMetaStore metaStore) throws KettleException {
String carteObjectId;
SlaveServer slaveServer = executionConfiguration.getRemoteServer();
if (slaveServer == null) {
throw new KettleException("No slave server specified");
}
if (Utils.isEmpty(transMeta.getName())) {
throw new KettleException("The transformation needs a name to uniquely identify it by on the remote server.");
}
// Inject certain internal variables to make it more intuitive.
//
Map<String, String> vars = new HashMap<>();
for (String var : Const.INTERNAL_TRANS_VARIABLES) {
vars.put(var, transMeta.getVariable(var));
}
for (String var : Const.INTERNAL_JOB_VARIABLES) {
vars.put(var, transMeta.getVariable(var));
}
executionConfiguration.getVariables().putAll(vars);
slaveServer.injectVariables(executionConfiguration.getVariables());
slaveServer.getLogChannel().setLogLevel(executionConfiguration.getLogLevel());
try {
if (executionConfiguration.isPassingExport()) {
// First export the job...
//
FileObject tempFile = KettleVFS.createTempFile("transExport", KettleVFS.Suffix.ZIP, transMeta);
// the executionConfiguration should not include a repository here because all the resources should be
// retrieved from the exported zip file
TransExecutionConfiguration clonedConfiguration = (TransExecutionConfiguration) executionConfiguration.clone();
clonedConfiguration.setRepository(null);
TopLevelResource topLevelResource = ResourceUtil.serializeResourceExportInterface(tempFile.getName().toString(), transMeta, transMeta, repository, metaStore, clonedConfiguration.getXML(), CONFIGURATION_IN_EXPORT_FILENAME);
// Send the zip file over to the slave server...
//
String result = slaveServer.sendExport(topLevelResource.getArchiveName(), RegisterPackageServlet.TYPE_TRANS, topLevelResource.getBaseResourceName());
WebResult webResult = WebResult.fromXMLString(result);
if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) {
throw new KettleException("There was an error passing the exported transformation to the remote server: " + Const.CR + webResult.getMessage());
}
carteObjectId = webResult.getId();
} else {
// Now send it off to the remote server...
//
String xml = new TransConfiguration(transMeta, executionConfiguration).getXML();
String reply = slaveServer.sendXML(xml, RegisterTransServlet.CONTEXT_PATH + "/?xml=Y");
WebResult webResult = WebResult.fromXMLString(reply);
if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) {
throw new KettleException("There was an error posting the transformation on the remote server: " + Const.CR + webResult.getMessage());
}
carteObjectId = webResult.getId();
}
// Prepare the transformation
//
String reply = slaveServer.execService(PrepareExecutionTransServlet.CONTEXT_PATH + "/?name=" + URLEncoder.encode(transMeta.getName(), "UTF-8") + "&xml=Y&id=" + carteObjectId);
WebResult webResult = WebResult.fromXMLString(reply);
if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) {
throw new KettleException("There was an error preparing the transformation for excution on the remote server: " + Const.CR + webResult.getMessage());
}
// Start the transformation
//
reply = slaveServer.execService(StartExecutionTransServlet.CONTEXT_PATH + "/?name=" + URLEncoder.encode(transMeta.getName(), "UTF-8") + "&xml=Y&id=" + carteObjectId);
webResult = WebResult.fromXMLString(reply);
if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) {
throw new KettleException("There was an error starting the transformation on the remote server: " + Const.CR + webResult.getMessage());
}
return carteObjectId;
} catch (KettleException ke) {
throw ke;
} catch (Exception e) {
throw new KettleException(e);
}
}
use of org.pentaho.di.cluster.SlaveServer in project pentaho-kettle by pentaho.
the class TransSplitter method checkClusterConfiguration.
private void checkClusterConfiguration() throws KettleException {
Map<String, ClusterSchema> map = new Hashtable<String, ClusterSchema>();
List<StepMeta> steps = originalTransformation.getSteps();
for (int i = 0; i < steps.size(); i++) {
StepMeta step = steps.get(i);
ClusterSchema clusterSchema = step.getClusterSchema();
if (clusterSchema != null) {
map.put(clusterSchema.getName(), clusterSchema);
//
if (clusterSchema.findMaster() == null) {
throw new KettleException("No master server was specified in cluster schema [" + clusterSchema + "]");
}
// Remember cluster details while we have the cluster handy
//
socketsBufferSize = Const.toInt(originalTransformation.environmentSubstitute(clusterSchema.getSocketsBufferSize()), 50000);
compressingSocketStreams = clusterSchema.isSocketsCompressed();
// Validate the number of slaves. We need at least one to have a valid cluster
//
List<SlaveServer> slaves = clusterSchema.getSlaveServersFromMasterOrLocal();
int count = 0;
for (int s = 0; s < slaves.size(); s++) {
if (!slaves.get(s).isMaster()) {
count++;
}
}
if (count <= 0) {
throw new KettleException("At least one slave server is required to be present in cluster schema [" + clusterSchema + "]");
}
}
}
if (map.size() == 0) {
throw new KettleException("No cluster schemas are being used. As such it is not possible to split and cluster this transformation.");
}
if (map.size() > 1) {
throw new KettleException("At this time we don't support the use of multiple cluster schemas in one and the same transformation.");
}
}
use of org.pentaho.di.cluster.SlaveServer in project pentaho-kettle by pentaho.
the class TransSplitter method getPort.
/**
* Get the port for the given cluster schema, slave server and step.
*
* If a port was allocated, that is returned, otherwise a new one is allocated. We need to verify that the port wasn't
* already used on the same host with perhaps several Carte instances on it. In order
*
* @param clusterSchema
* The cluster schema to use
*
* @return the port to use for that step/slaveserver/cluster combination
*/
private int getPort(ClusterSchema clusterSchema, SlaveServer sourceSlave, String sourceStepName, int sourceStepCopy, SlaveServer targetSlave, String targetStepName, int targetStepCopy) throws Exception {
SlaveServer masterSlave = clusterSchema.findMaster();
String portCacheKey = createPortCacheKey(sourceSlave, sourceStepName, sourceStepCopy, targetSlave, targetStepName, targetStepCopy);
Integer portNumber = portCache.get(portCacheKey);
if (portNumber != null) {
return portNumber.intValue();
}
String realHostname = sourceSlave.environmentSubstitute(sourceSlave.getHostname());
int port = masterSlave.allocateServerSocket(clusteredRunId, Const.toInt(clusterSchema.getBasePort(), 40000), realHostname, originalTransformation.getName(), sourceSlave.getName(), sourceStepName, Integer.toString(// Source
sourceStepCopy), targetSlave.getName(), targetStepName, // Target
Integer.toString(targetStepCopy));
portCache.put(portCacheKey, port);
return port;
}
use of org.pentaho.di.cluster.SlaveServer in project pentaho-kettle by pentaho.
the class TransSplitter method splitOriginalTransformation.
public void splitOriginalTransformation() throws KettleException {
clear();
// Mixing clusters is not supported at the moment
// Perform some basic checks on the cluster configuration.
//
findUsedOriginalSteps();
checkClusterConfiguration();
generateSlavePartitionSchemas();
try {
SlaveServer masterSlaveServer = getMasterServer();
masterTransMeta = getOriginalCopy(false, null, null);
ClusterSchema clusterSchema = originalTransformation.findFirstUsedClusterSchema();
List<SlaveServer> slaveServers = clusterSchema.getSlaveServers();
int nrSlavesNodes = clusterSchema.findNrSlaves();
boolean encrypt = false;
byte[] transformationKey = null;
PublicKey pubK = null;
if (encrypt) {
KeyPair pair = CertificateGenEncryptUtil.generateKeyPair();
pubK = pair.getPublic();
PrivateKey privK = pair.getPrivate();
Key key1 = CertificateGenEncryptUtil.generateSingleKey();
try {
transformationKey = CertificateGenEncryptUtil.encodeKeyForTransmission(privK, key1);
} catch (InvalidKeyException ex) {
masterTransMeta.getLogChannel().logError("Invalid key was used for encoding", ex);
} catch (IllegalBlockSizeException ex) {
masterTransMeta.getLogChannel().logError("Error happenned during key encoding", ex);
} catch (Exception ex) {
masterTransMeta.getLogChannel().logError("Error happenned during encryption initialization", ex);
}
}
for (int r = 0; r < referenceSteps.length; r++) {
StepMeta referenceStep = referenceSteps[r];
List<StepMeta> prevSteps = originalTransformation.findPreviousSteps(referenceStep);
int nrPreviousSteps = prevSteps.size();
for (int p = 0; p < nrPreviousSteps; p++) {
StepMeta previousStep = prevSteps.get(p);
if (!referenceStep.isClustered()) {
if (!previousStep.isClustered()) {
// No clustering involved here: just add the reference step to the master
//
StepMeta target = masterTransMeta.findStep(referenceStep.getName());
if (target == null) {
target = (StepMeta) referenceStep.clone();
masterTransMeta.addStep(target);
}
StepMeta source = masterTransMeta.findStep(previousStep.getName());
if (source == null) {
source = (StepMeta) previousStep.clone();
masterTransMeta.addStep(source);
}
// Add a hop too...
//
TransHopMeta masterHop = new TransHopMeta(source, target);
masterTransMeta.addTransHop(masterHop);
} else {
// reference step is NOT clustered
// Previous step is clustered
// --> We read from the slave server using socket readers.
// We need a reader for each slave server in the cluster
//
// Also add the reference step to the master. (cloned)
//
StepMeta masterStep = masterTransMeta.findStep(referenceStep.getName());
if (masterStep == null) {
masterStep = (StepMeta) referenceStep.clone();
masterStep.setLocation(masterStep.getLocation().x, masterStep.getLocation().y);
masterTransMeta.addStep(masterStep);
}
Queue<Integer> masterStepCopyNumbers = new LinkedList<Integer>();
for (int i = 0; i < masterStep.getCopies(); i++) {
masterStepCopyNumbers.add(i);
}
//
for (int slaveNr = 0; slaveNr < slaveServers.size(); slaveNr++) {
SlaveServer sourceSlaveServer = slaveServers.get(slaveNr);
if (!sourceSlaveServer.isMaster()) {
// MASTER: add remote input steps to the master step. That way it can receive data over sockets.
//
// SLAVE : add remote output steps to the previous step
//
TransMeta slave = getSlaveTransformation(clusterSchema, sourceSlaveServer);
// See if we can add a link to the previous using the Remote Steps concept.
//
StepMeta slaveStep = slave.findStep(previousStep.getName());
if (slaveStep == null) {
slaveStep = addSlaveCopy(slave, previousStep, sourceSlaveServer);
}
// Make sure the data finds its way back to the master.
//
// Verify the partitioning for this slave step.
// It's running in 1 or more copies depending on the number of partitions
// Get the number of target partitions...
//
StepPartitioningMeta previousStepPartitioningMeta = previousStep.getStepPartitioningMeta();
PartitionSchema previousPartitionSchema = previousStepPartitioningMeta.getPartitionSchema();
int nrOfSourceCopies = determineNrOfStepCopies(sourceSlaveServer, previousStep);
//
if (masterStep.getCopies() != 1 && masterStep.getCopies() != nrOfSourceCopies) {
// this case might be handled correctly later
String message = BaseMessages.getString(PKG, "TransSplitter.Clustering.CopyNumberStep", nrSlavesNodes, previousStep.getName(), masterStep.getName());
throw new KettleException(message);
}
//
for (int sourceCopyNr = 0; sourceCopyNr < nrOfSourceCopies; sourceCopyNr++) {
// The masterStepCopy number is increasing for each remote copy on each slave.
// This makes the master distribute to each copy of the slave properly.
// There is a check above to make sure that the master has either 1 copy or the same as slave*copies
Integer masterStepCopyNr = masterStepCopyNumbers.poll();
if (masterStepCopyNr == null) {
masterStepCopyNr = 0;
}
// We open a port on the various slave servers...
// So the source is the slave server, the target the master.
//
int port = getPort(clusterSchema, sourceSlaveServer, slaveStep.getName(), sourceCopyNr, masterSlaveServer, masterStep.getName(), masterStepCopyNr);
RemoteStep remoteMasterStep = new RemoteStep(sourceSlaveServer.getHostname(), masterSlaveServer.getHostname(), Integer.toString(port), slaveStep.getName(), sourceCopyNr, masterStep.getName(), masterStepCopyNr, sourceSlaveServer.getName(), masterSlaveServer.getName(), socketsBufferSize, compressingSocketStreams, originalTransformation.getStepFields(previousStep));
remoteMasterStep.setEncryptingStreams(encrypt);
remoteMasterStep.setKey(transformationKey);
masterStep.getRemoteInputSteps().add(remoteMasterStep);
RemoteStep remoteSlaveStep = new RemoteStep(sourceSlaveServer.getHostname(), masterSlaveServer.getHostname(), Integer.toString(port), slaveStep.getName(), sourceCopyNr, masterStep.getName(), masterStepCopyNr, sourceSlaveServer.getName(), masterSlaveServer.getName(), socketsBufferSize, compressingSocketStreams, originalTransformation.getStepFields(previousStep));
remoteSlaveStep.setEncryptingStreams(encrypt);
remoteSlaveStep.setKey(transformationKey);
slaveStep.getRemoteOutputSteps().add(remoteSlaveStep);
//
if (slaveStep.isPartitioned()) {
slaveStepCopyPartitionDistribution.addPartition(sourceSlaveServer.getName(), previousPartitionSchema.getName(), sourceCopyNr);
}
}
//
if (referenceStep.isPartitioned()) {
// Set the target partitioning schema for the source step (master)
//
StepPartitioningMeta stepPartitioningMeta = previousStepPartitioningMeta.clone();
PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema();
partitionSchema.setName(createTargetPartitionSchemaName(partitionSchema.getName()));
if (partitionSchema.isDynamicallyDefined()) {
// Expand the cluster definition to: nrOfSlaves*nrOfPartitionsPerSlave...
//
partitionSchema.expandPartitionsDynamically(clusterSchema.findNrSlaves(), originalTransformation);
}
masterStep.setTargetStepPartitioningMeta(stepPartitioningMeta);
masterTransMeta.addOrReplacePartitionSchema(partitionSchema);
// Now set the partitioning schema for the slave step...
// For the slave step, we only should those partition IDs that are interesting for the current
// slave...
//
stepPartitioningMeta = previousStepPartitioningMeta.clone();
partitionSchema = stepPartitioningMeta.getPartitionSchema();
partitionSchema.setName(createSlavePartitionSchemaName(partitionSchema.getName()));
if (partitionSchema.isDynamicallyDefined()) {
// Expand the cluster definition to: nrOfSlaves*nrOfPartitionsPerSlave...
//
partitionSchema.expandPartitionsDynamically(clusterSchema.findNrSlaves(), originalTransformation);
}
partitionSchema.retainPartitionsForSlaveServer(clusterSchema.findNrSlaves(), getSlaveServerNumber(clusterSchema, sourceSlaveServer));
slave.addOrReplacePartitionSchema(partitionSchema);
}
}
}
}
} else {
if (!previousStep.isClustered()) {
// reference step is clustered
// previous step is not clustered
// --> Add a socket writer for each slave server
//
// MASTER : add remote output step to the previous step
//
StepMeta sourceStep = masterTransMeta.findStep(previousStep.getName());
if (sourceStep == null) {
sourceStep = (StepMeta) previousStep.clone();
sourceStep.setLocation(previousStep.getLocation().x, previousStep.getLocation().y);
masterTransMeta.addStep(sourceStep);
}
Queue<Integer> masterStepCopyNumbers = new LinkedList<Integer>();
for (int i = 0; i < sourceStep.getCopies(); i++) {
masterStepCopyNumbers.add(i);
}
for (int s = 0; s < slaveServers.size(); s++) {
SlaveServer targetSlaveServer = slaveServers.get(s);
if (!targetSlaveServer.isMaster()) {
// SLAVE : add remote input step to the reference slave step...
//
TransMeta slaveTransMeta = getSlaveTransformation(clusterSchema, targetSlaveServer);
// also add the step itself.
StepMeta targetStep = slaveTransMeta.findStep(referenceStep.getName());
if (targetStep == null) {
targetStep = addSlaveCopy(slaveTransMeta, referenceStep, targetSlaveServer);
}
// Verify the partitioning for this slave step.
// It's running in 1 or more copies depending on the number of partitions
// Get the number of target partitions...
//
StepPartitioningMeta targetStepPartitioningMeta = referenceStep.getStepPartitioningMeta();
PartitionSchema targetPartitionSchema = targetStepPartitioningMeta.getPartitionSchema();
int nrOfTargetCopies = determineNrOfStepCopies(targetSlaveServer, referenceStep);
//
for (int targetCopyNr = 0; targetCopyNr < nrOfTargetCopies; targetCopyNr++) {
// The masterStepCopy number is increasing for each remote copy on each slave.
// This makes the master distribute to each copy of the slave properly.
// There is a check above to make sure that the master has either 1 copy or the same as slave*copies
Integer masterStepCopyNr = masterStepCopyNumbers.poll();
if (masterStepCopyNr == null) {
masterStepCopyNr = 0;
}
// The master step opens server socket ports
// So the IP address should be the same, in this case, the master...
//
int port = getPort(clusterSchema, masterSlaveServer, sourceStep.getName(), masterStepCopyNr, targetSlaveServer, referenceStep.getName(), targetCopyNr);
RemoteStep remoteMasterStep = new RemoteStep(masterSlaveServer.getHostname(), targetSlaveServer.getHostname(), Integer.toString(port), sourceStep.getName(), masterStepCopyNr, referenceStep.getName(), targetCopyNr, masterSlaveServer.getName(), targetSlaveServer.getName(), socketsBufferSize, compressingSocketStreams, originalTransformation.getStepFields(previousStep));
remoteMasterStep.setEncryptingStreams(encrypt);
remoteMasterStep.setKey(transformationKey);
sourceStep.getRemoteOutputSteps().add(remoteMasterStep);
RemoteStep remoteSlaveStep = new RemoteStep(masterSlaveServer.getHostname(), targetSlaveServer.getHostname(), Integer.toString(port), sourceStep.getName(), masterStepCopyNr, referenceStep.getName(), targetCopyNr, masterSlaveServer.getName(), targetSlaveServer.getName(), socketsBufferSize, compressingSocketStreams, originalTransformation.getStepFields(previousStep));
remoteSlaveStep.setEncryptingStreams(encrypt);
remoteSlaveStep.setKey(transformationKey);
targetStep.getRemoteInputSteps().add(remoteSlaveStep);
//
if (targetStep.isPartitioned()) {
slaveStepCopyPartitionDistribution.addPartition(targetSlaveServer.getName(), targetPartitionSchema.getName(), targetCopyNr);
}
}
//
if (targetStepPartitioningMeta.isPartitioned()) {
// Set the target partitioning schema for the source step (master)
//
StepPartitioningMeta stepPartitioningMeta = targetStepPartitioningMeta.clone();
PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema();
partitionSchema.setName(createTargetPartitionSchemaName(partitionSchema.getName()));
if (partitionSchema.isDynamicallyDefined()) {
// Expand the cluster definition to: nrOfSlaves*nrOfPartitionsPerSlave...
//
partitionSchema.expandPartitionsDynamically(clusterSchema.findNrSlaves(), originalTransformation);
}
sourceStep.setTargetStepPartitioningMeta(stepPartitioningMeta);
masterTransMeta.addOrReplacePartitionSchema(partitionSchema);
// Now set the partitioning schema for the slave step...
// For the slave step, we only should those partition IDs that are interesting for the current
// slave...
//
stepPartitioningMeta = targetStepPartitioningMeta.clone();
partitionSchema = stepPartitioningMeta.getPartitionSchema();
partitionSchema.setName(createSlavePartitionSchemaName(partitionSchema.getName()));
if (partitionSchema.isDynamicallyDefined()) {
// Expand the cluster definition to: nrOfSlaves*nrOfPartitionsPerSlave...
//
partitionSchema.expandPartitionsDynamically(clusterSchema.findNrSlaves(), originalTransformation);
}
partitionSchema.retainPartitionsForSlaveServer(clusterSchema.findNrSlaves(), getSlaveServerNumber(clusterSchema, targetSlaveServer));
slaveTransMeta.addOrReplacePartitionSchema(partitionSchema);
}
}
}
} else {
//
for (int slaveNr = 0; slaveNr < slaveServers.size(); slaveNr++) {
SlaveServer targetSlaveServer = slaveServers.get(slaveNr);
if (!targetSlaveServer.isMaster()) {
// SLAVE
TransMeta slaveTransMeta = getSlaveTransformation(clusterSchema, targetSlaveServer);
// This is the target step
//
StepMeta targetStep = slaveTransMeta.findStep(referenceStep.getName());
if (targetStep == null) {
targetStep = addSlaveCopy(slaveTransMeta, referenceStep, targetSlaveServer);
}
// This is the source step
//
StepMeta sourceStep = slaveTransMeta.findStep(previousStep.getName());
if (sourceStep == null) {
sourceStep = addSlaveCopy(slaveTransMeta, previousStep, targetSlaveServer);
}
// Add a hop between source and target
//
TransHopMeta slaveHop = new TransHopMeta(sourceStep, targetStep);
slaveTransMeta.addTransHop(slaveHop);
// Verify the partitioning
// That means is this case that it is possible that
//
// 1) the number of partitions is larger than the number of slaves
// 2) the partitioning method might change requiring the source step to do re-partitioning.
//
// We need to provide the source step with the information to re-partition correctly.
//
// Case 1: both source and target are partitioned on the same partition schema.
//
StepPartitioningMeta sourceStepPartitioningMeta = previousStep.getStepPartitioningMeta();
StepPartitioningMeta targetStepPartitioningMeta = referenceStep.getStepPartitioningMeta();
if (previousStep.isPartitioned() && referenceStep.isPartitioned() && sourceStepPartitioningMeta.equals(targetStepPartitioningMeta)) {
// Just divide the partitions over the available slaves...
// set the appropriate partition schema for both step...
//
StepPartitioningMeta stepPartitioningMeta = sourceStepPartitioningMeta.clone();
PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema();
partitionSchema.setName(createSlavePartitionSchemaName(partitionSchema.getName()));
if (partitionSchema.isDynamicallyDefined()) {
partitionSchema.expandPartitionsDynamically(clusterSchema.findNrSlaves(), originalTransformation);
}
partitionSchema.retainPartitionsForSlaveServer(clusterSchema.findNrSlaves(), getSlaveServerNumber(clusterSchema, targetSlaveServer));
sourceStep.setStepPartitioningMeta(stepPartitioningMeta);
targetStep.setStepPartitioningMeta(stepPartitioningMeta);
slaveTransMeta.addOrReplacePartitionSchema(partitionSchema);
} else if ((!previousStep.isPartitioned() && referenceStep.isPartitioned()) || (previousStep.isPartitioned() && referenceStep.isPartitioned() && !sourceStepPartitioningMeta.equals(targetStep.getStepPartitioningMeta()))) {
// Case 2: both source and target are partitioned on a different partition schema.
// Case 3: source is not partitioned, target is partitioned.
//
// --> This means that we're re-partitioning!!
//
PartitionSchema targetPartitionSchema = targetStepPartitioningMeta.getPartitionSchema();
PartitionSchema sourcePartitionSchema = sourceStepPartitioningMeta.getPartitionSchema();
//
for (int partSlaveNr = 0; partSlaveNr < slaveServers.size(); partSlaveNr++) {
SlaveServer sourceSlaveServer = slaveServers.get(partSlaveNr);
if (!sourceSlaveServer.isMaster()) {
// It's running in 1 or more copies depending on the number of partitions
// Get the number of target partitions...
//
Map<PartitionSchema, List<String>> partitionsMap = slaveServerPartitionsMap.get(sourceSlaveServer);
int nrOfTargetPartitions = 1;
if (targetStep.isPartitioned() && targetPartitionSchema != null) {
List<String> targetPartitionsList = partitionsMap.get(targetPartitionSchema);
nrOfTargetPartitions = targetPartitionsList.size();
} else if (targetStep.getCopies() > 1) {
nrOfTargetPartitions = targetStep.getCopies();
}
// Get the number of source partitions...
//
int nrOfSourcePartitions = 1;
if (sourceStep.isPartitioned() && sourcePartitionSchema != null) {
List<String> sourcePartitionsList = partitionsMap.get(sourcePartitionSchema);
nrOfSourcePartitions = sourcePartitionsList.size();
} else if (sourceStep.getCopies() > 1) {
nrOfSourcePartitions = sourceStep.getCopies();
}
//
for (int sourceCopyNr = 0; sourceCopyNr < nrOfSourcePartitions; sourceCopyNr++) {
for (int targetCopyNr = 0; targetCopyNr < nrOfTargetPartitions; targetCopyNr++) {
if (!targetSlaveServer.equals(sourceSlaveServer)) {
// We hit only get the remote steps, NOT the local ones.
// That's why it's OK to generate all combinations.
//
int outPort = getPort(clusterSchema, targetSlaveServer, sourceStep.getName(), sourceCopyNr, sourceSlaveServer, targetStep.getName(), targetCopyNr);
RemoteStep remoteOutputStep = new RemoteStep(targetSlaveServer.getHostname(), sourceSlaveServer.getHostname(), Integer.toString(outPort), sourceStep.getName(), sourceCopyNr, targetStep.getName(), targetCopyNr, targetSlaveServer.getName(), sourceSlaveServer.getName(), socketsBufferSize, compressingSocketStreams, originalTransformation.getStepFields(previousStep));
remoteOutputStep.setEncryptingStreams(encrypt);
remoteOutputStep.setKey(transformationKey);
sourceStep.getRemoteOutputSteps().add(remoteOutputStep);
// OK, so the source step is sending rows out on the reserved ports
// What we need to do now is link all the OTHER slaves up to them.
//
int inPort = getPort(clusterSchema, sourceSlaveServer, sourceStep.getName(), sourceCopyNr, targetSlaveServer, targetStep.getName(), targetCopyNr);
RemoteStep remoteInputStep = new RemoteStep(sourceSlaveServer.getHostname(), targetSlaveServer.getHostname(), Integer.toString(inPort), sourceStep.getName(), sourceCopyNr, targetStep.getName(), targetCopyNr, sourceSlaveServer.getName(), targetSlaveServer.getName(), socketsBufferSize, compressingSocketStreams, originalTransformation.getStepFields(previousStep));
remoteInputStep.setEncryptingStreams(encrypt);
remoteInputStep.setKey(transformationKey);
targetStep.getRemoteInputSteps().add(remoteInputStep);
}
// OK, save the partition number for the target step in the partition distribution...
//
slaveStepCopyPartitionDistribution.addPartition(sourceSlaveServer.getName(), targetPartitionSchema.getName(), targetCopyNr);
}
}
if (sourceStepPartitioningMeta.isPartitioned()) {
// Set the correct partitioning schema for the source step.
//
// Set the target partitioning schema for the target step (slave)
//
StepPartitioningMeta stepPartitioningMeta = sourceStepPartitioningMeta.clone();
PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema();
partitionSchema.setName(createSlavePartitionSchemaName(partitionSchema.getName()));
if (partitionSchema.isDynamicallyDefined()) {
// Expand the cluster definition to: nrOfSlaves*nrOfPartitionsPerSlave...
//
partitionSchema.expandPartitionsDynamically(clusterSchema.findNrSlaves(), originalTransformation);
}
partitionSchema.retainPartitionsForSlaveServer(clusterSchema.findNrSlaves(), getSlaveServerNumber(clusterSchema, targetSlaveServer));
sourceStep.setStepPartitioningMeta(stepPartitioningMeta);
slaveTransMeta.addOrReplacePartitionSchema(partitionSchema);
}
if (targetStepPartitioningMeta.isPartitioned()) {
// Set the target partitioning schema for the target step (slave)
//
StepPartitioningMeta stepPartitioningMeta = targetStepPartitioningMeta.clone();
PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema();
partitionSchema.setName(createSlavePartitionSchemaName(partitionSchema.getName()));
if (partitionSchema.isDynamicallyDefined()) {
partitionSchema.expandPartitionsDynamically(clusterSchema.findNrSlaves(), originalTransformation);
}
partitionSchema.retainPartitionsForSlaveServer(clusterSchema.findNrSlaves(), getSlaveServerNumber(clusterSchema, targetSlaveServer));
targetStep.setStepPartitioningMeta(stepPartitioningMeta);
slaveTransMeta.addOrReplacePartitionSchema(partitionSchema);
}
//
if (!sourceStepPartitioningMeta.isPartitioned() || !sourceStepPartitioningMeta.equals(targetStepPartitioningMeta)) {
// Not partitioned means the target is partitioned.
// Set the target partitioning on the source...
// Set the correct partitioning schema for the source step.
//
// Set the target partitioning schema for the target step (slave)
//
StepPartitioningMeta stepPartitioningMeta = targetStepPartitioningMeta.clone();
PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema();
partitionSchema.setName(createTargetPartitionSchemaName(partitionSchema.getName()));
if (partitionSchema.isDynamicallyDefined()) {
// Expand the cluster definition to: nrOfSlaves*nrOfPartitionsPerSlave...
//
partitionSchema.expandPartitionsDynamically(clusterSchema.findNrSlaves(), originalTransformation);
}
sourceStep.setTargetStepPartitioningMeta(stepPartitioningMeta);
slaveTransMeta.addOrReplacePartitionSchema(partitionSchema);
}
}
}
}
}
}
}
}
}
if (nrPreviousSteps == 0) {
if (!referenceStep.isClustered()) {
// Not clustered, simply add the step.
if (masterTransMeta.findStep(referenceStep.getName()) == null) {
masterTransMeta.addStep((StepMeta) referenceStep.clone());
}
} else {
for (int s = 0; s < slaveServers.size(); s++) {
SlaveServer slaveServer = slaveServers.get(s);
if (!slaveServer.isMaster()) {
// SLAVE
TransMeta slave = getSlaveTransformation(clusterSchema, slaveServer);
if (slave.findStep(referenceStep.getName()) == null) {
addSlaveCopy(slave, referenceStep, slaveServer);
}
}
}
}
}
}
//
for (int i = 0; i < referenceSteps.length; i++) {
StepMeta originalStep = referenceSteps[i];
// Also take care of the info steps...
// For example: StreamLookup, Table Input, etc.
//
StepMeta[] infoSteps = originalTransformation.getInfoStep(originalStep);
for (int p = 0; infoSteps != null && p < infoSteps.length; p++) {
StepMeta infoStep = infoSteps[p];
if (infoStep != null) {
if (!originalStep.isClustered()) {
if (!infoStep.isClustered()) {
// No clustering involved here: just add a link between the reference step and the infostep
//
StepMeta target = masterTransMeta.findStep(originalStep.getName());
StepMeta source = masterTransMeta.findStep(infoStep.getName());
// Add a hop too...
TransHopMeta masterHop = new TransHopMeta(source, target);
masterTransMeta.addTransHop(masterHop);
} else {
// reference step is NOT clustered
// Previous step is clustered
// --> We read from the slave server using socket readers.
// We need a reader for each slave server in the cluster
// On top of that we need to merge the data from all these steps using a dummy. (to make sure)
// That dummy needs to feed into Merge Join
//
int nrSlaves = clusterSchema.getSlaveServers().size();
for (int s = 0; s < nrSlaves; s++) {
SlaveServer sourceSlaveServer = clusterSchema.getSlaveServers().get(s);
if (!sourceSlaveServer.isMaster()) {
// //////////////////////////////////////////////////////////////////////////////////////////
// On the SLAVES: add a socket writer...
//
TransMeta slave = getSlaveTransformation(clusterSchema, sourceSlaveServer);
SocketWriterMeta socketWriterMeta = new SocketWriterMeta();
int port = getPort(clusterSchema, sourceSlaveServer, infoStep.getName(), 0, masterSlaveServer, originalStep.getName(), 0);
socketWriterMeta.setPort("" + port);
socketWriterMeta.setBufferSize(clusterSchema.getSocketsBufferSize());
socketWriterMeta.setFlushInterval(clusterSchema.getSocketsFlushInterval());
socketWriterMeta.setCompressed(clusterSchema.isSocketsCompressed());
StepMeta writerStep = new StepMeta(getWriterName(clusterSchema, sourceSlaveServer, infoStep.getName(), 0, masterSlaveServer, originalStep.getName(), 0), socketWriterMeta);
writerStep.setLocation(infoStep.getLocation().x + 50, infoStep.getLocation().y + 50);
writerStep.setDraw(true);
slave.addStep(writerStep);
// We also need to add a hop between infoStep and the new writer step
//
TransHopMeta slaveHop = new TransHopMeta(infoStep, writerStep);
if (slave.findTransHop(slaveHop) == null) {
slave.addTransHop(slaveHop);
}
// //////////////////////////////////////////////////////////////////////////////////////////
// On the MASTER : add a socket reader and a dummy step to merge the data...
//
SocketReaderMeta socketReaderMeta = new SocketReaderMeta();
socketReaderMeta.setPort("" + port);
socketReaderMeta.setBufferSize(clusterSchema.getSocketsBufferSize());
socketReaderMeta.setCompressed(clusterSchema.isSocketsCompressed());
StepMeta readerStep = new StepMeta(getReaderName(clusterSchema, sourceSlaveServer, infoStep.getName(), 0, masterSlaveServer, originalStep.getName(), 0), socketReaderMeta);
readerStep.setLocation(infoStep.getLocation().x, infoStep.getLocation().y + (s * FANOUT * 2) - (nrSlaves * FANOUT / 2));
readerStep.setDraw(true);
masterTransMeta.addStep(readerStep);
// Also add a single dummy step in the master that will merge the data from the slave
// transformations.
//
String dummyName = infoStep.getName();
StepMeta dummyStep = masterTransMeta.findStep(dummyName);
if (dummyStep == null) {
DummyTransMeta dummy = new DummyTransMeta();
dummyStep = new StepMeta(dummyName, dummy);
dummyStep.setLocation(infoStep.getLocation().x + (SPLIT / 2), infoStep.getLocation().y);
dummyStep.setDraw(true);
dummyStep.setDescription("This step merges the data from the various data streams coming " + "from the slave transformations.\nIt does that right before it hits the step that " + "reads from a specific (info) step.");
masterTransMeta.addStep(dummyStep);
// Now we need a hop from the dummy merge step to the actual target step (original step)
//
StepMeta masterTargetStep = masterTransMeta.findStep(originalStep.getName());
TransHopMeta targetHop = new TransHopMeta(dummyStep, masterTargetStep);
masterTransMeta.addTransHop(targetHop);
// Set the master target step as an info step... (use the cloned copy)
//
String[] infoStepNames = masterTargetStep.getStepMetaInterface().getStepIOMeta().getInfoStepnames();
if (infoStepNames != null) {
StepMeta[] is = new StepMeta[infoStepNames.length];
for (int n = 0; n < infoStepNames.length; n++) {
// OK, info steps moved to the slave steps
is[n] = slave.findStep(infoStepNames[n]);
if (infoStepNames[n].equals(infoStep.getName())) {
// We want to replace this one with the reader step: that's where we source from now
infoSteps[n] = readerStep;
}
}
masterTargetStep.getStepMetaInterface().getStepIOMeta().setInfoSteps(infoSteps);
}
}
// Add a hop between the reader step and the dummy
//
TransHopMeta mergeHop = new TransHopMeta(readerStep, dummyStep);
if (masterTransMeta.findTransHop(mergeHop) == null) {
masterTransMeta.addTransHop(mergeHop);
}
}
}
}
} else {
if (!infoStep.isClustered()) {
//
for (int s = 0; s < slaveServers.size(); s++) {
SlaveServer targetSlaveServer = slaveServers.get(s);
if (!targetSlaveServer.isMaster()) {
// MASTER
SocketWriterMeta socketWriterMeta = new SocketWriterMeta();
socketWriterMeta.setPort("" + getPort(clusterSchema, masterSlaveServer, infoStep.getName(), 0, targetSlaveServer, originalStep.getName(), 0));
socketWriterMeta.setBufferSize(clusterSchema.getSocketsBufferSize());
socketWriterMeta.setFlushInterval(clusterSchema.getSocketsFlushInterval());
socketWriterMeta.setCompressed(clusterSchema.isSocketsCompressed());
StepMeta writerStep = new StepMeta(getWriterName(clusterSchema, masterSlaveServer, infoStep.getName(), 0, targetSlaveServer, originalStep.getName(), 0), socketWriterMeta);
writerStep.setLocation(originalStep.getLocation().x, originalStep.getLocation().y + (s * FANOUT * 2) - (nrSlavesNodes * FANOUT / 2));
writerStep.setDraw(originalStep.isDrawn());
masterTransMeta.addStep(writerStep);
// The previous step: add a hop to it.
// It still has the original name as it is not clustered.
//
StepMeta previous = masterTransMeta.findStep(infoStep.getName());
if (previous == null) {
previous = (StepMeta) infoStep.clone();
masterTransMeta.addStep(previous);
}
TransHopMeta masterHop = new TransHopMeta(previous, writerStep);
masterTransMeta.addTransHop(masterHop);
// SLAVE
TransMeta slave = getSlaveTransformation(clusterSchema, targetSlaveServer);
SocketReaderMeta socketReaderMeta = new SocketReaderMeta();
socketReaderMeta.setHostname(masterSlaveServer.getHostname());
socketReaderMeta.setPort("" + getPort(clusterSchema, masterSlaveServer, infoStep.getName(), 0, targetSlaveServer, originalStep.getName(), 0));
socketReaderMeta.setBufferSize(clusterSchema.getSocketsBufferSize());
socketReaderMeta.setCompressed(clusterSchema.isSocketsCompressed());
StepMeta readerStep = new StepMeta(getReaderName(clusterSchema, masterSlaveServer, infoStep.getName(), 0, targetSlaveServer, originalStep.getName(), 0), socketReaderMeta);
readerStep.setLocation(originalStep.getLocation().x - (SPLIT / 2), originalStep.getLocation().y);
readerStep.setDraw(originalStep.isDrawn());
slave.addStep(readerStep);
// also add the step itself.
StepMeta slaveStep = slave.findStep(originalStep.getName());
if (slaveStep == null) {
slaveStep = addSlaveCopy(slave, originalStep, targetSlaveServer);
}
// And a hop from the
TransHopMeta slaveHop = new TransHopMeta(readerStep, slaveStep);
slave.addTransHop(slaveHop);
//
// Now we have to explain to the slaveStep that it has to source from previous
//
String[] infoStepNames = slaveStep.getStepMetaInterface().getStepIOMeta().getInfoStepnames();
if (infoStepNames != null) {
StepMeta[] is = new StepMeta[infoStepNames.length];
for (int n = 0; n < infoStepNames.length; n++) {
// OK, info steps moved to the slave steps
is[n] = slave.findStep(infoStepNames[n]);
if (infoStepNames[n].equals(infoStep.getName())) {
// We want to replace this one with the reader step: that's where we source from now
infoSteps[n] = readerStep;
}
}
slaveStep.getStepMetaInterface().getStepIOMeta().setInfoSteps(infoSteps);
}
}
}
} else {
//
for (int s = 0; s < slaveServers.size(); s++) {
SlaveServer slaveServer = slaveServers.get(s);
if (!slaveServer.isMaster()) {
TransMeta slave = getSlaveTransformation(clusterSchema, slaveServer);
StepMeta slaveStep = slave.findStep(originalStep.getName());
String[] infoStepNames = slaveStep.getStepMetaInterface().getStepIOMeta().getInfoStepnames();
if (infoStepNames != null) {
StepMeta[] is = new StepMeta[infoStepNames.length];
for (int n = 0; n < infoStepNames.length; n++) {
// OK, info steps moved to the slave steps
is[n] = slave.findStep(infoStepNames[n]);
// Hang on... is there a hop to the previous step?
if (slave.findTransHop(is[n], slaveStep) == null) {
TransHopMeta infoHop = new TransHopMeta(is[n], slaveStep);
slave.addTransHop(infoHop);
}
}
slaveStep.getStepMetaInterface().getStepIOMeta().setInfoSteps(infoSteps);
}
}
}
}
}
}
}
}
// Also add the original list of partition schemas to the slave step copy partition distribution...
//
slaveStepCopyPartitionDistribution.setOriginalPartitionSchemas(originalTransformation.getPartitionSchemas());
//
for (TransMeta transMeta : slaveTransMap.values()) {
transMeta.setSlaveStepCopyPartitionDistribution(slaveStepCopyPartitionDistribution);
if (encrypt) {
transMeta.setKey(pubK.getEncoded());
transMeta.setPrivateKey(false);
}
transMeta.clearChanged();
}
// do not erase partitioning schema for master transformation
// if some of steps is expected to run on master partitioned, that is the case
// when partition schema should exists as 'local' partition schema instead of slave's remote one
// see PDI-12766
// NOTE: PDI-18333 keep newly created partitionSchemas and add back in original per PDI-12766
masterTransMeta.addOrReplacePartitionSchema(originalTransformation.getPartitionSchemas());
masterTransMeta.setSlaveStepCopyPartitionDistribution(slaveStepCopyPartitionDistribution);
if (encrypt) {
masterTransMeta.setKey(pubK.getEncoded());
masterTransMeta.setPrivateKey(!false);
}
masterTransMeta.clearChanged();
// We're absolutely done here...
} catch (Exception e) {
throw new KettleException("Unexpected problem while generating master transformation", e);
}
}
use of org.pentaho.di.cluster.SlaveServer in project pentaho-kettle by pentaho.
the class Trans method getClusteredTransformationResult.
/**
* Gets the clustered transformation result.
*
* @param log the log channel interface
* @param transSplitter the TransSplitter object
* @param parentJob the parent job
* @param loggingRemoteWork log remote execution logs locally
* @return the clustered transformation result
*/
public static final Result getClusteredTransformationResult(LogChannelInterface log, TransSplitter transSplitter, Job parentJob, boolean loggingRemoteWork) {
Result result = new Result();
//
// See if the remote transformations have finished.
// We could just look at the master, but I doubt that that is enough in all situations.
//
// <-- ask these guys
SlaveServer[] slaveServers = transSplitter.getSlaveTargets();
TransMeta[] slaves = transSplitter.getSlaves();
SlaveServer masterServer;
try {
masterServer = transSplitter.getMasterServer();
} catch (KettleException e) {
log.logError("Error getting the master server", e);
masterServer = null;
result.setNrErrors(result.getNrErrors() + 1);
}
TransMeta master = transSplitter.getMaster();
//
for (int s = 0; s < slaveServers.length; s++) {
try {
// Get the detailed status of the slave transformation...
//
SlaveServerTransStatus transStatus = slaveServers[s].getTransStatus(slaves[s].getName(), "", 0);
Result transResult = transStatus.getResult(slaves[s]);
result.add(transResult);
if (loggingRemoteWork) {
log.logBasic("-- Slave : " + slaveServers[s].getName());
log.logBasic(transStatus.getLoggingString());
}
} catch (Exception e) {
result.setNrErrors(result.getNrErrors() + 1);
log.logError("Unable to contact slave server '" + slaveServers[s].getName() + "' to get result of slave transformation : " + e.toString());
}
}
//
if (master != null && master.nrSteps() > 0) {
try {
// Get the detailed status of the slave transformation...
//
SlaveServerTransStatus transStatus = masterServer.getTransStatus(master.getName(), "", 0);
Result transResult = transStatus.getResult(master);
result.add(transResult);
if (loggingRemoteWork) {
log.logBasic("-- Master : " + masterServer.getName());
log.logBasic(transStatus.getLoggingString());
}
} catch (Exception e) {
result.setNrErrors(result.getNrErrors() + 1);
log.logError("Unable to contact master server '" + masterServer.getName() + "' to get result of master transformation : " + e.toString());
}
}
return result;
}
Aggregations