Examples with PartitionSchema org.pentaho.di.partition.PartitionSchema used on opensource projects

Example 16 with PartitionSchema

use of org.pentaho.di.partition.PartitionSchema in project pentaho-kettle by pentaho.

the class SlaveStepCopyPartitionDistribution method getXML.

public String getXML() {
    StringBuilder xml = new StringBuilder(200);
    xml.append("  ").append(XMLHandler.openTag(XML_TAG)).append(Const.CR);
    List<SlaveStepCopy> list = new ArrayList<SlaveStepCopy>(distribution.keySet());
    Collections.sort(list);
    for (SlaveStepCopy copy : list) {
        int partition = getPartition(copy);
        xml.append("    ").append(XMLHandler.openTag("entry")).append(Const.CR);
        xml.append("      ").append(XMLHandler.addTagValue("slavename", copy.slaveServerName));
        xml.append("      ").append(XMLHandler.addTagValue("partition_schema_name", copy.partitionSchemaName));
        xml.append("      ").append(XMLHandler.addTagValue("stepcopy", copy.stepCopyNr));
        xml.append("      ").append(XMLHandler.addTagValue("partition", partition));
        xml.append("    ").append(XMLHandler.closeTag("entry")).append(Const.CR);
    }
    if (originalPartitionSchemas != null) {
        xml.append("    ").append(XMLHandler.openTag("original-partition-schemas")).append(Const.CR);
        for (PartitionSchema partitionSchema : originalPartitionSchemas) {
            xml.append(partitionSchema.getXML());
        }
        xml.append("    ").append(XMLHandler.closeTag("original-partition-schemas")).append(Const.CR);
    }
    xml.append("  ").append(XMLHandler.closeTag(XML_TAG)).append(Const.CR);
    return xml.toString();
}

Example 17 with PartitionSchema

use of org.pentaho.di.partition.PartitionSchema in project pentaho-kettle by pentaho.

the class BaseStep method init.

/*
   * (non-Javadoc)
   *
   * @see org.pentaho.di.trans.step.StepInterface#init(org.pentaho.di.trans.step.StepMetaInterface,
   * org.pentaho.di.trans.step.StepDataInterface)
   */
@Override
public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
    sdi.setStatus(StepExecutionStatus.STATUS_INIT);
    String slaveNr = transMeta.getVariable(Const.INTERNAL_VARIABLE_SLAVE_SERVER_NUMBER);
    String clusterSize = transMeta.getVariable(Const.INTERNAL_VARIABLE_CLUSTER_SIZE);
    boolean master = "Y".equalsIgnoreCase(transMeta.getVariable(Const.INTERNAL_VARIABLE_CLUSTER_MASTER));
    if (!Utils.isEmpty(slaveNr) && !Utils.isEmpty(clusterSize) && !master) {
        this.slaveNr = Integer.parseInt(slaveNr);
        this.clusterSize = Integer.parseInt(clusterSize);
        if (log.isDetailed()) {
            logDetailed(BaseMessages.getString(PKG, "BaseStep.Log.ReleasedServerSocketOnPort", slaveNr, clusterSize));
        }
    } else {
        this.slaveNr = 0;
        this.clusterSize = 0;
    }
    // Also set the internal variable for the partition
    // 
    SlaveStepCopyPartitionDistribution partitionDistribution = transMeta.getSlaveStepCopyPartitionDistribution();
    if (stepMeta.isPartitioned()) {
        // 
        if (partitionDistribution != null && !partitionDistribution.getDistribution().isEmpty()) {
            String slaveServerName = getVariable(Const.INTERNAL_VARIABLE_SLAVE_SERVER_NAME);
            int stepCopyNr = stepcopy;
            // Look up the partition nr...
            // Set the partition ID (string) as well as the partition nr [0..size[
            // 
            PartitionSchema partitionSchema = stepMeta.getStepPartitioningMeta().getPartitionSchema();
            int partitionNr = partitionDistribution.getPartition(slaveServerName, partitionSchema.getName(), stepCopyNr);
            if (partitionNr >= 0) {
                String partitionNrString = new DecimalFormat("000").format(partitionNr);
                setVariable(Const.INTERNAL_VARIABLE_STEP_PARTITION_NR, partitionNrString);
                if (partitionDistribution.getOriginalPartitionSchemas() != null) {
                    // What is the partition schema name?
                    // 
                    String partitionSchemaName = stepMeta.getStepPartitioningMeta().getPartitionSchema().getName();
                    // 
                    for (PartitionSchema originalPartitionSchema : partitionDistribution.getOriginalPartitionSchemas()) {
                        String slavePartitionSchemaName = TransSplitter.createSlavePartitionSchemaName(originalPartitionSchema.getName());
                        if (slavePartitionSchemaName.equals(partitionSchemaName)) {
                            PartitionSchema schema = (PartitionSchema) originalPartitionSchema.clone();
                            // 
                            if (schema.isDynamicallyDefined()) {
                                schema.expandPartitionsDynamically(this.clusterSize, this);
                            }
                            String partID = schema.getPartitionIDs().get(partitionNr);
                            setVariable(Const.INTERNAL_VARIABLE_STEP_PARTITION_ID, partID);
                            break;
                        }
                    }
                }
            }
        } else {
            // This is a locally partitioned step...
            // 
            int partitionNr = stepcopy;
            String partitionNrString = new DecimalFormat("000").format(partitionNr);
            setVariable(Const.INTERNAL_VARIABLE_STEP_PARTITION_NR, partitionNrString);
            final List<String> partitionIDList = stepMeta.getStepPartitioningMeta().getPartitionSchema().getPartitionIDs();
            if (partitionIDList.size() > 0) {
                String partitionID = partitionIDList.get(partitionNr);
                setVariable(Const.INTERNAL_VARIABLE_STEP_PARTITION_ID, partitionID);
            } else {
                logError(BaseMessages.getString(PKG, "BaseStep.Log.UnableToRetrievePartitionId", stepMeta.getStepPartitioningMeta().getPartitionSchema().getName()));
                return false;
            }
        }
    } else if (!Utils.isEmpty(partitionID)) {
        setVariable(Const.INTERNAL_VARIABLE_STEP_PARTITION_ID, partitionID);
    }
    // Set a unique step number across all slave servers
    // 
    // slaveNr * nrCopies + copyNr
    // 
    uniqueStepNrAcrossSlaves = this.slaveNr * getStepMeta().getCopies() + stepcopy;
    uniqueStepCountAcrossSlaves = this.clusterSize <= 1 ? getStepMeta().getCopies() : this.clusterSize * getStepMeta().getCopies();
    if (uniqueStepCountAcrossSlaves == 0) {
        uniqueStepCountAcrossSlaves = 1;
    }
    setVariable(Const.INTERNAL_VARIABLE_STEP_UNIQUE_NUMBER, Integer.toString(uniqueStepNrAcrossSlaves));
    setVariable(Const.INTERNAL_VARIABLE_STEP_UNIQUE_COUNT, Integer.toString(uniqueStepCountAcrossSlaves));
    setVariable(Const.INTERNAL_VARIABLE_STEP_COPYNR, Integer.toString(stepcopy));
    // BACKLOG-18004
    allowEmptyFieldNamesAndTypes = Boolean.parseBoolean(System.getProperties().getProperty(Const.KETTLE_ALLOW_EMPTY_FIELD_NAMES_AND_TYPES, "false"));
    // 
    try {
        // If this is on the master, separate logic applies.
        // 
        // boolean isMaster = "Y".equalsIgnoreCase(getVariable(Const.INTERNAL_VARIABLE_CLUSTER_MASTER));
        remoteOutputSteps = new ArrayList<RemoteStep>();
        for (int i = 0; i < stepMeta.getRemoteOutputSteps().size(); i++) {
            RemoteStep remoteStep = stepMeta.getRemoteOutputSteps().get(i);
            // 
            if (getCopy() == remoteStep.getSourceStepCopyNr()) {
                // Open a server socket to allow the remote output step to connect.
                // 
                RemoteStep copy = (RemoteStep) remoteStep.clone();
                try {
                    if (log.isDetailed()) {
                        logDetailed(BaseMessages.getString(PKG, "BaseStep.Log.SelectedRemoteOutputStepToServer", copy, copy.getTargetStep(), copy.getTargetStepCopyNr(), copy.getPort()));
                    }
                    copy.openServerSocket(this);
                    if (log.isDetailed()) {
                        logDetailed(BaseMessages.getString(PKG, "BaseStep.Log.OpenedServerSocketConnectionTo", copy));
                    }
                } catch (Exception e) {
                    logError("Unable to open server socket during step initialisation: " + copy.toString(), e);
                    throw e;
                }
                remoteOutputSteps.add(copy);
            }
        }
    } catch (Exception e) {
        for (RemoteStep remoteStep : remoteOutputSteps) {
            if (remoteStep.getServerSocket() != null) {
                try {
                    ServerSocket serverSocket = remoteStep.getServerSocket();
                    getTrans().getSocketRepository().releaseSocket(serverSocket.getLocalPort());
                } catch (IOException e1) {
                    logError("Unable to close server socket after error during step initialisation", e);
                }
            }
        }
        return false;
    }
    // 
    try {
        remoteInputSteps = new ArrayList<RemoteStep>();
        if ((stepMeta.isPartitioned() && getClusterSize() > 1) || stepMeta.getCopies() > 1) {
            // 
            for (int i = 0; i < stepMeta.getRemoteInputSteps().size(); i++) {
                RemoteStep remoteStep = stepMeta.getRemoteInputSteps().get(i);
                if (remoteStep.getTargetStepCopyNr() == stepcopy) {
                    RemoteStep copy = (RemoteStep) remoteStep.clone();
                    remoteInputSteps.add(copy);
                }
            }
        } else {
            for (RemoteStep remoteStep : stepMeta.getRemoteInputSteps()) {
                RemoteStep copy = (RemoteStep) remoteStep.clone();
                remoteInputSteps.add(copy);
            }
        }
    } catch (Exception e) {
        logError("Unable to initialize remote input steps during step initialisation", e);
        return false;
    }
    // Getting ans setting the error handling values
    // first, get the step meta
    StepErrorMeta stepErrorMeta = stepMeta.getStepErrorMeta();
    if (stepErrorMeta != null) {
        // do an environment substitute for stepErrorMeta.getMaxErrors(), stepErrorMeta.getMinPercentRows()
        // and stepErrorMeta.getMaxPercentErrors()
        // Catch NumberFormatException since the user can enter anything in the dialog- the value
        // they enter must be a number or a variable set to a number
        // We will use a boolean to indicate failure so that we can log all errors - not just the first one caught
        boolean envSubFailed = false;
        try {
            maxErrors = (!Utils.isEmpty(stepErrorMeta.getMaxErrors()) ? Long.valueOf(trans.environmentSubstitute(stepErrorMeta.getMaxErrors())) : -1L);
        } catch (NumberFormatException nfe) {
            log.logError(BaseMessages.getString(PKG, "BaseStep.Log.NumberFormatException", BaseMessages.getString(PKG, "BaseStep.Property.MaxErrors.Name"), this.stepname, (stepErrorMeta.getMaxErrors() != null ? stepErrorMeta.getMaxErrors() : "")));
            envSubFailed = true;
        }
        try {
            minRowsForMaxErrorPercent = (!Utils.isEmpty(stepErrorMeta.getMinPercentRows()) ? Long.valueOf(trans.environmentSubstitute(stepErrorMeta.getMinPercentRows())) : -1L);
        } catch (NumberFormatException nfe) {
            log.logError(BaseMessages.getString(PKG, "BaseStep.Log.NumberFormatException", BaseMessages.getString(PKG, "BaseStep.Property.MinRowsForErrorsPercentCalc.Name"), this.stepname, (stepErrorMeta.getMinPercentRows() != null ? stepErrorMeta.getMinPercentRows() : "")));
            envSubFailed = true;
        }
        try {
            maxPercentErrors = (!Utils.isEmpty(stepErrorMeta.getMaxPercentErrors()) ? Integer.valueOf(trans.environmentSubstitute(stepErrorMeta.getMaxPercentErrors())) : -1);
        } catch (NumberFormatException nfe) {
            log.logError(BaseMessages.getString(PKG, "BaseStep.Log.NumberFormatException", BaseMessages.getString(PKG, "BaseStep.Property.MaxPercentErrors.Name"), this.stepname, (stepErrorMeta.getMaxPercentErrors() != null ? stepErrorMeta.getMaxPercentErrors() : "")));
            envSubFailed = true;
        }
        // if we failed and environment subsutitue
        if (envSubFailed) {
            return false;
        }
    }
    return true;
}

Example 18 with PartitionSchema

use of org.pentaho.di.partition.PartitionSchema in project pentaho-kettle by pentaho.

the class SlaveStepCopyPartitionDistributionTest method hashCodeDifferentTest.

@Test
public void hashCodeDifferentTest() {
    SlaveStepCopyPartitionDistribution other = new SlaveStepCopyPartitionDistribution();
    List<PartitionSchema> schemas = new ArrayList<>();
    PartitionSchema schema = new PartitionSchema();
    schema.setName("Test");
    schemas.add(schema);
    other.setOriginalPartitionSchemas(schemas);
    Assert.assertNotEquals(slaveStep.hashCode(), other.hashCode());
}

Example 19 with PartitionSchema

use of org.pentaho.di.partition.PartitionSchema in project pentaho-kettle by pentaho.

the class TransPartitioningTest method prepareStepMetas_1_cl1.

/**
 * This is a case when we have 1 step to 1 clustered step distribution.
 *
 * @throws KettlePluginException
 */
private void prepareStepMetas_1_cl1() throws KettlePluginException {
    StepMeta dummy1 = new StepMeta(ONE, null);
    StepMeta dummy2 = new StepMeta(TWO, null);
    PartitionSchema schema = new PartitionSchema("p1", Arrays.asList(new String[] { PID1, PID2 }));
    StepPartitioningMeta partMeta = new StepPartitioningMeta("Mirror to all partitions", schema);
    dummy2.setStepPartitioningMeta(partMeta);
    chain.add(dummy1);
    chain.add(dummy2);
    for (StepMeta item : chain) {
        item.setStepMetaInterface(new DummyTransMeta());
    }
}

Example 20 with PartitionSchema

use of org.pentaho.di.partition.PartitionSchema in project pentaho-kettle by pentaho.

the class TransPartitioningTest method prepareStepMetas_x2_cl1.

/**
 * This is a case when first step running 2 copies and next is partitioned one.
 *
 * @throws KettlePluginException
 */
private void prepareStepMetas_x2_cl1() throws KettlePluginException {
    StepMeta dummy1 = new StepMeta(ONE, null);
    StepMeta dummy2 = new StepMeta(TWO, null);
    PartitionSchema schema1 = new PartitionSchema("p1", Arrays.asList(new String[] { PID1, PID2 }));
    StepPartitioningMeta partMeta1 = new StepPartitioningMeta("Mirror to all partitions", schema1);
    dummy2.setStepPartitioningMeta(partMeta1);
    dummy1.setCopies(2);
    chain.add(dummy1);
    chain.add(dummy2);
    for (StepMeta item : chain) {
        item.setStepMetaInterface(new DummyTransMeta());
    }
}