Examples with TransSplitter org.pentaho.di.trans.cluster.TransSplitter used on opensource projects

Example 11 with TransSplitter

use of org.pentaho.di.trans.cluster.TransSplitter in project pentaho-kettle by pentaho.

the class SpoonTransformationDelegate method splitTrans.

protected void splitTrans(final TransMeta transMeta, final TransExecutionConfiguration executionConfiguration) throws KettleException {
    try {
        final TransSplitter transSplitter = new TransSplitter(transMeta);
        transSplitter.splitOriginalTransformation();
        TransMeta master = transSplitter.getMaster();
        SlaveServer masterServer = null;
        List<StepMeta> masterSteps = master.getTransHopSteps(false);
        // add transgraph of transmetas if showing is true
        SlaveServer[] slaves = transSplitter.getSlaveTargets();
        if (executionConfiguration.isClusterShowingTransformation()) {
            if (masterSteps.size() > 0) {
                // If there is something that needs to be done on the master...
                masterServer = transSplitter.getMasterServer();
                addTransGraph(master);
            }
            // 
            for (int i = 0; i < slaves.length; i++) {
                TransMeta slaveTrans = transSplitter.getSlaveTransMap().get(slaves[i]);
                addTransGraph(slaveTrans);
            }
        }
        // 
        for (String var : Const.INTERNAL_TRANS_VARIABLES) {
            executionConfiguration.getVariables().put(var, transMeta.getVariable(var));
        }
        for (String var : Const.INTERNAL_JOB_VARIABLES) {
            executionConfiguration.getVariables().put(var, transMeta.getVariable(var));
        }
        // Parameters override the variables.
        // For the time being we're passing the parameters over the wire as variables...
        // 
        TransMeta ot = transSplitter.getOriginalTransformation();
        for (String param : ot.listParameters()) {
            String value = Const.NVL(ot.getParameterValue(param), Const.NVL(ot.getParameterDefault(param), ot.getVariable(param)));
            if (!Utils.isEmpty(value)) {
                executionConfiguration.getVariables().put(param, value);
            }
        }
        try {
            Trans.executeClustered(transSplitter, executionConfiguration);
        } catch (Exception e) {
            // We don't want to suppress original exception here.
            try {
                Trans.cleanupCluster(log, transSplitter);
            } catch (Exception ee) {
                throw new Exception("Error executing transformation and error to clenaup cluster", e);
            }
            // we still have execution error but cleanup ok here...
            throw e;
        }
        if (executionConfiguration.isClusterPosting()) {
            // 
            if (masterServer != null) {
                spoon.addSpoonSlave(masterServer);
                for (int i = 0; i < slaves.length; i++) {
                    spoon.addSpoonSlave(slaves[i]);
                }
            }
        }
        // OK, we should also start monitoring of the cluster in the background.
        // Stop them all if one goes bad.
        // Also clean up afterwards, close sockets, etc.
        // 
        // Launch in a separate thread to prevent GUI blocking...
        // 
        new Thread(new Runnable() {

            public void run() {
                Trans.monitorClusteredTransformation(log, transSplitter, null);
                Result result = Trans.getClusteredTransformationResult(log, transSplitter, null);
                log.logBasic("-----------------------------------------------------");
                log.logBasic("Got result back from clustered transformation:");
                log.logBasic(transMeta.toString() + "-----------------------------------------------------");
                log.logBasic(transMeta.toString() + " Errors : " + result.getNrErrors());
                log.logBasic(transMeta.toString() + " Input : " + result.getNrLinesInput());
                log.logBasic(transMeta.toString() + " Output : " + result.getNrLinesOutput());
                log.logBasic(transMeta.toString() + " Updated : " + result.getNrLinesUpdated());
                log.logBasic(transMeta.toString() + " Read : " + result.getNrLinesRead());
                log.logBasic(transMeta.toString() + " Written : " + result.getNrLinesWritten());
                log.logBasic(transMeta.toString() + " Rejected : " + result.getNrLinesRejected());
                log.logBasic(transMeta.toString() + "-----------------------------------------------------");
            }
        }).start();
    } catch (Exception e) {
        throw new KettleException(e);
    }
}

Example 12 with TransSplitter

use of org.pentaho.di.trans.cluster.TransSplitter in project pentaho-kettle by pentaho.

the class MasterSlaveIT method runMultipleCopiesOnMultipleSlaves2.

/**
 * This test reads a CSV file and sends the data to 3 copies on 3 slave servers.<br>
 */
public void runMultipleCopiesOnMultipleSlaves2() throws Exception {
    TransMeta transMeta = loadTransMetaReplaceSlavesInCluster(clusterGenerator, "test/org/pentaho/di/cluster/test-hops-between-multiple-copies-steps-on-cluster.ktr");
    TransExecutionConfiguration config = createClusteredTransExecutionConfiguration();
    TransSplitter transSplitter = Trans.executeClustered(transMeta, config);
    LogChannel logChannel = createLogChannel("cluster unit test <runMultipleCopiesOnMultipleSlaves2>");
    long nrErrors = Trans.monitorClusteredTransformation(logChannel, transSplitter, null, 1);
    assertEquals(0L, nrErrors);
    String result = loadFileContent(transMeta, "${java.io.tmpdir}/test-multiple-copies-on-multiple-slaves2.txt");
    assertEqualsIgnoreWhitespacesAndCase("90000", result);
}

Example 13 with TransSplitter

use of org.pentaho.di.trans.cluster.TransSplitter in project pentaho-kettle by pentaho.

the class MasterSlaveIT method runParallelFileReadOnMasterWithCopies.

/**
 * This test reads a CSV file in parallel on the master in 3 copies.<br>
 * It then passes the data over to a dummy step on the slaves.<br>
 */
public void runParallelFileReadOnMasterWithCopies() throws Exception {
    TransMeta transMeta = loadTransMetaReplaceSlavesInCluster(clusterGenerator, "test/org/pentaho/di/cluster/test-parallel-file-read-on-master-with-copies.ktr");
    TransExecutionConfiguration config = createClusteredTransExecutionConfiguration();
    TransSplitter transSplitter = Trans.executeClustered(transMeta, config);
    LogChannel logChannel = createLogChannel("cluster unit test <runParallelFileReadOnMasterWithCopies>");
    long nrErrors = Trans.monitorClusteredTransformation(logChannel, transSplitter, null, 1);
    assertEquals(0L, nrErrors);
    String result = loadFileContent(transMeta, "${java.io.tmpdir}/test-parallel-file-read-on-master-result-with-copies.txt");
    assertEqualsIgnoreWhitespacesAndCase("100", result);
}

Example 14 with TransSplitter

use of org.pentaho.di.trans.cluster.TransSplitter in project pentaho-kettle by pentaho.

the class MasterSlaveIT method runParallelFileReadOnMaster.

/**
 * This test reads a CSV file in parallel on the master in 1 copy.<br>
 * It then passes the data over to a dummy step on the slaves.<br>
 * We want to make sure that only 1 copy is considered.<br>
 */
public void runParallelFileReadOnMaster() throws Exception {
    TransMeta transMeta = loadTransMetaReplaceSlavesInCluster(clusterGenerator, "test/org/pentaho/di/cluster/test-parallel-file-read-on-master.ktr");
    TransExecutionConfiguration config = createClusteredTransExecutionConfiguration();
    TransSplitter transSplitter = Trans.executeClustered(transMeta, config);
    LogChannel logChannel = createLogChannel("cluster unit test <testParallelFileReadOnMaster>");
    long nrErrors = Trans.monitorClusteredTransformation(logChannel, transSplitter, null, 1);
    assertEquals(0L, nrErrors);
    String result = loadFileContent(transMeta, "${java.io.tmpdir}/test-parallel-file-read-on-master-result.txt");
    assertEqualsIgnoreWhitespacesAndCase("100", result);
}

Example 15 with TransSplitter

use of org.pentaho.di.trans.cluster.TransSplitter in project pentaho-kettle by pentaho.

the class MasterSlaveIT method runParallelFileReadOnSlaves.

/**
 * This test reads a CSV file in parallel on all 3 slaves, each with 1 copy.<br>
 * It then passes the data over to a dummy step on the slaves.<br>
 */
public void runParallelFileReadOnSlaves() throws Exception {
    TransMeta transMeta = loadTransMetaReplaceSlavesInCluster(clusterGenerator, "test/org/pentaho/di/cluster/test-parallel-file-read-on-slaves.ktr");
    TransExecutionConfiguration config = createClusteredTransExecutionConfiguration();
    TransSplitter transSplitter = Trans.executeClustered(transMeta, config);
    LogChannel logChannel = createLogChannel("cluster unit test <runParallelFileReadOnSlaves>");
    long nrErrors = Trans.monitorClusteredTransformation(logChannel, transSplitter, null, 1);
    assertEquals(0L, nrErrors);
    String result = loadFileContent(transMeta, "${java.io.tmpdir}/test-parallel-file-read-on-slaves.txt");
    assertEqualsIgnoreWhitespacesAndCase("100", result);
}