Examples with TransExecutionConfiguration org.pentaho.di.trans.TransExecutionConfiguration used on opensource projects

Example 16 with TransExecutionConfiguration

use of org.pentaho.di.trans.TransExecutionConfiguration in project pentaho-kettle by pentaho.

the class PartitioningIT method testPartitioningRepartitioningOnCluster.

/**
 * This test reads a CSV file in parallel on the cluster, one copy per slave.<br>
 * It then partitions the data on id in 12 partitions (4 per slave).<br>
 * After that it re-partitions the data in 9 partitions (3 per slave).<br>
 * As such we expect 9 result files on disk.<br>
 * File: "partitioning-repartitioning-on-cluster.ktr"<br>
 */
public void testPartitioningRepartitioningOnCluster() throws Exception {
    init();
    ClusterGenerator clusterGenerator = new ClusterGenerator();
    try {
        clusterGenerator.launchSlaveServers();
        TransMeta transMeta = loadAndModifyTestTransformation(clusterGenerator, "src/it/resources/org/pentaho/di/cluster/partitioning-repartitioning-on-cluster.ktr");
        TransExecutionConfiguration config = createClusteredTransExecutionConfiguration();
        TransSplitter transSplitter = Trans.executeClustered(transMeta, config);
        long nrErrors = Trans.monitorClusteredTransformation(new LogChannel("cluster unit test <testParallelFileReadOnMaster>"), transSplitter, null, 1);
        assertEquals(0L, nrErrors);
        String[] results = new String[] { "8", "9", "9", "9", "9", "8", "8", "8", "8", "8", "8", "8" };
        String[] files = new String[] { "000", "001", "002", "003", "004", "005", "006", "007", "008", "009", "010", "011" };
        for (int i = 0; i < results.length; i++) {
            String filename = "${java.io.tmpdir}/partitioning-repartitioning-on-cluster-" + files[i] + ".txt";
            String result = loadFileContent(transMeta, filename);
            assertEqualsIgnoreWhitespacesAndCase(results[i], result);
            // Remove the output file : we don't want to leave too much clutter around
            // 
            FileObject file = KettleVFS.getFileObject(transMeta.environmentSubstitute(filename));
            file.delete();
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.toString());
    } finally {
        try {
            clusterGenerator.stopSlaveServers();
        } catch (Exception e) {
            e.printStackTrace();
            fail(e.toString());
        }
    }
}

Example 17 with TransExecutionConfiguration

use of org.pentaho.di.trans.TransExecutionConfiguration in project pentaho-kettle by pentaho.

the class PartitioningIT method testPartitioningRepartitioningOnCluster3.

/**
 * Same as testPartitioningRepartitioningOnCluster() but passing the data to a non-partitioned step on the master.
 *
 * File: "partitioning-repartitioning-on-cluster3.ktr"<br>
 */
public void testPartitioningRepartitioningOnCluster3() throws Exception {
    init();
    ClusterGenerator clusterGenerator = new ClusterGenerator();
    try {
        clusterGenerator.launchSlaveServers();
        TransMeta transMeta = loadAndModifyTestTransformation(clusterGenerator, "src/it/resources/org/pentaho/di/cluster/partitioning-repartitioning-on-cluster3.ktr");
        TransExecutionConfiguration config = createClusteredTransExecutionConfiguration();
        TransSplitter transSplitter = Trans.executeClustered(transMeta, config);
        long nrErrors = Trans.monitorClusteredTransformation(new LogChannel("cluster unit test <testParallelFileReadOnMaster>"), transSplitter, null, 1);
        assertEquals(0L, nrErrors);
        String goldenData = "0;16\n1;17\n2;17\n3;17\n4;17\n5;16";
        String filename = "${java.io.tmpdir}/partitioning-repartitioning-on-cluster3.txt";
        String result = loadFileContent(transMeta, filename);
        assertEqualsIgnoreWhitespacesAndCase(goldenData, result);
    // Remove the output file : we don't want to leave too much clutter around
    // 
    // FileObject file = KettleVFS.getFileObject(transMeta.environmentSubstitute(filename));
    // file.delete();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.toString());
    } finally {
        try {
            clusterGenerator.stopSlaveServers();
        } catch (Exception e) {
            e.printStackTrace();
            fail(e.toString());
        }
    }
}

Example 18 with TransExecutionConfiguration

use of org.pentaho.di.trans.TransExecutionConfiguration in project pentaho-kettle by pentaho.

the class PartitioningIT method testPartitioningSwimmingLanesOnCluster.

/**
 * This test reads a CSV file in parallel on the cluster, one copy per slave.<br>
 * It then partitions the data on id in 12 partitions (4 per slave) and keeps the data partitioned until written to
 * file.<br>
 * As such we expect 12 files on disk.<br>
 * File: "partitioning-swimming-lanes-on-cluster.ktr"<br>
 */
public void testPartitioningSwimmingLanesOnCluster() throws Exception {
    init();
    ClusterGenerator clusterGenerator = new ClusterGenerator();
    try {
        clusterGenerator.launchSlaveServers();
        TransMeta transMeta = loadAndModifyTestTransformation(clusterGenerator, "src/it/resources/org/pentaho/di/cluster/partitioning-swimming-lanes-on-cluster.ktr");
        TransExecutionConfiguration config = createClusteredTransExecutionConfiguration();
        TransSplitter transSplitter = Trans.executeClustered(transMeta, config);
        long nrErrors = Trans.monitorClusteredTransformation(new LogChannel("cluster unit test <testParallelFileReadOnMaster>"), transSplitter, null, 1);
        assertEquals(0L, nrErrors);
        String[] results = new String[] { "8", "9", "9", "9", "9", "8", "8", "8", "8", "8", "8", "8" };
        String[] files = new String[] { "000", "001", "002", "003", "004", "005", "006", "007", "008", "009", "010", "011" };
        for (int i = 0; i < results.length; i++) {
            String filename = "${java.io.tmpdir}/partitioning-swimming-lanes-on-cluster-" + files[i] + ".txt";
            String result = loadFileContent(transMeta, filename);
            assertEqualsIgnoreWhitespacesAndCase(results[i], result);
            // Remove the output file : we don't want to leave too much clutter around
            // 
            FileObject file = KettleVFS.getFileObject(transMeta.environmentSubstitute(filename));
            file.delete();
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.toString());
    } finally {
        try {
            clusterGenerator.stopSlaveServers();
        } catch (Exception e) {
            e.printStackTrace();
            fail(e.toString());
        }
    }
}

Example 19 with TransExecutionConfiguration

use of org.pentaho.di.trans.TransExecutionConfiguration in project pentaho-kettle by pentaho.

the class RunTransServlet method doGet.

/**
 * <div id="mindtouch">
 *    <h1>/kettle/runTrans</h1>
 *    <a name="GET"></a>
 *    <h2>GET</h2>
 *    <p>Execute transformation from enterprise repository. Repository should be configured in Carte xml file.
 *  Response contains <code>ERROR</code> result if error happened during transformation execution.</p>
 *
 *    <p><b>Example Request:</b><br />
 *    <pre function="syntax.xml">
 *    GET /kettle/runTrans?trans=home%2Fadmin%2Fdummy-trans&level=Debug
 *    </pre>
 *
 *    </p>
 *    <h3>Parameters</h3>
 *    <table class="pentaho-table">
 *    <tbody>
 *    <tr>
 *      <th>name</th>
 *      <th>description</th>
 *      <th>type</th>
 *    </tr>
 *    <tr>
 *    <td>trans</td>
 *    <td>Full path to the transformation in repository.</td>
 *    <td>query</td>
 *    </tr>
 *    <tr>
 *    <td>level</td>
 *    <td>Logging level to be used for transformation execution (i.e. Debug).</td>
 *    <td>query</td>
 *    </tr>
 *    </tbody>
 *    </table>
 *
 *  <h3>Response Body</h3>
 *
 *  <table class="pentaho-table">
 *    <tbody>
 *      <tr>
 *        <td align="right">element:</td>
 *        <td>(custom)</td>
 *      </tr>
 *      <tr>
 *        <td align="right">media types:</td>
 *        <td>text/xml</td>
 *      </tr>
 *    </tbody>
 *  </table>
 *    <p>Response contains result of the operation. It is either <code>OK</code> or <code>ERROR</code>.
 *     If an error occurred during transformation execution, response also contains information about the error.</p>
 *
 *    <p><b>Example Response:</b></p>
 *    <pre function="syntax.xml">
 *    <webresult>
 *      <result>OK</result>
 *      <message>Transformation started</message>
 *      <id>7c082e8f-b4fe-40bc-b424-e0f881a61874</id>
 *    </webresult>
 *    </pre>
 *
 *    <h3>Status Codes</h3>
 *    <table class="pentaho-table">
 *  <tbody>
 *    <tr>
 *      <th>code</th>
 *      <th>description</th>
 *    </tr>
 *    <tr>
 *      <td>200</td>
 *      <td>Request was processed.</td>
 *    </tr>
 *    <tr>
 *      <td>500</td>
 *      <td>Internal server error occurs during request processing.</td>
 *    </tr>
 *  </tbody>
 *</table>
 *</div>
 */
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
        return;
    }
    if (log.isDebug()) {
        logDebug(BaseMessages.getString(PKG, "RunTransServlet.Log.RunTransRequested"));
    }
    // Options taken from PAN
    // 
    String[] knownOptions = new String[] { "trans", "level" };
    String transOption = request.getParameter("trans");
    String levelOption = request.getParameter("level");
    response.setStatus(HttpServletResponse.SC_OK);
    String encoding = System.getProperty("KETTLE_DEFAULT_SERVLET_ENCODING", null);
    if (encoding != null && !Utils.isEmpty(encoding.trim())) {
        response.setCharacterEncoding(encoding);
        response.setContentType("text/html; charset=" + encoding);
    }
    PrintWriter out = response.getWriter();
    try {
        final Repository repository = transformationMap.getSlaveServerConfig().getRepository();
        final TransMeta transMeta = loadTrans(repository, transOption);
        // Set the servlet parameters as variables in the transformation
        // 
        String[] parameters = transMeta.listParameters();
        Enumeration<?> parameterNames = request.getParameterNames();
        while (parameterNames.hasMoreElements()) {
            String parameter = (String) parameterNames.nextElement();
            String[] values = request.getParameterValues(parameter);
            // 
            if (Const.indexOfString(parameter, knownOptions) < 0) {
                // 
                if (Const.indexOfString(parameter, parameters) < 0) {
                    transMeta.setVariable(parameter, values[0]);
                } else {
                    transMeta.setParameterValue(parameter, values[0]);
                }
            }
        }
        TransExecutionConfiguration transExecutionConfiguration = new TransExecutionConfiguration();
        LogLevel logLevel = LogLevel.getLogLevelForCode(levelOption);
        transExecutionConfiguration.setLogLevel(logLevel);
        TransConfiguration transConfiguration = new TransConfiguration(transMeta, transExecutionConfiguration);
        String carteObjectId = UUID.randomUUID().toString();
        SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.CARTE, null);
        servletLoggingObject.setContainerObjectId(carteObjectId);
        servletLoggingObject.setLogLevel(logLevel);
        // Create the transformation and store in the list...
        // 
        final Trans trans = createTrans(transMeta, servletLoggingObject);
        // Pass information
        // 
        trans.setRepository(repository);
        trans.setServletPrintWriter(out);
        trans.setServletReponse(response);
        trans.setServletRequest(request);
        // Setting variables
        // 
        trans.initializeVariablesFrom(null);
        trans.getTransMeta().setInternalKettleVariables(trans);
        trans.injectVariables(transConfiguration.getTransExecutionConfiguration().getVariables());
        // Also copy the parameters over...
        // 
        trans.copyParametersFrom(transMeta);
        /*
       * String[] parameterNames = job.listParameters(); for (int idx = 0; idx < parameterNames.length; idx++) { // Grab
       * the parameter value set in the job entry // String thisValue =
       * jobExecutionConfiguration.getParams().get(parameterNames[idx]); if (!Utils.isEmpty(thisValue)) { // Set the
       * value as specified by the user in the job entry // jobMeta.setParameterValue(parameterNames[idx], thisValue); }
       * }
       */
        transMeta.activateParameters();
        trans.setSocketRepository(getSocketRepository());
        getTransformationMap().addTransformation(trans.getName(), carteObjectId, trans, transConfiguration);
        // DO NOT disconnect from the shared repository connection when the job finishes.
        // 
        String message = "Transformation '" + trans.getName() + "' was added to the list with id " + carteObjectId;
        logBasic(message);
        try {
            // Execute the transformation...
            // 
            trans.execute(null);
            finishProcessing(trans, out);
        } catch (Exception executionException) {
            String logging = KettleLogStore.getAppender().getBuffer(trans.getLogChannelId(), false).toString();
            throw new KettleException("Error executing Transformation: " + logging, executionException);
        }
    } catch (Exception ex) {
        out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "RunTransServlet.Error.UnexpectedError", Const.CR + Const.getStackTracker(ex))));
    }
}

Example 20 with TransExecutionConfiguration

use of org.pentaho.di.trans.TransExecutionConfiguration in project pentaho-kettle by pentaho.

the class TransformationResource method addTransformation.

@PUT
@Path("/add")
@Produces({ MediaType.APPLICATION_JSON })
public TransformationStatus addTransformation(String xml) {
    TransConfiguration transConfiguration;
    try {
        transConfiguration = TransConfiguration.fromXML(xml.toString());
        TransMeta transMeta = transConfiguration.getTransMeta();
        TransExecutionConfiguration transExecutionConfiguration = transConfiguration.getTransExecutionConfiguration();
        transMeta.setLogLevel(transExecutionConfiguration.getLogLevel());
        LogChannelInterface log = CarteSingleton.getInstance().getLog();
        if (log.isDetailed()) {
            log.logDetailed("Logging level set to " + log.getLogLevel().getDescription());
        }
        transMeta.injectVariables(transExecutionConfiguration.getVariables());
        // Also copy the parameters over...
        // 
        Map<String, String> params = transExecutionConfiguration.getParams();
        for (String param : params.keySet()) {
            String value = params.get(param);
            transMeta.setParameterValue(param, value);
        }
        // If there was a repository, we know about it at this point in time.
        // 
        TransExecutionConfiguration executionConfiguration = transConfiguration.getTransExecutionConfiguration();
        final Repository repository = transConfiguration.getTransExecutionConfiguration().getRepository();
        String carteObjectId = UUID.randomUUID().toString();
        SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(getClass().getName(), LoggingObjectType.CARTE, null);
        servletLoggingObject.setContainerObjectId(carteObjectId);
        servletLoggingObject.setLogLevel(executionConfiguration.getLogLevel());
        // Create the transformation and store in the list...
        // 
        final Trans trans = new Trans(transMeta, servletLoggingObject);
        trans.setRepository(repository);
        trans.setSocketRepository(CarteSingleton.getInstance().getSocketRepository());
        CarteSingleton.getInstance().getTransformationMap().addTransformation(transMeta.getName(), carteObjectId, trans, transConfiguration);
        trans.setContainerObjectId(carteObjectId);
        if (repository != null) {
            // The repository connection is open: make sure we disconnect from the repository once we
            // are done with this transformation.
            // 
            trans.addTransListener(new TransAdapter() {

                public void transFinished(Trans trans) {
                    repository.disconnect();
                }
            });
        }
        return getTransformationStatus(carteObjectId);
    } catch (KettleException e) {
        e.printStackTrace();
    }
    return null;
}