Examples with Repository org.pentaho.di.repository.Repository used on opensource projects

Example 1 with Repository

use of org.pentaho.di.repository.Repository in project pentaho-kettle by pentaho.

the class KitchenCommandExecutor method execute.

public int execute(String repoName, String noRepo, String username, String trustUser, String password, String dirName, String filename, String jobName, String listJobs, String listDirs, String exportRepo, String initialDir, String listRepos, String listParams, NamedParams params, NamedParams customParams, String[] arguments) throws Throwable {
    getLog().logMinimal(BaseMessages.getString(getPkgClazz(), "Kitchen.Log.Starting"));
    Date start = Calendar.getInstance().getTime();
    logDebug("Kitchen.Log.AllocateNewJob");
    Job job = null;
    // In case we use a repository...
    Repository repository = null;
    try {
        if (getMetaStore() == null) {
            setMetaStore(createDefaultMetastore());
        }
        // Read kettle job specified on command-line?
        if (!Utils.isEmpty(repoName) || !Utils.isEmpty(filename)) {
            logDebug("Kitchen.Log.ParsingCommandLine");
            if (!Utils.isEmpty(repoName) && !YES.equalsIgnoreCase(noRepo)) {
                /**
                 * if set, _trust_user_ needs to be considered. See pur-plugin's:
                 *
                 * @link https://github.com/pentaho/pentaho-kettle/blob/8.0.0.0-R/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepositoryConnector.java#L97-L101
                 * @link https://github.com/pentaho/pentaho-kettle/blob/8.0.0.0-R/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/WebServiceManager.java#L130-L133
                 */
                if (YES.equalsIgnoreCase(trustUser)) {
                    System.setProperty("pentaho.repository.client.attemptTrust", YES);
                }
                // In case we use a repository...
                // some commands are to load a Trans from the repo; others are merely to print some repo-related information
                RepositoryMeta repositoryMeta = loadRepositoryConnection(repoName, "Kitchen.Log.LoadingRep", "Kitchen.Error.NoRepDefinied", "Kitchen.Log.FindingRep");
                repository = establishRepositoryConnection(repositoryMeta, username, password, RepositoryOperation.EXECUTE_JOB);
                job = executeRepositoryBasedCommand(repository, repositoryMeta, dirName, jobName, listJobs, listDirs);
            }
            // Try to load if from file anyway.
            if (!Utils.isEmpty(filename) && job == null) {
                // Try to load the job from file, even if it failed to load from the repository
                job = executeFilesystemBasedCommand(initialDir, filename);
            }
        } else if (YES.equalsIgnoreCase(listRepos)) {
            // list the repositories placed at repositories.xml
            printRepositories(loadRepositoryInfo("Kitchen.Log.ListRep", "Kitchen.Error.NoRepDefinied"));
        }
    } catch (KettleException e) {
        job = null;
        if (repository != null) {
            repository.disconnect();
        }
        System.out.println(BaseMessages.getString(getPkgClazz(), "Kitchen.Error.StopProcess", e.getMessage()));
    }
    if (job == null) {
        if (!YES.equalsIgnoreCase(listJobs) && !YES.equalsIgnoreCase(listDirs) && !YES.equalsIgnoreCase(listRepos)) {
            System.out.println(BaseMessages.getString(getPkgClazz(), "Kitchen.Error.canNotLoadJob"));
        }
        return CommandExecutorCodes.Kitchen.COULD_NOT_LOAD_JOB.getCode();
    }
    if (!Utils.isEmpty(exportRepo)) {
        try {
            // Export the resources linked to the currently loaded file...
            TopLevelResource topLevelResource = ResourceUtil.serializeResourceExportInterface(exportRepo, job.getJobMeta(), job, repository, getMetaStore());
            String launchFile = topLevelResource.getResourceName();
            String message = ResourceUtil.getExplanation(exportRepo, launchFile, job.getJobMeta());
            System.out.println();
            System.out.println(message);
            // Setting the list parameters option will make kitchen exit below in the parameters section
            listParams = YES;
        } catch (Exception e) {
            System.out.println(Const.getStackTracker(e));
            return CommandExecutorCodes.Kitchen.UNEXPECTED_ERROR.getCode();
        }
    }
    Result result = null;
    int returnCode = CommandExecutorCodes.Kitchen.SUCCESS.getCode();
    try {
        // Set the command line arguments on the job ...
        job.setArguments(arguments != null ? arguments : null);
        job.initializeVariablesFrom(null);
        job.setLogLevel(getLog().getLogLevel());
        job.getJobMeta().setInternalKettleVariables(job);
        job.setRepository(repository);
        job.getJobMeta().setRepository(repository);
        job.getJobMeta().setMetaStore(getMetaStore());
        // Map the command line named parameters to the actual named parameters. Skip for
        // the moment any extra command line parameter not known in the job.
        String[] jobParams = job.getJobMeta().listParameters();
        for (String param : jobParams) {
            String value = params.getParameterValue(param);
            if (value != null) {
                job.getJobMeta().setParameterValue(param, value);
            }
        }
        job.copyParametersFrom(job.getJobMeta());
        // Put the parameters over the already defined variable space. Parameters get priority.
        job.activateParameters();
        // Set custom options in the job extension map as Strings
        for (String optionName : customParams.listParameters()) {
            String optionValue = customParams.getParameterValue(optionName);
            if (optionName != null && optionValue != null) {
                job.getExtensionDataMap().put(optionName, optionValue);
            }
        }
        // List the parameters defined in this job, then simply exit...
        if (YES.equalsIgnoreCase(listParams)) {
            printJobParameters(job);
            // same as the other list options
            return CommandExecutorCodes.Kitchen.COULD_NOT_LOAD_JOB.getCode();
        }
        job.start();
        job.waitUntilFinished();
        // Execute the selected job.
        result = job.getResult();
    } finally {
        if (repository != null) {
            repository.disconnect();
        }
        if (YES.equalsIgnoreCase(trustUser)) {
            // we set it, now we sanitize it
            System.clearProperty("pentaho.repository.client.attemptTrust");
        }
    }
    getLog().logMinimal(BaseMessages.getString(getPkgClazz(), "Kitchen.Log.Finished"));
    if (result != null && result.getNrErrors() != 0) {
        getLog().logError(BaseMessages.getString(getPkgClazz(), "Kitchen.Error.FinishedWithErrors"));
        returnCode = CommandExecutorCodes.Kitchen.ERRORS_DURING_PROCESSING.getCode();
    }
    Date stop = Calendar.getInstance().getTime();
    calculateAndPrintElapsedTime(start, stop, "Kitchen.Log.StartStop", "Kitchen.Log.ProcessEndAfter", "Kitchen.Log.ProcessEndAfterLong", "Kitchen.Log.ProcessEndAfterLonger", "Kitchen.Log.ProcessEndAfterLongest");
    return returnCode;
}

Example 2 with Repository

use of org.pentaho.di.repository.Repository in project pdi-platform-plugin by pentaho.

the class PdiAction method connectToRepository.

/**
 * Connects to the PDI repository
 *
 * @param logWriter
 * @return
 * @throws KettleException
 * @throws KettleSecurityException
 * @throws ActionExecutionException
 */
protected Repository connectToRepository(final LogWriter logWriter) throws KettleSecurityException, KettleException, ActionExecutionException {
    if (log.isDebugEnabled()) {
        // $NON-NLS-1$
        log.debug(Messages.getInstance().getString("Kettle.DEBUG_META_REPOSITORY"));
    }
    RepositoriesMeta repositoriesMeta = new RepositoriesMeta();
    if (log.isDebugEnabled()) {
        // $NON-NLS-1$
        log.debug(Messages.getInstance().getString("Kettle.DEBUG_POPULATING_META"));
    }
    boolean singleDiServerInstance = // $NON-NLS-1$ //$NON-NLS-2$
    "true".equals(PentahoSystem.getSystemSetting(SINGLE_DI_SERVER_INSTANCE, "true"));
    try {
        if (singleDiServerInstance) {
            if (log.isDebugEnabled()) {
                // $NON-NLS-1$
                log.debug("singleDiServerInstance=true, loading default repository");
            }
            // only load a default enterprise repository. If this option is set, then you cannot load
            // transformations or jobs from anywhere but the local server.
            String repositoriesXml = // $NON-NLS-1$
            "<?xml version=\"1.0\" encoding=\"UTF-8\"?><repositories>" + // $NON-NLS-1$
            "<repository><id>PentahoEnterpriseRepository</id>" + "<name>" + SINGLE_DI_SERVER_INSTANCE + // $NON-NLS-1$ //$NON-NLS-2$
            "</name>" + "<description>" + SINGLE_DI_SERVER_INSTANCE + // $NON-NLS-1$ //$NON-NLS-2$
            "</description>" + "<repository_location_url>" + PentahoSystem.getApplicationContext().getFullyQualifiedServerURL() + // $NON-NLS-1$ //$NON-NLS-2$
            "</repository_location_url>" + // $NON-NLS-1$
            "<version_comment_mandatory>N</version_comment_mandatory>" + // $NON-NLS-1$
            "</repository>" + // $NON-NLS-1$
            "</repositories>";
            ByteArrayInputStream sbis = new ByteArrayInputStream(repositoriesXml.getBytes("UTF8"));
            repositoriesMeta.readDataFromInputStream(sbis);
        } else {
            // TODO: add support for specified repositories.xml files...
            // Read from the default $HOME/.kettle/repositories.xml file.
            repositoriesMeta.readData();
        }
    } catch (Exception e) {
        throw new ActionExecutionException(Messages.getInstance().getErrorString("Kettle.ERROR_0018_META_REPOSITORY_NOT_POPULATED"), // $NON-NLS-1$
        e);
    }
    if (log.isDebugEnabled()) {
        // $NON-NLS-1$
        log.debug(Messages.getInstance().getString("Kettle.DEBUG_FINDING_REPOSITORY"));
    }
    // Find the specified repository.
    RepositoryMeta repositoryMeta = null;
    try {
        if (singleDiServerInstance) {
            repositoryMeta = repositoriesMeta.findRepository(SINGLE_DI_SERVER_INSTANCE);
        } else {
            repositoryMeta = repositoriesMeta.findRepository(repositoryName);
        }
    } catch (Exception e) {
        throw new ActionExecutionException(Messages.getInstance().getErrorString("Kettle.ERROR_0004_REPOSITORY_NOT_FOUND", repositoryName), // $NON-NLS-1$
        e);
    }
    if (repositoryMeta == null) {
        if (log.isDebugEnabled()) {
            log.debug(pdiUserAppender.getBuffer().toString());
        }
        throw new ActionExecutionException(Messages.getInstance().getErrorString("Kettle.ERROR_0004_REPOSITORY_NOT_FOUND", // $NON-NLS-1$
        repositoryName));
    }
    if (log.isDebugEnabled()) {
        // $NON-NLS-1$
        log.debug(Messages.getInstance().getString("Kettle.DEBUG_GETTING_REPOSITORY"));
    }
    Repository repository = null;
    try {
        repository = PluginRegistry.getInstance().loadClass(RepositoryPluginType.class, repositoryMeta.getId(), Repository.class);
        repository.init(repositoryMeta);
    } catch (Exception e) {
        throw new ActionExecutionException(Messages.getInstance().getErrorString("Kettle.ERROR_0016_COULD_NOT_GET_REPOSITORY_INSTANCE"), // $NON-NLS-1$
        e);
    }
    // OK, now try the username and password
    if (log.isDebugEnabled()) {
        // $NON-NLS-1$
        log.debug(Messages.getInstance().getString("Kettle.DEBUG_CONNECTING"));
    }
    // Two scenarios here: internal to server or external to server. If internal, you are already authenticated. If
    // external, you must provide a username and additionally specify that the IP address of the machine running this
    // code is trusted.
    repository.connect(PentahoSessionHolder.getSession().getName(), "password");
    // OK, the repository is open and ready to use.
    if (log.isDebugEnabled()) {
        // $NON-NLS-1$
        log.debug(Messages.getInstance().getString("Kettle.DEBUG_FINDING_DIRECTORY"));
    }
    return repository;
}

Example 3 with Repository

use of org.pentaho.di.repository.Repository in project pdi-platform-plugin by pentaho.

the class PdiAction method execute.

/**
 * Execute the specified transformation in the chosen repository.
 */
public void execute() throws Exception {
    // Reset the flag
    transPrepExecutionFailure = false;
    IAuthorizationPolicy authorizationPolicy = PentahoSystem.get(IAuthorizationPolicy.class, PentahoSessionHolder.getSession());
    if (!authorizationPolicy.isAllowed(RepositoryExecuteAction.NAME)) {
        throw new IllegalStateException(org.pentaho.platform.plugin.kettle.messages.Messages.getInstance().getErrorString(// $NON-NLS-1$
        "PdiAction.ERROR_0010_NO_PERMISSION_TO_EXECUTE"));
    }
    if (log.isDebugEnabled()) {
        // $NON-NLS-1$
        log.debug(Messages.getInstance().getString("Kettle.DEBUG_START"));
    }
    validate();
    TransMeta transMeta = null;
    JobMeta jobMeta = null;
    // $NON-NLS-1$
    LogWriter logWriter = LogWriter.getInstance("Kettle-pentaho", false);
    // initialize environment variables
    KettleSystemListener.environmentInit(PentahoSessionHolder.getSession());
    pdiUserAppender = KettleLogStore.getAppender();
    Repository repository = connectToRepository(logWriter);
    LoggingBufferAppender loggingBufferAppender = new LoggingBufferAppender(pdiUserAppender);
    logWriter.addAppender(loggingBufferAppender);
    try {
        if (transformation != null) {
            // to populate available databases, etc in "standard" kettle fashion
            try {
                transMeta = createTransMetaJCR(repository);
            } catch (Throwable t) {
            // ignored
            }
            if (transMeta == null) {
                transMeta = createTransMeta(repository, logWriter);
            }
            if (transMeta == null) {
                throw new IllegalStateException(org.pentaho.platform.plugin.kettle.messages.Messages.getInstance().getErrorString(// $NON-NLS-1$
                "PdiAction.ERROR_0004_FAILED_TRANSMETA_CREATION"));
            }
            executeTransformation(transMeta, logWriter);
        } else if (job != null) {
            // to populate available databases, etc in "standard" kettle fashion
            try {
                jobMeta = createJobMetaJCR(repository);
            } catch (Throwable t) {
            // ignored
            }
            if (jobMeta == null) {
                jobMeta = createJobMeta(repository, logWriter);
            }
            if (jobMeta == null) {
                throw new IllegalStateException(org.pentaho.platform.plugin.kettle.messages.Messages.getInstance().getErrorString(// $NON-NLS-1$
                "PdiAction.ERROR_0005_FAILED_JOBMETA_CREATION"));
            }
            executeJob(jobMeta, repository, logWriter);
        }
    } finally {
        logWriter.removeAppender(loggingBufferAppender);
        if (repository != null) {
            if (log.isDebugEnabled()) {
                // $NON-NLS-1$
                log.debug(Messages.getInstance().getString("Kettle.DEBUG_DISCONNECTING"));
            }
            repository.disconnect();
        }
    }
    XMLHandlerCache.getInstance().clear();
}

Example 4 with Repository

use of org.pentaho.di.repository.Repository in project pentaho-kettle by pentaho.

the class BaseJobServlet method createTrans.

protected Trans createTrans(TransConfiguration transConfiguration) throws UnknownParamException {
    TransMeta transMeta = transConfiguration.getTransMeta();
    TransExecutionConfiguration transExecutionConfiguration = transConfiguration.getTransExecutionConfiguration();
    transMeta.setLogLevel(transExecutionConfiguration.getLogLevel());
    transMeta.injectVariables(transExecutionConfiguration.getVariables());
    // Also copy the parameters over...
    copyParameters(transMeta, transExecutionConfiguration.getParams());
    String carteObjectId = UUID.randomUUID().toString();
    SimpleLoggingObject servletLoggingObject = getServletLogging(carteObjectId, transExecutionConfiguration.getLogLevel());
    // Create the transformation and store in the list...
    final Trans trans = new Trans(transMeta, servletLoggingObject);
    trans.setMetaStore(transformationMap.getSlaveServerConfig().getMetaStore());
    if (transExecutionConfiguration.isSetLogfile()) {
        String realLogFilename = transExecutionConfiguration.getLogFileName();
        try {
            FileUtil.createParentFolder(AddTransServlet.class, realLogFilename, transExecutionConfiguration.isCreateParentFolder(), trans.getLogChannel(), trans);
            final LogChannelFileWriter logChannelFileWriter = new LogChannelFileWriter(servletLoggingObject.getLogChannelId(), KettleVFS.getFileObject(realLogFilename), transExecutionConfiguration.isSetAppendLogfile());
            logChannelFileWriter.startLogging();
            trans.addTransListener(new TransAdapter() {

                @Override
                public void transFinished(Trans trans) throws KettleException {
                    if (logChannelFileWriter != null) {
                        logChannelFileWriter.stopLogging();
                    }
                }
            });
        } catch (KettleException e) {
            logError(Const.getStackTracker(e));
        }
    }
    // If there was a repository, we know about it at this point in time.
    final Repository repository = transExecutionConfiguration.getRepository();
    trans.setRepository(repository);
    trans.setSocketRepository(getSocketRepository());
    trans.setContainerObjectId(carteObjectId);
    getTransformationMap().addTransformation(transMeta.getName(), carteObjectId, trans, transConfiguration);
    if (repository != null) {
        // The repository connection is open: make sure we disconnect from the repository once we
        // are done with this transformation.
        trans.addTransListener(new TransAdapter() {

            @Override
            public void transFinished(Trans trans) {
                repository.disconnect();
            }
        });
    }
    final Long passedBatchId = transExecutionConfiguration.getPassedBatchId();
    if (passedBatchId != null) {
        trans.setPassedBatchId(passedBatchId);
    }
    return trans;
}

Example 5 with Repository

use of org.pentaho.di.repository.Repository in project pentaho-kettle by pentaho.

the class RunTransServlet method doGet.

/**
 * <div id="mindtouch">
 *    <h1>/kettle/runTrans</h1>
 *    <a name="GET"></a>
 *    <h2>GET</h2>
 *    <p>Execute transformation from enterprise repository. Repository should be configured in Carte xml file.
 *  Response contains <code>ERROR</code> result if error happened during transformation execution.</p>
 *
 *    <p><b>Example Request:</b><br />
 *    <pre function="syntax.xml">
 *    GET /kettle/runTrans?trans=home%2Fadmin%2Fdummy-trans&level=Debug
 *    </pre>
 *
 *    </p>
 *    <h3>Parameters</h3>
 *    <table class="pentaho-table">
 *    <tbody>
 *    <tr>
 *      <th>name</th>
 *      <th>description</th>
 *      <th>type</th>
 *    </tr>
 *    <tr>
 *    <td>trans</td>
 *    <td>Full path to the transformation in repository.</td>
 *    <td>query</td>
 *    </tr>
 *    <tr>
 *    <td>level</td>
 *    <td>Logging level to be used for transformation execution (i.e. Debug).</td>
 *    <td>query</td>
 *    </tr>
 *    </tbody>
 *    </table>
 *
 *  <h3>Response Body</h3>
 *
 *  <table class="pentaho-table">
 *    <tbody>
 *      <tr>
 *        <td align="right">element:</td>
 *        <td>(custom)</td>
 *      </tr>
 *      <tr>
 *        <td align="right">media types:</td>
 *        <td>text/xml</td>
 *      </tr>
 *    </tbody>
 *  </table>
 *    <p>Response contains result of the operation. It is either <code>OK</code> or <code>ERROR</code>.
 *     If an error occurred during transformation execution, response also contains information about the error.</p>
 *
 *    <p><b>Example Response:</b></p>
 *    <pre function="syntax.xml">
 *    <webresult>
 *      <result>OK</result>
 *      <message>Transformation started</message>
 *      <id>7c082e8f-b4fe-40bc-b424-e0f881a61874</id>
 *    </webresult>
 *    </pre>
 *
 *    <h3>Status Codes</h3>
 *    <table class="pentaho-table">
 *  <tbody>
 *    <tr>
 *      <th>code</th>
 *      <th>description</th>
 *    </tr>
 *    <tr>
 *      <td>200</td>
 *      <td>Request was processed.</td>
 *    </tr>
 *    <tr>
 *      <td>500</td>
 *      <td>Internal server error occurs during request processing.</td>
 *    </tr>
 *  </tbody>
 *</table>
 *</div>
 */
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
        return;
    }
    if (log.isDebug()) {
        logDebug(BaseMessages.getString(PKG, "RunTransServlet.Log.RunTransRequested"));
    }
    // Options taken from PAN
    // 
    String[] knownOptions = new String[] { "trans", "level" };
    String transOption = request.getParameter("trans");
    String levelOption = request.getParameter("level");
    response.setStatus(HttpServletResponse.SC_OK);
    String encoding = System.getProperty("KETTLE_DEFAULT_SERVLET_ENCODING", null);
    if (encoding != null && !Utils.isEmpty(encoding.trim())) {
        response.setCharacterEncoding(encoding);
        response.setContentType("text/html; charset=" + encoding);
    }
    PrintWriter out = response.getWriter();
    try {
        final Repository repository = transformationMap.getSlaveServerConfig().getRepository();
        final TransMeta transMeta = loadTrans(repository, transOption);
        // Set the servlet parameters as variables in the transformation
        // 
        String[] parameters = transMeta.listParameters();
        Enumeration<?> parameterNames = request.getParameterNames();
        while (parameterNames.hasMoreElements()) {
            String parameter = (String) parameterNames.nextElement();
            String[] values = request.getParameterValues(parameter);
            // 
            if (Const.indexOfString(parameter, knownOptions) < 0) {
                // 
                if (Const.indexOfString(parameter, parameters) < 0) {
                    transMeta.setVariable(parameter, values[0]);
                } else {
                    transMeta.setParameterValue(parameter, values[0]);
                }
            }
        }
        TransExecutionConfiguration transExecutionConfiguration = new TransExecutionConfiguration();
        LogLevel logLevel = LogLevel.getLogLevelForCode(levelOption);
        transExecutionConfiguration.setLogLevel(logLevel);
        TransConfiguration transConfiguration = new TransConfiguration(transMeta, transExecutionConfiguration);
        String carteObjectId = UUID.randomUUID().toString();
        SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.CARTE, null);
        servletLoggingObject.setContainerObjectId(carteObjectId);
        servletLoggingObject.setLogLevel(logLevel);
        // Create the transformation and store in the list...
        // 
        final Trans trans = createTrans(transMeta, servletLoggingObject);
        // Pass information
        // 
        trans.setRepository(repository);
        trans.setServletPrintWriter(out);
        trans.setServletReponse(response);
        trans.setServletRequest(request);
        // Setting variables
        // 
        trans.initializeVariablesFrom(null);
        trans.getTransMeta().setInternalKettleVariables(trans);
        trans.injectVariables(transConfiguration.getTransExecutionConfiguration().getVariables());
        // Also copy the parameters over...
        // 
        trans.copyParametersFrom(transMeta);
        /*
       * String[] parameterNames = job.listParameters(); for (int idx = 0; idx < parameterNames.length; idx++) { // Grab
       * the parameter value set in the job entry // String thisValue =
       * jobExecutionConfiguration.getParams().get(parameterNames[idx]); if (!Utils.isEmpty(thisValue)) { // Set the
       * value as specified by the user in the job entry // jobMeta.setParameterValue(parameterNames[idx], thisValue); }
       * }
       */
        transMeta.activateParameters();
        trans.setSocketRepository(getSocketRepository());
        getTransformationMap().addTransformation(trans.getName(), carteObjectId, trans, transConfiguration);
        // DO NOT disconnect from the shared repository connection when the job finishes.
        // 
        String message = "Transformation '" + trans.getName() + "' was added to the list with id " + carteObjectId;
        logBasic(message);
        try {
            // Execute the transformation...
            // 
            trans.execute(null);
            finishProcessing(trans, out);
        } catch (Exception executionException) {
            String logging = KettleLogStore.getAppender().getBuffer(trans.getLogChannelId(), false).toString();
            throw new KettleException("Error executing Transformation: " + logging, executionException);
        }
    } catch (Exception ex) {
        out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "RunTransServlet.Error.UnexpectedError", Const.CR + Const.getStackTracker(ex))));
    }
}