use of org.pentaho.di.repository.Repository in project pentaho-kettle by pentaho.
the class RunJobServlet method doGet.
/**
* <div id="mindtouch">
* <h1>/kettle/runJob</h1>
* <a name="GET"></a>
* <h2>GET</h2>
* <p>Execute job from enterprise repository. Repository should be configured in Carte xml file.
* Response contains <code>ERROR</code> result if error happened during job execution.</p>
*
* <p><b>Example Request:</b><br />
* <pre function="syntax.xml">
* GET /kettle/runJob?job=home%2Fadmin%2Fdummy_job&level=Debug
* </pre>
*
* </p>
* <h3>Parameters</h3>
* <table class="pentaho-table">
* <tbody>
* <tr>
* <th>name</th>
* <th>description</th>
* <th>type</th>
* </tr>
* <tr>
* <td>job</td>
* <td>Full path to the job in repository.</td>
* <td>query</td>
* </tr>
* <tr>
* <td>level</td>
* <td>Logging level to be used for job execution (i.e. Debug).</td>
* <td>query</td>
* </tr>
* </tbody>
* </table>
*
* <h3>Response Body</h3>
*
* <table class="pentaho-table">
* <tbody>
* <tr>
* <td align="right">element:</td>
* <td>(custom)</td>
* </tr>
* <tr>
* <td align="right">media types:</td>
* <td>text/xml</td>
* </tr>
* </tbody>
* </table>
* <p>Response contains result of the operation. It is either <code>OK</code> or <code>ERROR</code>.
* If an error occurred during job execution, response also contains information about the error.</p>
*
* <p><b>Example Response:</b></p>
* <pre function="syntax.xml">
* <webresult>
* <result>OK</result>
* <message>Job started</message>
* <id>05d919b0-74a3-48d6-84d8-afce359d0449</id>
* </webresult>
* </pre>
*
* <h3>Status Codes</h3>
* <table class="pentaho-table">
* <tbody>
* <tr>
* <th>code</th>
* <th>description</th>
* </tr>
* <tr>
* <td>200</td>
* <td>Request was processed.</td>
* </tr>
* <tr>
* <td>400</td>
* <td>Bad Request: Mandatory parameter job missing</td>
* </tr>
* <tr>
* <td>401</td>
* <td>Unauthorized access to the repository</td>
* </tr>
* <tr>
* <td>404</td>
* <td>Not found: Job not found</td>
* </tr>
* <tr>
* <td>500</td>
* <td>Internal server error occurs during request processing.</td>
* </tr>
* </tbody>
*</table>
*</div>
*/
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
return;
}
if (log.isDebug()) {
logDebug(BaseMessages.getString(PKG, "RunJobServlet.Log.RunJobRequested"));
}
// Options taken from PAN
//
String[] knownOptions = new String[] { "job", "level" };
String transOption = request.getParameter("job");
String levelOption = request.getParameter("level");
response.setStatus(HttpServletResponse.SC_OK);
PrintWriter out = response.getWriter();
SlaveServerConfig serverConfig = transformationMap.getSlaveServerConfig();
try {
Repository slaveServerRepository = serverConfig.getRepository();
if (slaveServerRepository == null || !slaveServerRepository.isConnected()) {
response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "RunJobServlet.Error.UnableToConnectToRepository", serverConfig.getRepositoryId())));
return;
}
if (transOption == null) {
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "RunJobServlet.Error.MissingMandatoryParameterJob")));
return;
}
final JobMeta jobMeta = loadJob(slaveServerRepository, transOption);
// Set the servlet parameters as variables in the transformation
//
String[] parameters = jobMeta.listParameters();
Enumeration<?> parameterNames = request.getParameterNames();
while (parameterNames.hasMoreElements()) {
String parameter = (String) parameterNames.nextElement();
String[] values = request.getParameterValues(parameter);
//
if (Const.indexOfString(parameter, knownOptions) < 0) {
//
if (Const.indexOfString(parameter, parameters) < 0) {
jobMeta.setVariable(parameter, values[0]);
} else {
jobMeta.setParameterValue(parameter, values[0]);
}
}
}
JobExecutionConfiguration jobExecutionConfiguration = new JobExecutionConfiguration();
if (levelOption != null && !isValidLogLevel(levelOption)) {
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "RunJobServlet.Error.InvalidLogLevel")));
return;
}
LogLevel logLevel = LogLevel.getLogLevelForCode(levelOption);
jobExecutionConfiguration.setLogLevel(logLevel);
// Create new repository connection for this job
//
final Repository repository = jobExecutionConfiguration.connectRepository(serverConfig.getRepositoryId(), serverConfig.getRepositoryUsername(), serverConfig.getRepositoryPassword());
JobConfiguration jobConfiguration = new JobConfiguration(jobMeta, jobExecutionConfiguration);
String carteObjectId = UUID.randomUUID().toString();
SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.CARTE, null);
servletLoggingObject.setContainerObjectId(carteObjectId);
servletLoggingObject.setLogLevel(logLevel);
// Create the transformation and store in the list...
//
final Job job = new Job(repository, jobMeta, servletLoggingObject);
// Setting variables
//
job.initializeVariablesFrom(null);
job.getJobMeta().setInternalKettleVariables(job);
job.injectVariables(jobConfiguration.getJobExecutionConfiguration().getVariables());
// Also copy the parameters over...
//
job.copyParametersFrom(jobMeta);
job.clearParameters();
/*
* String[] parameterNames = job.listParameters(); for (int idx = 0; idx < parameterNames.length; idx++) { // Grab
* the parameter value set in the job entry // String thisValue =
* jobExecutionConfiguration.getParams().get(parameterNames[idx]); if (!Utils.isEmpty(thisValue)) { // Set the
* value as specified by the user in the job entry // jobMeta.setParameterValue(parameterNames[idx], thisValue); }
* }
*/
jobMeta.activateParameters();
job.setSocketRepository(getSocketRepository());
JobMap jobMap = getJobMap();
jobMap.addJob(job.getJobname(), carteObjectId, job, jobConfiguration);
// Disconnect from the job's repository when the job finishes.
//
job.addJobListener(new JobAdapter() {
public void jobFinished(Job job) {
repository.disconnect();
}
});
String message = "Job '" + job.getJobname() + "' was added to the list with id " + carteObjectId;
logBasic(message);
try {
runJob(job);
WebResult webResult = new WebResult(WebResult.STRING_OK, "Job started", carteObjectId);
out.println(webResult.getXML());
out.flush();
} catch (Exception executionException) {
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
String logging = KettleLogStore.getAppender().getBuffer(job.getLogChannelId(), false).toString();
out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "RunJobServlet.Error.ErrorExecutingJob", serverConfig.getRepositoryId(), logging)));
}
} catch (IdNotFoundException idEx) {
response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "RunJobServlet.Error.UnableToRunJob", serverConfig.getRepositoryId())));
} catch (Exception ex) {
if (ex.getMessage().contains(UNAUTHORIZED_ACCESS_TO_REPOSITORY)) {
response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "RunJobServlet.Error.UnableToConnectToRepository", serverConfig.getRepositoryId())));
return;
} else if (ex.getMessage().contains(UNABLE_TO_LOAD_JOB)) {
response.setStatus(HttpServletResponse.SC_NOT_FOUND);
out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "RunJobServlet.Error.UnableToFindJob", serverConfig.getRepositoryId())));
return;
}
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "RunJobServlet.Error.UnexpectedError", Const.CR + Const.getStackTracker(ex))));
}
}
use of org.pentaho.di.repository.Repository in project pentaho-kettle by pentaho.
the class AddJobServlet method doGet.
/**
* /**
*
* <div id="mindtouch">
* <h1>/kettle/addJob</h1>
* <a name="POST"></a>
* <h2>POST</h2>
* <p>Uploads and executes job configuration XML file.
* Uploads xml file containing job and job_execution_configuration (wrapped in job_configuration tag)
* to be executed and executes it. Method relies on the input parameter to determine if xml or html
* reply should be produced. The job_configuration xml is
* transferred within request body.
*
* <code>Job name of the executed job </code> will be returned in the Response object
* or <code>message</code> describing error occurred. To determine if the call successful or not you should
* rely on <code>result</code> parameter in response.</p>
*
* <p><b>Example Request:</b><br />
* <pre function="syntax.xml">
* POST /kettle/addJob/?xml=Y
* </pre>
* <p>Request body should contain xml containing job_configuration (job + job_execution_configuration
* wrapped in job_configuration tag).</p>
* </p>
* <h3>Parameters</h3>
* <table class="pentaho-table">
* <tbody>
* <tr>
* <th>name</th>
* <th>description</th>
* <th>type</th>
* </tr>
* <tr>
* <td>xml</td>
* <td>Boolean flag set to either <code>Y</code> or <code>N</code> describing if xml or html reply
* should be produced.</td>
* <td>boolean, optional</td>
* </tr>
* </tbody>
* </table>
*
* <h3>Response Body</h3>
*
* <table class="pentaho-table">
* <tbody>
* <tr>
* <td align="right">element:</td>
* <td>(custom)</td>
* </tr>
* <tr>
* <td align="right">media types:</td>
* <td>text/xml, text/html</td>
* </tr>
* </tbody>
* </table>
* <p>Response wraps job name that was executed or error stack trace
* if an error occurred. Response has <code>result</code> OK if there were no errors. Otherwise it returns ERROR.</p>
*
* <p><b>Example Response:</b></p>
* <pre function="syntax.xml">
* <?xml version="1.0" encoding="UTF-8"?>
* <webresult>
* <result>OK</result>
* <message>Job 'dummy_job' was added to the list with id 1e90eca8-4d4c-47f7-8e5c-99ec36525e7c</message>
* <id>1e90eca8-4d4c-47f7-8e5c-99ec36525e7c</id>
* </webresult>
* </pre>
*
* <h3>Status Codes</h3>
* <table class="pentaho-table">
* <tbody>
* <tr>
* <th>code</th>
* <th>description</th>
* </tr>
* <tr>
* <td>200</td>
* <td>Request was processed and XML response is returned.</td>
* </tr>
* <tr>
* <td>500</td>
* <td>Internal server error occurs during request processing.</td>
* </tr>
* </tbody>
*</table>
*</div>
*/
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
if (isJettyMode() && !request.getRequestURI().startsWith(CONTEXT_PATH)) {
return;
}
if (log.isDebug()) {
logDebug("Addition of job requested");
}
boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml"));
PrintWriter out = response.getWriter();
// read from the client
BufferedReader in = request.getReader();
if (log.isDetailed()) {
logDetailed("Encoding: " + request.getCharacterEncoding());
}
if (useXML) {
response.setContentType("text/xml");
out.print(XMLHandler.getXMLHeader());
} else {
response.setContentType("text/html");
out.println("<HTML>");
out.println("<HEAD><TITLE>Add job</TITLE></HEAD>");
out.println("<BODY>");
}
response.setStatus(HttpServletResponse.SC_OK);
try {
// First read the complete transformation in memory from the request
int c;
StringBuilder xml = new StringBuilder();
while ((c = in.read()) != -1) {
xml.append((char) c);
}
// Parse the XML, create a job configuration
//
JobConfiguration jobConfiguration = JobConfiguration.fromXML(xml.toString());
JobMeta jobMeta = jobConfiguration.getJobMeta();
JobExecutionConfiguration jobExecutionConfiguration = jobConfiguration.getJobExecutionConfiguration();
jobMeta.setLogLevel(jobExecutionConfiguration.getLogLevel());
jobMeta.injectVariables(jobExecutionConfiguration.getVariables());
// If there was a repository, we know about it at this point in time.
//
final Repository repository = jobConfiguration.getJobExecutionConfiguration().getRepository();
String carteObjectId = UUID.randomUUID().toString();
SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.CARTE, null);
servletLoggingObject.setContainerObjectId(carteObjectId);
servletLoggingObject.setLogLevel(jobExecutionConfiguration.getLogLevel());
// Create the transformation and store in the list...
//
final Job job = new Job(repository, jobMeta, servletLoggingObject);
// Setting variables
//
job.initializeVariablesFrom(null);
job.getJobMeta().setInternalKettleVariables(job);
job.injectVariables(jobConfiguration.getJobExecutionConfiguration().getVariables());
job.setArguments(jobExecutionConfiguration.getArgumentStrings());
// Also copy the parameters over...
//
job.copyParametersFrom(jobMeta);
job.clearParameters();
String[] parameterNames = job.listParameters();
for (int idx = 0; idx < parameterNames.length; idx++) {
// Grab the parameter value set in the job entry
//
String thisValue = jobExecutionConfiguration.getParams().get(parameterNames[idx]);
if (!Utils.isEmpty(thisValue)) {
// Set the value as specified by the user in the job entry
//
jobMeta.setParameterValue(parameterNames[idx], thisValue);
}
}
jobMeta.activateParameters();
// Check if there is a starting point specified.
String startCopyName = jobExecutionConfiguration.getStartCopyName();
if (startCopyName != null && !startCopyName.isEmpty()) {
int startCopyNr = jobExecutionConfiguration.getStartCopyNr();
JobEntryCopy startJobEntryCopy = jobMeta.findJobEntry(startCopyName, startCopyNr, false);
job.setStartJobEntryCopy(startJobEntryCopy);
}
job.setSocketRepository(getSocketRepository());
//
if (jobExecutionConfiguration.isExpandingRemoteJob()) {
job.addDelegationListener(new CarteDelegationHandler(getTransformationMap(), getJobMap()));
}
getJobMap().addJob(job.getJobname(), carteObjectId, job, jobConfiguration);
//
if (repository != null) {
job.addJobListener(new JobAdapter() {
@Override
public void jobFinished(Job job) {
repository.disconnect();
}
});
}
String message = Encode.forHtml("Job '" + job.getJobname() + "' was added to the list with id " + carteObjectId);
if (useXML) {
out.println(new WebResult(WebResult.STRING_OK, message, carteObjectId));
} else {
out.println("<H1>" + message + "</H1>");
out.println("<p><a href=\"" + convertContextPath(GetJobStatusServlet.CONTEXT_PATH) + "?name=" + Encode.forUriComponent(job.getJobname()) + "&id=" + carteObjectId + "\">Go to the job status page</a><p>");
}
} catch (Exception ex) {
if (useXML) {
out.println(new WebResult(WebResult.STRING_ERROR, Const.getStackTracker(ex)));
} else {
out.println("<p>");
out.println("<pre>");
ex.printStackTrace(out);
out.println("</pre>");
}
}
if (!useXML) {
out.println("<p>");
out.println("</BODY>");
out.println("</HTML>");
}
}
use of org.pentaho.di.repository.Repository in project pentaho-kettle by pentaho.
the class ExecuteJobServlet method openRepository.
@VisibleForTesting
Repository openRepository(String repositoryName, String user, String pass) throws KettleException {
if (Utils.isEmpty(repositoryName)) {
return null;
}
RepositoriesMeta repositoriesMeta = new RepositoriesMeta();
repositoriesMeta.readData();
RepositoryMeta repositoryMeta = repositoriesMeta.findRepository(repositoryName);
if (repositoryMeta == null) {
String message = BaseMessages.getString(PKG, "ExecuteJobServlet.Error.UnableToFindRepository", repositoryName);
throw new KettleRepositoryNotFoundException(message);
}
PluginRegistry registry = PluginRegistry.getInstance();
Repository repository = registry.loadClass(RepositoryPluginType.class, repositoryMeta, Repository.class);
repository.init(repositoryMeta);
repository.connect(user, pass);
return repository;
}
use of org.pentaho.di.repository.Repository in project pentaho-kettle by pentaho.
the class ZipFileMetaTest method testReadRep.
@Test
public void testReadRep() throws Exception {
ZipFileMeta zipFileMeta = new ZipFileMeta();
Repository rep = mock(Repository.class);
IMetaStore metastore = mock(IMetaStore.class);
DatabaseMeta dbMeta = mock(DatabaseMeta.class);
StringObjectId oid = new StringObjectId("oid");
when(rep.getStepAttributeString(oid, "sourcefilenamefield")).thenReturn(SOURCE_FILENAME);
when(rep.getStepAttributeString(oid, "targetfilenamefield")).thenReturn(TARGET_FILENAME);
when(rep.getStepAttributeString(oid, "baseFolderField")).thenReturn(BASE_FOLDER);
when(rep.getStepAttributeString(oid, "operation_type")).thenReturn(OPERATION_TYPE);
when(rep.getStepAttributeBoolean(oid, "addresultfilenames")).thenReturn(ADD_RESULT_FILENAME);
when(rep.getStepAttributeBoolean(oid, "overwritezipentry")).thenReturn(OVERWRITE_ZIP_ENTRY);
when(rep.getStepAttributeBoolean(oid, "createparentfolder")).thenReturn(CREATE_PARENT_FOLDER);
when(rep.getStepAttributeBoolean(oid, "keepsourcefolder")).thenReturn(KEEP_SOURCE_FOLDER);
when(rep.getStepAttributeString(oid, "movetofolderfield")).thenReturn(MOVE_TO_FOLDER_FIELD);
zipFileMeta.readRep(rep, metastore, oid, Collections.singletonList(dbMeta));
assertEquals(SOURCE_FILENAME, zipFileMeta.getDynamicSourceFileNameField());
assertEquals(TARGET_FILENAME, zipFileMeta.getDynamicTargetFileNameField());
assertEquals(BASE_FOLDER, zipFileMeta.getBaseFolderField());
assertEquals(ZipFileMeta.getOperationTypeByDesc(OPERATION_TYPE), zipFileMeta.getOperationType());
assertEquals(MOVE_TO_FOLDER_FIELD, zipFileMeta.getMoveToFolderField());
assertTrue(zipFileMeta.isaddTargetFileNametoResult());
assertTrue(zipFileMeta.isOverwriteZipEntry());
assertTrue(zipFileMeta.isKeepSouceFolder());
assertTrue(zipFileMeta.isCreateParentFolder());
Mockito.reset(rep, metastore);
StringObjectId transid = new StringObjectId("transid");
zipFileMeta.saveRep(rep, metastore, transid, oid);
verify(rep).saveStepAttribute(transid, oid, "sourcefilenamefield", SOURCE_FILENAME);
verify(rep).saveStepAttribute(transid, oid, "targetfilenamefield", TARGET_FILENAME);
verify(rep).saveStepAttribute(transid, oid, "baseFolderField", BASE_FOLDER);
verify(rep).saveStepAttribute(transid, oid, "operation_type", OPERATION_TYPE);
verify(rep).saveStepAttribute(transid, oid, "addresultfilenames", ADD_RESULT_FILENAME);
verify(rep).saveStepAttribute(transid, oid, "overwritezipentry", OVERWRITE_ZIP_ENTRY);
verify(rep).saveStepAttribute(transid, oid, "createparentfolder", CREATE_PARENT_FOLDER);
verify(rep).saveStepAttribute(transid, oid, "keepsourcefolder", KEEP_SOURCE_FOLDER);
verify(rep).saveStepAttribute(transid, oid, "movetofolderfield", MOVE_TO_FOLDER_FIELD);
Mockito.verifyNoMoreInteractions(rep, metastore);
}
use of org.pentaho.di.repository.Repository in project pentaho-kettle by pentaho.
the class ExecuteTransServletTest method initMocksForTransExecution.
private Trans initMocksForTransExecution(HttpServletRequest mockHttpServletRequest, HttpServletResponse mockHttpServletResponse) throws Exception {
RepositoriesMeta repositoriesMeta = mock(RepositoriesMeta.class);
RepositoryMeta repositoryMeta = mock(RepositoryMeta.class);
Repository repository = mock(Repository.class);
PluginRegistry pluginRegistry = mock(PluginRegistry.class);
RepositoryDirectoryInterface repositoryDirectoryInterface = mock(RepositoryDirectoryInterface.class);
ObjectId objectId = mock(ObjectId.class);
TransMeta transMeta = mock(TransMeta.class);
Trans trans = mock(Trans.class);
KettleLogStore.init();
StringWriter out = new StringWriter();
PrintWriter printWriter = new PrintWriter(out);
when(mockHttpServletRequest.getParameter("rep")).thenReturn("Repo");
when(mockHttpServletRequest.getParameter("trans")).thenReturn("Trans");
when(mockHttpServletRequest.getParameter("user")).thenReturn("user");
when(mockHttpServletRequest.getParameter("pass")).thenReturn("pass");
when(mockHttpServletResponse.getWriter()).thenReturn(printWriter);
whenNew(RepositoriesMeta.class).withNoArguments().thenReturn(repositoriesMeta);
when(repositoriesMeta.findRepository("Repo")).thenReturn(repositoryMeta);
mockStatic(PluginRegistry.class);
when(PluginRegistry.getInstance()).thenReturn(pluginRegistry);
when(pluginRegistry.loadClass(RepositoryPluginType.class, repositoryMeta, Repository.class)).thenReturn(repository);
when(repository.isConnected()).thenReturn(true);
when(repository.loadRepositoryDirectoryTree()).thenReturn(repositoryDirectoryInterface);
when(repositoryDirectoryInterface.findDirectory("/")).thenReturn(repositoryDirectoryInterface);
when(repository.getTransformationID("Trans", repositoryDirectoryInterface)).thenReturn(objectId);
when(repository.loadTransformation(objectId, null)).thenReturn(transMeta);
when(mockHttpServletRequest.getParameterNames()).thenReturn(Collections.enumeration(new ArrayList<>()));
whenNew(Trans.class).withAnyArguments().thenReturn(trans);
setInternalState(executeTransServlet, "socketRepository", mock(SocketRepository.class));
setInternalState(executeTransServlet, "transformationMap", mock(TransformationMap.class));
return trans;
}
Aggregations