Example 1 with MultiMetadataProvider
use of org.apache.hop.metadata.serializer.multi.MultiMetadataProvider in project hop by apache.
the class HopImportBase method runImport.
@Override
public void runImport(IProgressMonitor monitor) throws HopException {
this.monitor = monitor;
//
if (metadataProvider == null) {
this.metadataTargetFolder = outputFolder.getName().getURI() + "/metadata";
metadataProvider = new MultiMetadataProvider(Encr.getEncoder(), Arrays.asList(new JsonMetadataProvider(Encr.getEncoder(), this.metadataTargetFolder, variables)), variables);
}
if (monitor != null) {
monitor.setTaskName("Finding files to import");
}
findFilesToImport();
if (monitor != null) {
if (monitor.isCanceled()) {
return;
}
monitor.worked(1);
monitor.setTaskName("Importing files");
}
importFiles();
if (monitor != null) {
if (monitor.isCanceled()) {
return;
}
monitor.worked(1);
monitor.setTaskName("Importing connections");
}
importConnections();
if (monitor != null) {
if (monitor.isCanceled()) {
return;
}
monitor.worked(1);
monitor.setTaskName("Importing variables");
}
importVariables();
if (monitor != null) {
monitor.worked(1);
}
}
Also used :
JsonMetadataProvider(org.apache.hop.metadata.serializer.json.JsonMetadataProvider)
MultiMetadataProvider(org.apache.hop.metadata.serializer.multi.MultiMetadataProvider)
Example 2 with MultiMetadataProvider
use of org.apache.hop.metadata.serializer.multi.MultiMetadataProvider in project hop by apache.
the class AddPipelineServlet method doGet.
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
if (isJettyMode() && !request.getRequestURI().startsWith(CONTEXT_PATH)) {
return;
}
if (log.isDebug()) {
logDebug("Addition of pipeline requested");
}
boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml"));
PrintWriter out = response.getWriter();
BufferedReader in = request.getReader();
if (log.isDetailed()) {
logDetailed("Encoding: " + request.getCharacterEncoding());
}
if (useXML) {
response.setContentType("text/xml");
out.print(XmlHandler.getXmlHeader());
} else {
response.setContentType("text/html");
out.println("<HTML>");
out.println("<HEAD><TITLE>Add pipeline</TITLE></HEAD>");
out.println("<BODY>");
}
response.setStatus(HttpServletResponse.SC_OK);
String realLogFilename = null;
PipelineExecutionConfiguration pipelineExecutionConfiguration = null;
try {
// First read the complete pipeline in memory from the request
//
StringBuilder xml = new StringBuilder(request.getContentLength());
int c;
while ((c = in.read()) != -1) {
xml.append((char) c);
}
// Parse the XML, create a pipeline configuration
//
PipelineConfiguration pipelineConfiguration = PipelineConfiguration.fromXml(xml.toString());
PipelineMeta pipelineMeta = pipelineConfiguration.getPipelineMeta();
pipelineExecutionConfiguration = pipelineConfiguration.getPipelineExecutionConfiguration();
if (log.isDetailed()) {
logDetailed("Logging level set to " + log.getLogLevel().getDescription());
}
String serverObjectId = UUID.randomUUID().toString();
SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.HOP_SERVER, null);
servletLoggingObject.setContainerObjectId(serverObjectId);
servletLoggingObject.setLogLevel(pipelineExecutionConfiguration.getLogLevel());
IHopMetadataProvider metadataProvider = new MultiMetadataProvider(variables, getServerConfig().getMetadataProvider(), pipelineConfiguration.getMetadataProvider());
String runConfigurationName = pipelineExecutionConfiguration.getRunConfiguration();
final IPipelineEngine<PipelineMeta> pipeline = PipelineEngineFactory.createPipelineEngine(variables, runConfigurationName, metadataProvider, pipelineMeta);
pipeline.setParent(servletLoggingObject);
if (pipelineExecutionConfiguration.isSetLogfile()) {
realLogFilename = pipelineExecutionConfiguration.getLogFileName();
final LogChannelFileWriter logChannelFileWriter;
try {
FileUtil.createParentFolder(AddPipelineServlet.class, realLogFilename, pipelineExecutionConfiguration.isCreateParentFolder(), pipeline.getLogChannel());
logChannelFileWriter = new LogChannelFileWriter(servletLoggingObject.getLogChannelId(), HopVfs.getFileObject(realLogFilename), pipelineExecutionConfiguration.isSetAppendLogfile());
logChannelFileWriter.startLogging();
pipeline.addExecutionFinishedListener(pipelineEngine -> {
if (logChannelFileWriter != null) {
logChannelFileWriter.stopLogging();
}
});
} catch (HopException e) {
logError(Const.getStackTracker(e));
}
}
getPipelineMap().addPipeline(pipelineMeta.getName(), serverObjectId, pipeline, pipelineConfiguration);
pipeline.setContainerId(serverObjectId);
String message = "Pipeline '" + pipeline.getPipelineMeta().getName() + "' was added to HopServer with id " + serverObjectId;
if (useXML) {
// Return the log channel id as well
//
out.println(new WebResult(WebResult.STRING_OK, message, serverObjectId));
} else {
out.println("<H1>" + message + "</H1>");
out.println("<p><a href=\"" + convertContextPath(GetPipelineStatusServlet.CONTEXT_PATH) + "?name=" + pipeline.getPipelineMeta().getName() + "&id=" + serverObjectId + "\">Go to the pipeline status page</a><p>");
}
} catch (Exception ex) {
if (useXML) {
out.println(new WebResult(WebResult.STRING_ERROR, Const.getStackTracker(ex)));
} else {
out.println("<p>");
out.println("<pre>");
ex.printStackTrace(out);
out.println("</pre>");
}
}
if (!useXML) {
out.println("<p>");
out.println("</BODY>");
out.println("</HTML>");
}
}
Also used :
LogChannelFileWriter(org.apache.hop.core.logging.LogChannelFileWriter)
HopException(org.apache.hop.core.exception.HopException)
PipelineConfiguration(org.apache.hop.pipeline.PipelineConfiguration)
SimpleLoggingObject(org.apache.hop.core.logging.SimpleLoggingObject)
ServletException(javax.servlet.ServletException)
HopException(org.apache.hop.core.exception.HopException)
IOException(java.io.IOException)
PipelineMeta(org.apache.hop.pipeline.PipelineMeta)
MultiMetadataProvider(org.apache.hop.metadata.serializer.multi.MultiMetadataProvider)
BufferedReader(java.io.BufferedReader)
IHopMetadataProvider(org.apache.hop.metadata.api.IHopMetadataProvider)
PipelineExecutionConfiguration(org.apache.hop.pipeline.PipelineExecutionConfiguration)
PrintWriter(java.io.PrintWriter)
Example 3 with MultiMetadataProvider
use of org.apache.hop.metadata.serializer.multi.MultiMetadataProvider in project hop by apache.
the class AddWorkflowServlet method doGet.
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
if (isJettyMode() && !request.getRequestURI().startsWith(CONTEXT_PATH)) {
return;
}
if (log.isDebug()) {
logDebug("Addition of workflow requested");
}
boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml"));
PrintWriter out = response.getWriter();
// read from the client
BufferedReader in = request.getReader();
if (log.isDetailed()) {
logDetailed("Encoding: " + request.getCharacterEncoding());
}
if (useXML) {
response.setContentType("text/xml");
out.print(XmlHandler.getXmlHeader());
} else {
response.setContentType("text/html");
out.println("<HTML>");
out.println("<HEAD><TITLE>Add workflow</TITLE></HEAD>");
out.println("<BODY>");
}
response.setStatus(HttpServletResponse.SC_OK);
try {
// First read the complete pipeline in memory from the request
int c;
StringBuilder xml = new StringBuilder();
while ((c = in.read()) != -1) {
xml.append((char) c);
}
// Parse the XML, create a workflow configuration
//
WorkflowConfiguration workflowConfiguration = WorkflowConfiguration.fromXml(xml.toString(), variables);
IHopMetadataProvider metadataProvider = new MultiMetadataProvider(variables, getServerConfig().getMetadataProvider(), workflowConfiguration.getMetadataProvider());
WorkflowMeta workflowMeta = workflowConfiguration.getWorkflowMeta();
WorkflowExecutionConfiguration workflowExecutionConfiguration = workflowConfiguration.getWorkflowExecutionConfiguration();
String serverObjectId = UUID.randomUUID().toString();
SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.HOP_SERVER, null);
servletLoggingObject.setContainerObjectId(serverObjectId);
servletLoggingObject.setLogLevel(workflowExecutionConfiguration.getLogLevel());
// Create the workflow and store in the list...
//
String runConfigurationName = workflowExecutionConfiguration.getRunConfiguration();
final IWorkflowEngine<WorkflowMeta> workflow = WorkflowEngineFactory.createWorkflowEngine(variables, runConfigurationName, metadataProvider, workflowMeta, servletLoggingObject);
// Setting variables
//
workflow.initializeFrom(null);
workflow.getWorkflowMeta().setInternalHopVariables(workflow);
workflow.setVariables(workflowConfiguration.getWorkflowExecutionConfiguration().getVariablesMap());
// Also copy the parameters over...
//
workflow.copyParametersFromDefinitions(workflowMeta);
workflow.clearParameterValues();
String[] parameterNames = workflow.listParameters();
for (int idx = 0; idx < parameterNames.length; idx++) {
// Grab the parameter value set in the action
//
String thisValue = workflowExecutionConfiguration.getParametersMap().get(parameterNames[idx]);
if (!Utils.isEmpty(thisValue)) {
// Set the value as specified by the user in the action
//
workflow.setParameterValue(parameterNames[idx], thisValue);
}
}
workflow.activateParameters(workflow);
// Check if there is a starting point specified.
String startActionName = workflowExecutionConfiguration.getStartActionName();
if (startActionName != null && !startActionName.isEmpty()) {
ActionMeta startActionMeta = workflowMeta.findAction(startActionName);
workflow.setStartActionMeta(startActionMeta);
}
getWorkflowMap().addWorkflow(workflow.getWorkflowName(), serverObjectId, workflow, workflowConfiguration);
String message = "Workflow '" + workflow.getWorkflowName() + "' was added to the list with id " + serverObjectId;
if (useXML) {
out.println(new WebResult(WebResult.STRING_OK, message, serverObjectId));
} else {
out.println("<H1>" + message + "</H1>");
out.println("<p><a href=\"" + convertContextPath(GetWorkflowStatusServlet.CONTEXT_PATH) + "?name=" + workflow.getWorkflowName() + "&id=" + serverObjectId + "\">Go to the workflow status page</a><p>");
}
} catch (Exception ex) {
if (useXML) {
out.println(new WebResult(WebResult.STRING_ERROR, Const.getStackTracker(ex)));
} else {
out.println("<p>");
out.println("<pre>");
ex.printStackTrace(out);
out.println("</pre>");
}
}
if (!useXML) {
out.println("<p>");
out.println("</BODY>");
out.println("</HTML>");
}
}
Also used :
SimpleLoggingObject(org.apache.hop.core.logging.SimpleLoggingObject)
WorkflowExecutionConfiguration(org.apache.hop.workflow.WorkflowExecutionConfiguration)
ServletException(javax.servlet.ServletException)
IOException(java.io.IOException)
MultiMetadataProvider(org.apache.hop.metadata.serializer.multi.MultiMetadataProvider)
WorkflowMeta(org.apache.hop.workflow.WorkflowMeta)
WorkflowConfiguration(org.apache.hop.workflow.WorkflowConfiguration)
ActionMeta(org.apache.hop.workflow.action.ActionMeta)
BufferedReader(java.io.BufferedReader)
IHopMetadataProvider(org.apache.hop.metadata.api.IHopMetadataProvider)
PrintWriter(java.io.PrintWriter)
Example 4 with MultiMetadataProvider
use of org.apache.hop.metadata.serializer.multi.MultiMetadataProvider in project hop by apache.
the class AsyncStatusServlet method doGet.
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
return;
}
if (log.isDebug()) {
logDebug(BaseMessages.getString(PKG, "AsyncStatusServlet.Log.AsyncStatusRequested"));
}
IVariables variables = pipelineMap.getHopServerConfig().getVariables();
MultiMetadataProvider metadataProvider = new MultiMetadataProvider(Encr.getEncoder(), new ArrayList<>(), variables);
metadataProvider.getProviders().add(HopMetadataUtil.getStandardHopMetadataProvider(variables));
String metadataFolder = pipelineMap.getHopServerConfig().getMetadataFolder();
if (StringUtils.isNotEmpty(metadataFolder)) {
// Get the metadata from the specified metadata folder...
//
metadataProvider.getProviders().add(new JsonMetadataProvider(Encr.getEncoder(), metadataFolder, variables));
}
String webServiceName = request.getParameter("service");
if (StringUtils.isEmpty(webServiceName)) {
throw new ServletException("Please specify a service parameter pointing to the name of the asynchronous webservice object");
}
String serverObjectId = request.getParameter("id");
if (StringUtils.isEmpty(serverObjectId)) {
throw new ServletException("Please specify an id parameter pointing to the unique ID of the asynchronous webservice object");
}
try {
// Load the web service metadata
//
IHopMetadataSerializer<AsyncWebService> serializer = metadataProvider.getSerializer(AsyncWebService.class);
AsyncWebService webService = serializer.load(webServiceName);
if (webService == null) {
throw new HopException("Unable to find asynchronous web service '" + webServiceName + "'. You can set option metadata_folder in the Hop server XML configuration");
}
// Get the workflow...
//
IWorkflowEngine<WorkflowMeta> workflow = workflowMap.findWorkflow(webServiceName, serverObjectId);
// Report back in JSON format
//
AsyncStatus status = new AsyncStatus();
status.setService(webServiceName);
status.setId(serverObjectId);
status.setStartDate(workflow.getExecutionStartDate());
status.setEndDate(workflow.getExecutionEndDate());
status.setStatusDescription(workflow.getStatusDescription());
//
for (String statusVariable : webService.getStatusVariablesList(variables)) {
String statusValue = workflow.getVariable(statusVariable);
status.getStatusVariables().put(statusVariable, statusValue);
}
//
for (Object dataValue : workflow.getExtensionDataMap().values()) {
if (dataValue instanceof HopServerPipelineStatus) {
status.getPipelineStatuses().add((HopServerPipelineStatus) dataValue);
}
}
// We give back all this information about the executing workflow in JSON format...
//
response.setContentType("application/json");
response.setCharacterEncoding(Const.XML_ENCODING);
final OutputStream outputStream = response.getOutputStream();
ObjectMapper mapper = new ObjectMapper();
String jsonString = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(status);
byte[] data = jsonString.getBytes(StandardCharsets.UTF_8);
response.setContentLength(data.length);
outputStream.write(data);
outputStream.flush();
response.setStatus(HttpServletResponse.SC_OK);
} catch (Exception e) {
throw new ServletException("Error getting asynchronous web service status", e);
}
}
Also used :
HopException(org.apache.hop.core.exception.HopException)
OutputStream(java.io.OutputStream)
JsonMetadataProvider(org.apache.hop.metadata.serializer.json.JsonMetadataProvider)
ServletException(javax.servlet.ServletException)
HopException(org.apache.hop.core.exception.HopException)
IOException(java.io.IOException)
MultiMetadataProvider(org.apache.hop.metadata.serializer.multi.MultiMetadataProvider)
WorkflowMeta(org.apache.hop.workflow.WorkflowMeta)
ServletException(javax.servlet.ServletException)
IVariables(org.apache.hop.core.variables.IVariables)
ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper)
Example 5 with MultiMetadataProvider
use of org.apache.hop.metadata.serializer.multi.MultiMetadataProvider in project hop by apache.
the class HopConfig method buildMetadataProvider.
private void buildMetadataProvider() throws HopException {
List<IHopMetadataProvider> providers = new ArrayList<>();
String folder = variables.getVariable(Const.HOP_METADATA_FOLDER);
if (StringUtils.isEmpty(folder)) {
providers.add(new JsonMetadataProvider());
} else {
ITwoWayPasswordEncoder passwordEncoder = Encr.getEncoder();
if (passwordEncoder == null) {
passwordEncoder = new HopTwoWayPasswordEncoder();
}
providers.add(new JsonMetadataProvider(passwordEncoder, folder, variables));
}
metadataProvider = new MultiMetadataProvider(Encr.getEncoder(), providers, variables);
}
Also used :
HopTwoWayPasswordEncoder(org.apache.hop.core.encryption.HopTwoWayPasswordEncoder)
ITwoWayPasswordEncoder(org.apache.hop.core.encryption.ITwoWayPasswordEncoder)
ArrayList(java.util.ArrayList)
IHopMetadataProvider(org.apache.hop.metadata.api.IHopMetadataProvider)
JsonMetadataProvider(org.apache.hop.metadata.serializer.json.JsonMetadataProvider)
MultiMetadataProvider(org.apache.hop.metadata.serializer.multi.MultiMetadataProvider)
Aggregations
MultiMetadataProvider (org.apache.hop.metadata.serializer.multi.MultiMetadataProvider)15
IHopMetadataProvider (org.apache.hop.metadata.api.IHopMetadataProvider)11
SimpleLoggingObject (org.apache.hop.core.logging.SimpleLoggingObject)9
WorkflowMeta (org.apache.hop.workflow.WorkflowMeta)7
HopException (org.apache.hop.core.exception.HopException)6
JsonMetadataProvider (org.apache.hop.metadata.serializer.json.JsonMetadataProvider)6
IOException (java.io.IOException)5
IVariables (org.apache.hop.core.variables.IVariables)5
WorkflowConfiguration (org.apache.hop.workflow.WorkflowConfiguration)5
ServletException (javax.servlet.ServletException)4
WorkflowExecutionConfiguration (org.apache.hop.workflow.WorkflowExecutionConfiguration)4
PrintWriter (java.io.PrintWriter)3
PipelineExecutionConfiguration (org.apache.hop.pipeline.PipelineExecutionConfiguration)3
PipelineMeta (org.apache.hop.pipeline.PipelineMeta)3
BufferedReader (java.io.BufferedReader)2
OutputStream (java.io.OutputStream)2
ArrayList (java.util.ArrayList)2
HopTwoWayPasswordEncoder (org.apache.hop.core.encryption.HopTwoWayPasswordEncoder)2
ITwoWayPasswordEncoder (org.apache.hop.core.encryption.ITwoWayPasswordEncoder)2
LogChannelFileWriter (org.apache.hop.core.logging.LogChannelFileWriter)2
