Examples with TopLevelResource org.apache.hop.resource.TopLevelResource used on opensource projects

Example 1 with TopLevelResource

use of org.apache.hop.resource.TopLevelResource in project hop by apache.

the class RemotePipelineEngine method sendToHopServer.

/**
 * Send the pipeline for execution to a HopServer hop server.
 *
 * @param pipelineMeta the pipeline meta-data
 * @param executionConfiguration the pipeline execution configuration
 * @throws HopException if any errors occur during the dispatch to the hop server
 */
private void sendToHopServer(PipelineMeta pipelineMeta, PipelineExecutionConfiguration executionConfiguration, IHopMetadataProvider metadataProvider) throws HopException {
    if (hopServer == null) {
        throw new HopException("No remote server specified");
    }
    if (Utils.isEmpty(pipelineMeta.getName())) {
        throw new HopException("The pipeline needs a name to uniquely identify it by on the remote server.");
    }
    // Inject certain internal variables to make it more intuitive.
    // 
    Map<String, String> vars = new HashMap<>();
    // 
    for (String var : getVariableNames()) {
        if (isVariablePassedToRemoteServer(var)) {
            vars.put(var, getVariable(var));
        }
    }
    executionConfiguration.getVariablesMap().putAll(vars);
    hopServer.getLogChannel().setLogLevel(executionConfiguration.getLogLevel());
    try {
        if (executionConfiguration.isPassingExport()) {
            // First export the workflow...
            // 
            FileObject tempFile = HopVfs.createTempFile("pipelineExport", HopVfs.Suffix.ZIP);
            // The executionConfiguration should not include external references here because all the
            // resources should be
            // retrieved from the exported zip file
            // TODO: Serialize metadata objects to JSON and include it in the zip file
            // 
            PipelineExecutionConfiguration clonedConfiguration = (PipelineExecutionConfiguration) executionConfiguration.clone();
            TopLevelResource topLevelResource = ResourceUtil.serializeResourceExportInterface(tempFile.getName().toString(), pipelineMeta, this, metadataProvider, clonedConfiguration, CONFIGURATION_IN_EXPORT_FILENAME, remotePipelineRunConfiguration.getNamedResourcesSourceFolder(), remotePipelineRunConfiguration.getNamedResourcesTargetFolder(), executionConfiguration.getVariablesMap());
            // Send the zip file over to the hop server...
            // 
            String result = hopServer.sendExport(this, topLevelResource.getArchiveName(), RegisterPackageServlet.TYPE_PIPELINE, topLevelResource.getBaseResourceName());
            WebResult webResult = WebResult.fromXmlString(result);
            if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) {
                String message = cleanupMessage(webResult.getMessage());
                throw new HopException("There was an error passing the exported pipeline to the remote server: " + Const.CR + message);
            }
            containerId = webResult.getId();
        } else {
            // Now send it off to the remote server...
            // Include the JSON of the whole content of the current metadata
            // 
            SerializableMetadataProvider serializableMetadataProvider = new SerializableMetadataProvider(metadataProvider);
            String xml = new PipelineConfiguration(pipelineMeta, executionConfiguration, serializableMetadataProvider).getXml(this);
            String reply = hopServer.sendXml(this, xml, RegisterPipelineServlet.CONTEXT_PATH + "/?xml=Y");
            WebResult webResult = WebResult.fromXmlString(reply);
            if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) {
                String message = cleanupMessage(webResult.getMessage());
                throw new HopException("There was an error posting the pipeline on the remote server: " + Const.CR + message);
            }
            containerId = webResult.getId();
        }
        // Prepare the pipeline
        // 
        String reply = hopServer.execService(this, PrepareExecutionPipelineServlet.CONTEXT_PATH + "/?name=" + URLEncoder.encode(pipelineMeta.getName(), "UTF-8") + "&xml=Y&id=" + containerId);
        WebResult webResult = WebResult.fromXmlString(reply);
        if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) {
            String message = cleanupMessage(webResult.getMessage());
            throw new HopException("There was an error preparing the pipeline for execution on the remote server: " + Const.CR + message);
        }
        // Get the status right after preparation.
        // 
        getPipelineStatus();
    } catch (HopException ke) {
        throw ke;
    } catch (Exception e) {
        throw new HopException(e);
    }
}

Example 2 with TopLevelResource

use of org.apache.hop.resource.TopLevelResource in project hop by apache.

the class RemoteWorkflowEngine method sendToHopServer.

/**
 * Send to hop server.
 *
 * @param workflowMeta the workflow meta
 * @param executionConfiguration the execution configuration
 * @param metadataProvider the metadataProvider
 * @throws HopException the hop exception
 */
public void sendToHopServer(IVariables variables, WorkflowMeta workflowMeta, WorkflowExecutionConfiguration executionConfiguration, IHopMetadataProvider metadataProvider) throws HopException {
    if (hopServer == null) {
        throw new HopException(BaseMessages.getString(PKG, "Workflow.Log.NoHopServerSpecified"));
    }
    if (Utils.isEmpty(workflowMeta.getName())) {
        throw new HopException(BaseMessages.getString(PKG, "Workflow.Log.UniqueWorkflowName"));
    }
    // Align logging levels between execution configuration and remote server
    hopServer.getLogChannel().setLogLevel(executionConfiguration.getLogLevel());
    try {
        // 
        for (String var : getVariableNames()) {
            if (RemotePipelineEngine.isVariablePassedToRemoteServer(var)) {
                executionConfiguration.getVariablesMap().put(var, getVariable(var));
            }
        }
        if (remoteWorkflowRunConfiguration.isExportingResources()) {
            // First export the workflow...
            // 
            FileObject tempFile = HopVfs.createTempFile("workflowExport", ".zip", System.getProperty("java.io.tmpdir"));
            TopLevelResource topLevelResource = ResourceUtil.serializeResourceExportInterface(tempFile.getName().toString(), workflowMeta, this, metadataProvider, executionConfiguration, CONFIGURATION_IN_EXPORT_FILENAME, remoteWorkflowRunConfiguration.getNamedResourcesSourceFolder(), remoteWorkflowRunConfiguration.getNamedResourcesTargetFolder(), executionConfiguration.getVariablesMap());
            // Send the zip file over to the hop server...
            String result = hopServer.sendExport(this, topLevelResource.getArchiveName(), RegisterPackageServlet.TYPE_WORKFLOW, topLevelResource.getBaseResourceName());
            WebResult webResult = WebResult.fromXmlString(result);
            if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) {
                throw new HopException("There was an error passing the exported workflow to the remote server: " + Const.CR + webResult.getMessage());
            }
            containerId = webResult.getId();
        } else {
            String xml = new WorkflowConfiguration(workflowMeta, executionConfiguration, metadataProvider).getXml(variables);
            String reply = hopServer.sendXml(this, xml, RegisterWorkflowServlet.CONTEXT_PATH + "/?xml=Y");
            WebResult webResult = WebResult.fromXmlString(reply);
            if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) {
                throw new HopException("There was an error posting the workflow on the remote server: " + Const.CR + webResult.getMessage());
            }
            containerId = webResult.getId();
        }
        // Start the workflow
        // 
        WebResult webResult = hopServer.startWorkflow(this, workflowMeta.getName(), containerId);
        if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) {
            throw new HopException("There was an error starting the workflow on the remote server: " + Const.CR + webResult.getMessage().replace('\t', '\n'));
        }
    } catch (HopException ke) {
        throw ke;
    } catch (Exception e) {
        throw new HopException(e);
    }
}