Examples with JsonMetadataProvider org.apache.hop.metadata.serializer.json.JsonMetadataProvider used on opensource projects

Example 1 with JsonMetadataProvider

use of org.apache.hop.metadata.serializer.json.JsonMetadataProvider in project hop by apache.

the class HopImportBase method runImport.

@Override
public void runImport(IProgressMonitor monitor) throws HopException {
    this.monitor = monitor;
    // 
    if (metadataProvider == null) {
        this.metadataTargetFolder = outputFolder.getName().getURI() + "/metadata";
        metadataProvider = new MultiMetadataProvider(Encr.getEncoder(), Arrays.asList(new JsonMetadataProvider(Encr.getEncoder(), this.metadataTargetFolder, variables)), variables);
    }
    if (monitor != null) {
        monitor.setTaskName("Finding files to import");
    }
    findFilesToImport();
    if (monitor != null) {
        if (monitor.isCanceled()) {
            return;
        }
        monitor.worked(1);
        monitor.setTaskName("Importing files");
    }
    importFiles();
    if (monitor != null) {
        if (monitor.isCanceled()) {
            return;
        }
        monitor.worked(1);
        monitor.setTaskName("Importing connections");
    }
    importConnections();
    if (monitor != null) {
        if (monitor.isCanceled()) {
            return;
        }
        monitor.worked(1);
        monitor.setTaskName("Importing variables");
    }
    importVariables();
    if (monitor != null) {
        monitor.worked(1);
    }
}

Example 2 with JsonMetadataProvider

use of org.apache.hop.metadata.serializer.json.JsonMetadataProvider in project hop by apache.

the class AsyncStatusServlet method doGet.

public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
        return;
    }
    if (log.isDebug()) {
        logDebug(BaseMessages.getString(PKG, "AsyncStatusServlet.Log.AsyncStatusRequested"));
    }
    IVariables variables = pipelineMap.getHopServerConfig().getVariables();
    MultiMetadataProvider metadataProvider = new MultiMetadataProvider(Encr.getEncoder(), new ArrayList<>(), variables);
    metadataProvider.getProviders().add(HopMetadataUtil.getStandardHopMetadataProvider(variables));
    String metadataFolder = pipelineMap.getHopServerConfig().getMetadataFolder();
    if (StringUtils.isNotEmpty(metadataFolder)) {
        // Get the metadata from the specified metadata folder...
        // 
        metadataProvider.getProviders().add(new JsonMetadataProvider(Encr.getEncoder(), metadataFolder, variables));
    }
    String webServiceName = request.getParameter("service");
    if (StringUtils.isEmpty(webServiceName)) {
        throw new ServletException("Please specify a service parameter pointing to the name of the asynchronous webservice object");
    }
    String serverObjectId = request.getParameter("id");
    if (StringUtils.isEmpty(serverObjectId)) {
        throw new ServletException("Please specify an id parameter pointing to the unique ID of the asynchronous webservice object");
    }
    try {
        // Load the web service metadata
        // 
        IHopMetadataSerializer<AsyncWebService> serializer = metadataProvider.getSerializer(AsyncWebService.class);
        AsyncWebService webService = serializer.load(webServiceName);
        if (webService == null) {
            throw new HopException("Unable to find asynchronous web service '" + webServiceName + "'.  You can set option metadata_folder in the Hop server XML configuration");
        }
        // Get the workflow...
        // 
        IWorkflowEngine<WorkflowMeta> workflow = workflowMap.findWorkflow(webServiceName, serverObjectId);
        // Report back in JSON format
        // 
        AsyncStatus status = new AsyncStatus();
        status.setService(webServiceName);
        status.setId(serverObjectId);
        status.setStartDate(workflow.getExecutionStartDate());
        status.setEndDate(workflow.getExecutionEndDate());
        status.setStatusDescription(workflow.getStatusDescription());
        // 
        for (String statusVariable : webService.getStatusVariablesList(variables)) {
            String statusValue = workflow.getVariable(statusVariable);
            status.getStatusVariables().put(statusVariable, statusValue);
        }
        // 
        for (Object dataValue : workflow.getExtensionDataMap().values()) {
            if (dataValue instanceof HopServerPipelineStatus) {
                status.getPipelineStatuses().add((HopServerPipelineStatus) dataValue);
            }
        }
        // We give back all this information about the executing workflow in JSON format...
        // 
        response.setContentType("application/json");
        response.setCharacterEncoding(Const.XML_ENCODING);
        final OutputStream outputStream = response.getOutputStream();
        ObjectMapper mapper = new ObjectMapper();
        String jsonString = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(status);
        byte[] data = jsonString.getBytes(StandardCharsets.UTF_8);
        response.setContentLength(data.length);
        outputStream.write(data);
        outputStream.flush();
        response.setStatus(HttpServletResponse.SC_OK);
    } catch (Exception e) {
        throw new ServletException("Error getting asynchronous web service status", e);
    }
}

Example 3 with JsonMetadataProvider

use of org.apache.hop.metadata.serializer.json.JsonMetadataProvider in project hop by apache.

the class HopConfig method buildMetadataProvider.

private void buildMetadataProvider() throws HopException {
    List<IHopMetadataProvider> providers = new ArrayList<>();
    String folder = variables.getVariable(Const.HOP_METADATA_FOLDER);
    if (StringUtils.isEmpty(folder)) {
        providers.add(new JsonMetadataProvider());
    } else {
        ITwoWayPasswordEncoder passwordEncoder = Encr.getEncoder();
        if (passwordEncoder == null) {
            passwordEncoder = new HopTwoWayPasswordEncoder();
        }
        providers.add(new JsonMetadataProvider(passwordEncoder, folder, variables));
    }
    metadataProvider = new MultiMetadataProvider(Encr.getEncoder(), providers, variables);
}

Example 4 with JsonMetadataProvider

use of org.apache.hop.metadata.serializer.json.JsonMetadataProvider in project hop by apache.

the class AsyncRunServlet method doGet.

@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) {
    if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
        return;
    }
    if (log.isDebug()) {
        logDebug(BaseMessages.getString(PKG, "AsyncRunServlet.Log.AsyncRunRequested"));
    }
    IVariables variables = pipelineMap.getHopServerConfig().getVariables();
    MultiMetadataProvider metadataProvider = new MultiMetadataProvider(Encr.getEncoder(), new ArrayList<>(), variables);
    metadataProvider.getProviders().add(HopMetadataUtil.getStandardHopMetadataProvider(variables));
    String metadataFolder = pipelineMap.getHopServerConfig().getMetadataFolder();
    if (StringUtils.isNotEmpty(metadataFolder)) {
        // Get the metadata from the specified metadata folder...
        // 
        metadataProvider.getProviders().add(new JsonMetadataProvider(Encr.getEncoder(), metadataFolder, variables));
    }
    String webServiceName = request.getParameter("service");
    if (StringUtils.isEmpty(webServiceName)) {
        log.logError("Please specify a service parameter pointing to the name of the asynchronous webservice object");
    }
    try {
        IHopMetadataSerializer<AsyncWebService> serializer = metadataProvider.getSerializer(AsyncWebService.class);
        AsyncWebService webService = serializer.load(webServiceName);
        if (webService == null) {
            throw new HopException("Unable to find asynchronous web service '" + webServiceName + "'.  You can set option metadata_folder in the Hop server XML configuration");
        }
        if (!webService.isEnabled()) {
            throw new HopException("Asynchronous Web service '" + webServiceName + "' is disabled.");
        }
        String filename = variables.resolve(webService.getFilename());
        // We give back the ID of the executing workflow...
        // 
        response.setContentType("application/json");
        response.setCharacterEncoding(Const.XML_ENCODING);
        String serverObjectId = UUID.randomUUID().toString();
        SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.HOP_SERVER, null);
        servletLoggingObject.setContainerObjectId(serverObjectId);
        // Load and start the workflow
        // Output the ID to the response output stream...
        // 
        WorkflowMeta workflowMeta = new WorkflowMeta(variables, filename, metadataProvider);
        LocalWorkflowEngine workflow = new LocalWorkflowEngine(workflowMeta, servletLoggingObject);
        workflow.setContainerId(serverObjectId);
        workflow.setMetadataProvider(metadataProvider);
        workflow.setLogLevel(LogLevel.BASIC);
        workflow.initializeFrom(variables);
        workflow.setVariable("SERVER_OBJECT_ID", serverObjectId);
        // See if we need to pass a variable with the content in it...
        // 
        // Read the content posted?
        // 
        String contentVariable = variables.resolve(webService.getBodyContentVariable());
        String content = "";
        if (StringUtils.isNotEmpty(contentVariable)) {
            try (InputStream in = request.getInputStream()) {
                try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
                    byte[] buffer = new byte[1024];
                    int length;
                    while ((length = in.read(buffer)) != -1) {
                        outputStream.write(buffer, 0, length);
                    }
                    outputStream.flush();
                    // Now we have the content...
                    // 
                    content = new String(outputStream.toByteArray(), StandardCharsets.UTF_8);
                }
            }
            workflow.setVariable(contentVariable, Const.NVL(content, ""));
        }
        // Set all the other parameters as variables/parameters...
        // 
        String[] pipelineParameters = workflowMeta.listParameters();
        workflow.copyParametersFromDefinitions(workflowMeta);
        for (String requestParameter : request.getParameterMap().keySet()) {
            if ("service".equals(requestParameter)) {
                continue;
            }
            String requestParameterValue = request.getParameter(requestParameter);
            if (Const.indexOfString(requestParameter, pipelineParameters) < 0) {
                workflow.setVariable(requestParameter, Const.NVL(requestParameterValue, ""));
            } else {
                try {
                    workflow.setParameterValue(requestParameter, Const.NVL(requestParameterValue, ""));
                } catch (UnknownParamException e) {
                    log.logError("Error running asynchronous web service", e);
                }
            }
        }
        workflow.activateParameters(workflow);
        // Add the workflow to the status map, so we can retrieve statuses later on
        // 
        WorkflowExecutionConfiguration workflowExecutionConfiguration = new WorkflowExecutionConfiguration();
        WorkflowConfiguration workflowConfiguration = new WorkflowConfiguration(workflowMeta, workflowExecutionConfiguration, new SerializableMetadataProvider(metadataProvider));
        // We use the service name to store the workflow under!
        // That way we don't have to look up the name of the workflow when retrieving the status.
        // 
        getWorkflowMap().addWorkflow(webServiceName, serverObjectId, workflow, workflowConfiguration);
        // Allocate the workflow in the background...
        // 
        new Thread(workflow::startExecution).start();
        try (OutputStream outputStream = response.getOutputStream()) {
            // Report the ID in a JSON block
            // 
            JSONObject json = new JSONObject();
            json.put("name", workflowMeta.getName());
            json.put("id", serverObjectId);
            String jsonString = json.toJSONString();
            outputStream.write(jsonString.getBytes(StandardCharsets.UTF_8));
            outputStream.write("\n".getBytes(StandardCharsets.UTF_8));
            outputStream.flush();
        } catch (IOException e) {
            log.logError("Error running asynchronous web service", e);
        }
        response.setStatus(HttpServletResponse.SC_OK);
    } catch (IOException | HopException e) {
        log.logError("Error running asynchronous web service", e);
    }
}

Example 5 with JsonMetadataProvider

use of org.apache.hop.metadata.serializer.json.JsonMetadataProvider in project hop by apache.

the class HopSearch method buildMetadataProvider.

private void buildMetadataProvider() throws HopException {
    List<IHopMetadataProvider> providers = new ArrayList<>();
    String folder = variables.getVariable(Const.HOP_METADATA_FOLDER);
    if (StringUtils.isEmpty(folder)) {
        providers.add(new JsonMetadataProvider());
    } else {
        ITwoWayPasswordEncoder passwordEncoder = Encr.getEncoder();
        if (passwordEncoder == null) {
            passwordEncoder = new HopTwoWayPasswordEncoder();
        }
        providers.add(new JsonMetadataProvider(passwordEncoder, folder, variables));
    }
    metadataProvider = new MultiMetadataProvider(Encr.getEncoder(), providers, variables);
}