Examples with CustomProcessorInfo com.hortonworks.streamline.streams.catalog.processor.CustomProcessorInfo used on opensource projects

Example 6 with CustomProcessorInfo

use of com.hortonworks.streamline.streams.catalog.processor.CustomProcessorInfo in project streamline by hortonworks.

the class StreamCatalogService method handleCustomProcessorJar.

private void handleCustomProcessorJar(InputStream jarFile, CustomProcessorInfo customProcessorInfo, boolean verify) throws NoSuchAlgorithmException, IOException {
    MessageDigest md = MessageDigest.getInstance("SHA-256");
    java.io.File tmpFile;
    try (DigestInputStream dis = new DigestInputStream(jarFile, md)) {
        tmpFile = FileUtil.writeInputStreamToTempFile(dis, ".jar");
    }
    customProcessorInfo.setDigest(Hex.encodeHexString(md.digest()));
    LOG.debug("Digest: {}", customProcessorInfo.getDigest());
    if (verify && !verifyCustomProcessorImplFromJar(tmpFile, customProcessorInfo)) {
        String message = "Custom Processor jar file is missing customProcessorImpl class " + customProcessorInfo.getCustomProcessorImpl();
        LOG.debug(message);
        throw new RuntimeException(message);
    }
    Collection<CustomProcessorInfo> customProcessorInfos = this.listCustomProcessorsFromBundleWithFilter(Collections.singletonList(new QueryParam(CustomProcessorInfo.DIGEST, customProcessorInfo.getDigest())));
    if (!customProcessorInfos.isEmpty()) {
        customProcessorInfo.setJarFileName(customProcessorInfos.iterator().next().getJarFileName());
    } else {
        customProcessorInfo.setJarFileName(String.format("custom-processor-%s.jar", UUID.randomUUID().toString()));
        try (InputStream inputStream = new FileInputStream(tmpFile)) {
            uploadFileToStorage(inputStream, customProcessorInfo.getJarFileName());
        }
    }
}

Example 7 with CustomProcessorInfo

use of com.hortonworks.streamline.streams.catalog.processor.CustomProcessorInfo in project streamline by hortonworks.

the class CustomProcessorUploadHandler method getCustomProcessorInfo.

private CustomProcessorInfo getCustomProcessorInfo(File tarFile) {
    CustomProcessorInfo customProcessorInfo = null;
    byte[] jsonInfoFileBytes = getFileAsByteArray(tarFile, this.jsonInfoFile);
    if (jsonInfoFileBytes != null && jsonInfoFileBytes.length > 0) {
        ObjectMapper mapper = new ObjectMapper();
        try {
            customProcessorInfo = mapper.readValue(jsonInfoFileBytes, CustomProcessorInfo.class);
        } catch (IOException e) {
            LOG.warn("Error while deserializing custom processor info json.", e);
        }
    } else {
        LOG.warn(jsonInfoFile + " not present in tar file: " + tarFile);
    }
    return customProcessorInfo;
}

Example 8 with CustomProcessorInfo

use of com.hortonworks.streamline.streams.catalog.processor.CustomProcessorInfo in project streamline by hortonworks.

the class CustomProcessorUploadHandler method created.

@Override
public void created(Path path) {
    File createdFile = path.toFile();
    LOG.info("Created called with " + createdFile);
    boolean succeeded = false;
    try {
        if (createdFile.getName().endsWith(".tar")) {
            LOG.info("Processing file at " + path);
            CustomProcessorInfo customProcessorInfo = this.getCustomProcessorInfo(createdFile);
            if (customProcessorInfo == null) {
                LOG.warn("No information found for CustomProcessorRuntime in " + createdFile);
                return;
            }
            InputStream jarFile = this.getJarFile(createdFile);
            if (jarFile == null) {
                LOG.warn("No jar file found for CustomProcessorRuntime in " + createdFile);
                return;
            }
            this.catalogService.addCustomProcessorInfoAsBundle(customProcessorInfo, jarFile);
            succeeded = true;
        } else {
            LOG.info("Failing unsupported file that was received: " + path);
        }
    } catch (IOException e) {
        LOG.warn("Exception occured while processing tar file: " + createdFile, e);
    } catch (ComponentConfigException e) {
        LOG.warn("UI specification for custom processor incorrect for custom processor file: " + createdFile, e);
    } catch (NoSuchAlgorithmException e) {
        LOG.warn("Got NoSuchAlgorithmException while calculating digest for jar: " + createdFile, e);
    } finally {
        try {
            if (succeeded) {
                LOG.info("CustomProcessorRuntime uploaded successfully from  " + createdFile + " Moving file to " + uploadSuccessPath);
                moveFileToSuccessDirectory(createdFile);
            } else {
                LOG.warn("CustomProcessorRuntime failed to upload from " + createdFile + " Moving file to " + uploadFailPath);
                moveFileToFailDirectory(createdFile);
            }
        } catch (IOException e1) {
            LOG.warn("Error moving " + createdFile.getAbsolutePath() + " to " + (succeeded ? uploadSuccessPath : uploadFailPath), e1);
        }
    }
}

Example 9 with CustomProcessorInfo

use of com.hortonworks.streamline.streams.catalog.processor.CustomProcessorInfo in project streamline by hortonworks.

the class StreamCatalogService method listCustomProcessorsFromBundleWithFilter.

public Collection<CustomProcessorInfo> listCustomProcessorsFromBundleWithFilter(List<QueryParam> params) throws IOException {
    Collection<TopologyComponentBundle> customProcessors = this.listCustomProcessorBundlesWithFilter(params);
    Collection<CustomProcessorInfo> result = new ArrayList<>();
    for (TopologyComponentBundle cp : customProcessors) {
        result.add(CustomProcessorInfo.fromTopologyComponentBundle(cp));
    }
    return result;
}

Example 10 with CustomProcessorInfo

use of com.hortonworks.streamline.streams.catalog.processor.CustomProcessorInfo in project streamline by hortonworks.

the class StreamCatalogService method upgradeCustomProcessorsWithDigest.

public Collection<CustomProcessorInfo> upgradeCustomProcessorsWithDigest() throws IOException, ComponentConfigException, NoSuchAlgorithmException {
    Collection<CustomProcessorInfo> customProcessorInfos = this.listCustomProcessorsFromBundleWithFilter(new ArrayList<>());
    if (customProcessorInfos.isEmpty()) {
        // Most likely a fresh install or no CPs registered so far
        LOG.info("No custom processors registered. No need to update with digest");
        return customProcessorInfos;
    } else {
        Collection<CustomProcessorInfo> updatedCustomProcessorInfos = new ArrayList<>();
        for (CustomProcessorInfo customProcessorInfo : customProcessorInfos) {
            if (customProcessorInfo.getDigest() != null) {
                // if a digest is found that means its HDF-3.1.0.0 or higher and hence no upgrade needed
                LOG.info("Digest already present for custom processor {}. No need to upgrade.", customProcessorInfo.getName());
            } else {
                LOG.info("Digest not present for custom processor {}", customProcessorInfo.getName());
                String oldJarToDelete = customProcessorInfo.getJarFileName();
                updateCustomProcessorInfoAsBundle(customProcessorInfo, downloadFileFromStorage(oldJarToDelete), false);
                deleteFileFromStorage(oldJarToDelete);
                LOG.info("Updated custom processor {} with digest {} and jarFileName {}", customProcessorInfo.getName(), customProcessorInfo.getDigest(), customProcessorInfo.getJarFileName());
                updatedCustomProcessorInfos.add(customProcessorInfo);
            }
        }
        return updatedCustomProcessorInfos;
    }
}