Examples with EnvironmentService com.hortonworks.streamline.streams.cluster.service.EnvironmentService used on opensource projects

Example 1 with EnvironmentService

use of com.hortonworks.streamline.streams.cluster.service.EnvironmentService in project streamline by hortonworks.

the class StormTopologyActionsImpl method init.

@Override
public void init(Map<String, Object> conf) {
    this.conf = conf;
    if (conf != null) {
        if (conf.containsKey(StormTopologyLayoutConstants.STORM_ARTIFACTS_LOCATION_KEY)) {
            stormArtifactsLocation = (String) conf.get(StormTopologyLayoutConstants.STORM_ARTIFACTS_LOCATION_KEY);
        }
        if (conf.containsKey(StormTopologyLayoutConstants.STORM_HOME_DIR)) {
            String stormHomeDir = (String) conf.get(StormTopologyLayoutConstants.STORM_HOME_DIR);
            if (!stormHomeDir.endsWith(File.separator)) {
                stormHomeDir += File.separator;
            }
            stormCliPath = stormHomeDir + "bin" + File.separator + "storm";
        }
        this.stormJarLocation = (String) conf.get(StormTopologyLayoutConstants.STORM_JAR_LOCATION_KEY);
        catalogRootUrl = (String) conf.get(StormTopologyLayoutConstants.YAML_KEY_CATALOG_ROOT_URL);
        Map<String, String> env = System.getenv();
        String javaHomeStr = env.get("JAVA_HOME");
        if (StringUtils.isNotEmpty(javaHomeStr)) {
            if (!javaHomeStr.endsWith(File.separator)) {
                javaHomeStr += File.separator;
            }
            javaJarCommand = javaHomeStr + "bin" + File.separator + "jar";
        } else {
            javaJarCommand = "jar";
        }
        String stormApiRootUrl = (String) conf.get(TopologyLayoutConstants.STORM_API_ROOT_URL_KEY);
        Subject subject = (Subject) conf.get(TopologyLayoutConstants.SUBJECT_OBJECT);
        Client restClient = ClientBuilder.newClient(new ClientConfig());
        this.client = new StormRestAPIClient(restClient, stormApiRootUrl, subject);
        nimbusSeeds = (String) conf.get(NIMBUS_SEEDS);
        nimbusPort = Integer.valueOf((String) conf.get(NIMBUS_PORT));
        if (conf.containsKey(TopologyLayoutConstants.NIMBUS_THRIFT_MAX_BUFFER_SIZE)) {
            nimbusThriftMaxBufferSize = (Long) conf.get(TopologyLayoutConstants.NIMBUS_THRIFT_MAX_BUFFER_SIZE);
        } else {
            nimbusThriftMaxBufferSize = DEFAULT_NIMBUS_THRIFT_MAX_BUFFER_SIZE;
        }
        setupSecuredStormCluster(conf);
        EnvironmentService environmentService = (EnvironmentService) conf.get(TopologyLayoutConstants.ENVIRONMENT_SERVICE_OBJECT);
        Number namespaceId = (Number) conf.get(TopologyLayoutConstants.NAMESPACE_ID);
        this.serviceConfigurationReader = new AutoCredsServiceConfigurationReader(environmentService, namespaceId.longValue());
    }
    File f = new File(stormArtifactsLocation);
    if (!f.exists() && !f.mkdirs()) {
        throw new RuntimeException("Could not create directory " + f.getAbsolutePath());
    }
}

Example 2 with EnvironmentService

use of com.hortonworks.streamline.streams.cluster.service.EnvironmentService in project streamline by hortonworks.

the class StreamsModule method getResources.

@Override
public List<Object> getResources() {
    List<Object> result = new ArrayList<>();
    String catalogRootUrl = (String) config.get(Constants.CONFIG_CATALOG_ROOT_URL);
    // Authorized subject
    final Subject subject = (Subject) config.get(Constants.CONFIG_SUBJECT);
    MLModelRegistryClient modelRegistryClient = new MLModelRegistryClient(catalogRootUrl, subject);
    final StreamCatalogService streamcatalogService = new StreamCatalogService(storageManager, fileStorage, modelRegistryClient);
    final EnvironmentService environmentService = new EnvironmentService(storageManager);
    TagClient tagClient = new TagClient(catalogRootUrl);
    final CatalogService catalogService = new CatalogService(storageManager, fileStorage, tagClient);
    final TopologyActionsService topologyActionsService = new TopologyActionsService(streamcatalogService, environmentService, fileStorage, modelRegistryClient, config, subject, transactionManager);
    final TopologyMetricsService topologyMetricsService = new TopologyMetricsService(environmentService, subject);
    final TopologyLogSearchService topologyLogSearchService = new TopologyLogSearchService(environmentService, subject);
    environmentService.addNamespaceAwareContainer(topologyActionsService);
    environmentService.addNamespaceAwareContainer(topologyMetricsService);
    environmentService.addNamespaceAwareContainer(topologyLogSearchService);
    // authorizer
    final StreamlineAuthorizer authorizer = (StreamlineAuthorizer) config.get(Constants.CONFIG_AUTHORIZER);
    if (authorizer == null) {
        throw new IllegalStateException("Authorizer not set");
    }
    final SecurityCatalogService securityCatalogService = (SecurityCatalogService) config.get(Constants.CONFIG_SECURITY_CATALOG_SERVICE);
    result.addAll(getAuthorizerResources(authorizer, securityCatalogService));
    result.add(new MetricsResource(authorizer, streamcatalogService, topologyMetricsService));
    result.addAll(getClusterRelatedResources(authorizer, environmentService));
    result.add(new FileCatalogResource(authorizer, catalogService));
    result.addAll(getTopologyRelatedResources(authorizer, streamcatalogService, environmentService, topologyActionsService, topologyMetricsService, topologyLogSearchService, securityCatalogService, subject));
    result.add(new UDFCatalogResource(authorizer, streamcatalogService, fileStorage));
    result.addAll(getNotificationsRelatedResources(authorizer, streamcatalogService));
    result.add(new SchemaResource(createSchemaRegistryClient()));
    result.addAll(getServiceMetadataResources(authorizer, environmentService, subject));
    result.add(new NamespaceCatalogResource(authorizer, streamcatalogService, topologyActionsService, environmentService));
    result.add(new SearchCatalogResource(authorizer, streamcatalogService, environmentService, topologyActionsService, topologyMetricsService));
    watchFiles(streamcatalogService);
    setupPlaceholderEntities(streamcatalogService, environmentService);
    return result;
}

Example 3 with EnvironmentService

use of com.hortonworks.streamline.streams.cluster.service.EnvironmentService in project streamline by hortonworks.

the class CatalogResourceUtil method enrichTopology.

static TopologyDashboardResponse enrichTopology(Topology topology, String asUser, Integer latencyTopN, EnvironmentService environmentService, TopologyActionsService actionsService, TopologyMetricsService metricsService, StreamCatalogService catalogService) {
    LOG.debug("[START] enrichTopology - topology id: {}", topology.getId());
    Stopwatch stopwatch = Stopwatch.createStarted();
    try {
        if (latencyTopN == null) {
            latencyTopN = DEFAULT_N_OF_TOP_N_LATENCY;
        }
        TopologyDashboardResponse detailedResponse;
        String namespaceName = null;
        Namespace namespace = environmentService.getNamespace(topology.getNamespaceId());
        if (namespace != null) {
            namespaceName = namespace.getName();
        }
        try {
            String runtimeTopologyId = actionsService.getRuntimeTopologyId(topology, asUser);
            TopologyMetrics.TopologyMetric topologyMetric = metricsService.getTopologyMetric(topology, asUser);
            List<Pair<String, Double>> latenciesTopN = metricsService.getTopNAndOtherComponentsLatency(topology, asUser, latencyTopN);
            detailedResponse = new TopologyDashboardResponse(topology, TopologyRunningStatus.RUNNING, namespaceName);
            detailedResponse.setRuntime(new TopologyRuntimeResponse(runtimeTopologyId, topologyMetric, latenciesTopN));
        } catch (TopologyNotAliveException e) {
            LOG.debug("Topology {} is not alive", topology.getId());
            detailedResponse = new TopologyDashboardResponse(topology, TopologyRunningStatus.NOT_RUNNING, namespaceName);
            catalogService.getTopologyState(topology.getId()).ifPresent(state -> {
                if (TopologyStateFactory.getInstance().getTopologyState(state.getName()) == TopologyStates.TOPOLOGY_STATE_DEPLOYED) {
                    try {
                        LOG.info("Force killing streamline topology since its not alive in the cluster");
                        actionsService.killTopology(topology, asUser);
                    } catch (Exception ex) {
                        LOG.error("Error trying to kill topology", ex);
                    }
                }
            });
        } catch (StormNotReachableException | IOException e) {
            LOG.error("Storm is not reachable or fail to operate", e);
            detailedResponse = new TopologyDashboardResponse(topology, TopologyRunningStatus.UNKNOWN, namespaceName);
        } catch (Exception e) {
            LOG.error("Unhandled exception occurs while operate with Storm", e);
            detailedResponse = new TopologyDashboardResponse(topology, TopologyRunningStatus.UNKNOWN, namespaceName);
        }
        LOG.debug("[END] enrichTopology - topology id: {}, elapsed: {} ms", topology.getId(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
        return detailedResponse;
    } finally {
        stopwatch.stop();
    }
}

Example 4 with EnvironmentService

use of com.hortonworks.streamline.streams.cluster.service.EnvironmentService in project streamline by hortonworks.

the class NamespaceCatalogResource method setServicesToClusterInNamespace.

@POST
@Path("/namespaces/{id}/mapping/bulk")
@Timed
public Response setServicesToClusterInNamespace(@PathParam("id") Long namespaceId, List<NamespaceServiceClusterMap> mappings, @Context SecurityContext securityContext) {
    SecurityUtil.checkRoleOrPermissions(authorizer, securityContext, Roles.ROLE_ENVIRONMENT_SUPER_ADMIN, Namespace.NAMESPACE, namespaceId, WRITE);
    Namespace namespace = environmentService.getNamespace(namespaceId);
    if (namespace == null) {
        throw EntityNotFoundException.byId(namespaceId.toString());
    }
    String streamingEngine = namespace.getStreamingEngine();
    String timeSeriesDB = namespace.getTimeSeriesDB();
    Collection<NamespaceServiceClusterMap> existing = environmentService.listServiceClusterMapping(namespaceId);
    Optional<NamespaceServiceClusterMap> existingStreamingEngine = existing.stream().filter(m -> m.getServiceName().equals(streamingEngine)).findFirst();
    // indicates that mapping of streaming engine has been changed or removed
    if (existingStreamingEngine.isPresent() && !mappings.contains(existingStreamingEngine.get())) {
        assertNoTopologyReferringNamespaceIsRunning(namespaceId, WSUtils.getUserFromSecurityContext(securityContext));
    }
    // we're OK to just check with new mappings since we will remove existing mappings
    assertServiceIsUnique(mappings, streamingEngine);
    assertServiceIsUnique(mappings, timeSeriesDB);
    // remove any existing mapping for (namespace, service name) pairs
    Collection<NamespaceServiceClusterMap> existingMappings = environmentService.listServiceClusterMapping(namespaceId);
    if (existingMappings != null) {
        existingMappings.forEach(m -> environmentService.removeServiceClusterMapping(m.getNamespaceId(), m.getServiceName(), m.getClusterId()));
    }
    List<NamespaceServiceClusterMap> newMappings = mappings.stream().map(environmentService::addOrUpdateServiceClusterMapping).collect(toList());
    return WSUtils.respondEntities(newMappings, CREATED);
}

Example 5 with EnvironmentService

use of com.hortonworks.streamline.streams.cluster.service.EnvironmentService in project streamline by hortonworks.

the class ServiceCatalogResource method registerService.

@POST
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Path("/clusters/{clusterId}/services/register/{serviceName}")
@Timed
public Response registerService(@PathParam("clusterId") Long clusterId, @PathParam("serviceName") String serviceName, FormDataMultiPart form) {
    ServiceBundle serviceBundle = environmentService.getServiceBundleByName(serviceName);
    if (serviceBundle == null) {
        throw BadRequestException.message("Not supported service: " + serviceName);
    }
    ManualServiceRegistrar registrar;
    try {
        Class<?> clazz = Class.forName(serviceBundle.getRegisterClass());
        registrar = (ManualServiceRegistrar) clazz.newInstance();
    } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) {
        throw new RuntimeException(e);
    }
    Cluster cluster = environmentService.getCluster(clusterId);
    if (cluster == null) {
        throw EntityNotFoundException.byId("Cluster " + clusterId);
    }
    Service service = environmentService.getServiceByName(clusterId, serviceName);
    if (service != null) {
        throw EntityAlreadyExistsException.byName("Service " + serviceName + " is already exist in Cluster " + clusterId);
    }
    registrar.init(environmentService);
    ComponentUISpecification specification = serviceBundle.getServiceUISpecification();
    List<String> fileFieldNames = specification.getFields().stream().filter(uiField -> uiField.getType().equals(ComponentUISpecification.UIFieldType.FILE)).map(uiField -> uiField.getFieldName()).collect(toList());
    Map<String, List<FormDataBodyPart>> fields = form.getFields();
    List<FormDataBodyPart> cfgFormList = fields.getOrDefault("config", Collections.emptyList());
    Config config;
    if (!cfgFormList.isEmpty()) {
        String jsonConfig = cfgFormList.get(0).getEntityAs(String.class);
        try {
            config = objectMapper.readValue(jsonConfig, Config.class);
        } catch (IOException e) {
            throw BadRequestException.message("config is missing");
        }
    } else {
        config = new Config();
    }
    List<ManualServiceRegistrar.ConfigFileInfo> configFileInfos = fields.entrySet().stream().filter(entry -> fileFieldNames.contains(entry.getKey())).flatMap(entry -> {
        String key = entry.getKey();
        List<FormDataBodyPart> values = entry.getValue();
        return values.stream().map(val -> new ManualServiceRegistrar.ConfigFileInfo(key, val.getEntityAs(InputStream.class)));
    }).collect(toList());
    try {
        Service registeredService = registrar.register(cluster, config, configFileInfos);
        return WSUtils.respondEntity(buildManualServiceRegisterResult(registeredService), CREATED);
    } catch (IllegalArgumentException e) {
        throw BadRequestException.message(e.getMessage());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}