Examples with Service com.hortonworks.streamline.streams.cluster.catalog.Service used on opensource projects

Example 46 with Service

use of com.hortonworks.streamline.streams.cluster.catalog.Service in project streamline by hortonworks.

the class ClusterImporter method storeService.

private void storeService(ServiceInformation serviceInformation) {
    Service service = serviceInformation.getService();
    Long serviceId = environmentService.addService(service).getId();
    serviceInformation.getComponents().forEach(componentInformation -> storeComponent(serviceId, componentInformation));
    ServiceConfigurationInformation scInfo = serviceInformation.getServiceConfigurationInfo();
    scInfo.getServiceConfigurations().forEach(sc -> storeServiceConfiguration(serviceId, sc));
}

Example 47 with Service

use of com.hortonworks.streamline.streams.cluster.catalog.Service in project streamline by hortonworks.

the class DruidSinkBundleHintProvider method getHintsOnCluster.

@Override
public Map<String, Object> getHintsOnCluster(Cluster cluster, SecurityContext securityContext, Subject subject) {
    Map<String, Object> hintMap = new HashMap<>();
    try {
        Service druid = environmentService.getServiceByName(cluster.getId(), Constants.Druid.SERVICE_NAME);
        if (druid == null) {
            throw new ServiceNotFoundException(Constants.Druid.SERVICE_NAME);
        }
        ServiceConfiguration commonRuntime = environmentService.getServiceConfigurationByName(druid.getId(), Constants.Druid.CONF_TYPE_COMMON_RUNTIME);
        if (commonRuntime == null) {
            throw new ServiceConfigurationNotFoundException(Constants.Druid.CONF_TYPE_COMMON_RUNTIME);
        }
        Map<String, String> configurationMap = commonRuntime.getConfigurationMap();
        putToHintMapIfAvailable(configurationMap, hintMap, Constants.Druid.PROPERTY_KEY_ZK_SERVICE_HOSTS, FIELD_NAME_ZK_CONNECT);
        putToHintMapIfAvailable(configurationMap, hintMap, Constants.Druid.PROPERTY_KEY_INDEXING_SERVICE_NAME, FIELD_NAME_INDEX_SERVICE);
        putToHintMapIfAvailable(configurationMap, hintMap, Constants.Druid.PROPERTY_KEY_DISCOVERY_CURATOR_PATH, FIELD_NAME_DISCOVERY_PATH);
        // exceptional case for Ambari import
        if (!hintMap.containsKey(FIELD_NAME_INDEX_SERVICE)) {
            ServiceConfiguration druidOverload = environmentService.getServiceConfigurationByName(druid.getId(), Constants.Druid.CONF_TYPE_DRUID_OVERLOAD);
            if (druidOverload != null) {
                putToHintMapIfAvailable(druidOverload.getConfigurationMap(), hintMap, Constants.Druid.PROPERTY_KEY_SERVICE_NAME, FIELD_NAME_INDEX_SERVICE);
            }
        }
    } catch (ServiceNotFoundException e) {
        // we access it from mapping information so shouldn't be here
        throw new IllegalStateException("Service " + Constants.Druid.SERVICE_NAME + " in cluster " + cluster.getName() + " not found but mapping information exists.");
    } catch (ServiceConfigurationNotFoundException e) {
    // there's Druid service configuration but not having enough information
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    return hintMap;
}

Example 48 with Service

use of com.hortonworks.streamline.streams.cluster.catalog.Service in project streamline by hortonworks.

the class TopologyActionsService method setUpClusterArtifacts.

public void setUpClusterArtifacts(Topology topology, TopologyActions topologyActions) throws IOException {
    Namespace namespace = environmentService.getNamespace(topology.getNamespaceId());
    if (namespace == null) {
        throw new RuntimeException("Corresponding namespace not found: " + topology.getNamespaceId());
    }
    ObjectMapper objectMapper = new ObjectMapper();
    Path artifactsDir = topologyActions.getArtifactsLocation(CatalogToLayoutConverter.getTopologyLayout(topology));
    makeEmptyDir(artifactsDir);
    Collection<NamespaceServiceClusterMap> serviceClusterMappings = environmentService.listServiceClusterMapping(namespace.getId());
    for (NamespaceServiceClusterMap serviceClusterMapping : serviceClusterMappings) {
        Service service = environmentService.getServiceByName(serviceClusterMapping.getClusterId(), serviceClusterMapping.getServiceName());
        if (service != null) {
            Collection<ServiceConfiguration> serviceConfigurations = environmentService.listServiceConfigurations(service.getId());
            if (serviceConfigurations != null) {
                for (ServiceConfiguration serviceConfiguration : serviceConfigurations) {
                    writeConfigurationFile(objectMapper, artifactsDir, serviceConfiguration);
                }
            }
        }
    }
}

Example 49 with Service

use of com.hortonworks.streamline.streams.cluster.catalog.Service in project streamline by hortonworks.

the class TopologyActionsContainer method buildStormTopologyActionsConfigMap.

private Map<String, Object> buildStormTopologyActionsConfigMap(Namespace namespace, String streamingEngine, Subject subject) {
    // Assuming that a namespace has one mapping of streaming engine except test environment
    Service streamingEngineService = getFirstOccurenceServiceForNamespace(namespace, streamingEngine);
    if (streamingEngineService == null) {
        if (!namespace.getInternal()) {
            throw new RuntimeException("Streaming Engine " + streamingEngine + " is not associated to the namespace " + namespace.getName() + "(" + namespace.getId() + ")");
        } else {
            // the namespace is purposed for test run
            return buildStormTopologyActionsConfigMapForTestRun(namespace, subject);
        }
    }
    Component uiServer = getComponent(streamingEngineService, COMPONENT_NAME_STORM_UI_SERVER).orElseThrow(() -> new RuntimeException(streamingEngine + " doesn't have " + COMPONENT_NAME_STORM_UI_SERVER + " as component"));
    Collection<ComponentProcess> uiServerProcesses = environmentService.listComponentProcesses(uiServer.getId());
    if (uiServerProcesses.isEmpty()) {
        throw new RuntimeException(streamingEngine + " doesn't have component process " + COMPONENT_NAME_STORM_UI_SERVER);
    }
    ComponentProcess uiServerProcess = uiServerProcesses.iterator().next();
    final String uiHost = uiServerProcess.getHost();
    Integer uiPort = uiServerProcess.getPort();
    assertHostAndPort(uiServer.getName(), uiHost, uiPort);
    Component nimbus = getComponent(streamingEngineService, COMPONENT_NAME_NIMBUS).orElseThrow(() -> new RuntimeException(streamingEngine + " doesn't have " + COMPONENT_NAME_NIMBUS + " as component"));
    Collection<ComponentProcess> nimbusProcesses = environmentService.listComponentProcesses(nimbus.getId());
    if (nimbusProcesses.isEmpty()) {
        throw new RuntimeException(streamingEngine + " doesn't have component process " + COMPONENT_NAME_NIMBUS);
    }
    List<String> nimbusHosts = nimbusProcesses.stream().map(ComponentProcess::getHost).collect(Collectors.toList());
    Integer nimbusPort = nimbusProcesses.stream().map(ComponentProcess::getPort).findAny().get();
    assertHostsAndPort(nimbus.getName(), nimbusHosts, nimbusPort);
    Map<String, Object> conf = new HashMap<>();
    // We need to have some local configurations anyway because topology submission can't be done with REST API.
    String stormJarLocation = streamlineConf.get(STREAMLINE_STORM_JAR);
    if (stormJarLocation == null) {
        String jarFindDir = applyReservedPaths(DEFAULT_STORM_JAR_LOCATION_DIR);
        stormJarLocation = findFirstMatchingJarLocation(jarFindDir);
    } else {
        stormJarLocation = applyReservedPaths(stormJarLocation);
    }
    conf.put(STREAMLINE_STORM_JAR, stormJarLocation);
    conf.put(STORM_HOME_DIR, streamlineConf.get(STORM_HOME_DIR));
    // Since we're loading the class dynamically so we can't rely on any enums or constants from there
    conf.put(NIMBUS_SEEDS, String.join(",", nimbusHosts));
    conf.put(NIMBUS_PORT, String.valueOf(nimbusPort));
    conf.put(TopologyLayoutConstants.STORM_API_ROOT_URL_KEY, buildStormRestApiRootUrl(uiHost, uiPort));
    conf.put(TopologyLayoutConstants.SUBJECT_OBJECT, subject);
    putStormConfigurations(streamingEngineService, conf);
    // Topology during run-time will require few critical configs such as schemaRegistryUrl and catalogRootUrl
    // Hence its important to pass StreamlineConfig to TopologyConfig
    conf.putAll(streamlineConf);
    // TopologyActionImpl needs 'EnvironmentService' and namespace ID to load service configurations
    // for specific cluster associated to the namespace
    conf.put(TopologyLayoutConstants.ENVIRONMENT_SERVICE_OBJECT, environmentService);
    conf.put(TopologyLayoutConstants.NAMESPACE_ID, namespace.getId());
    return conf;
}

Example 50 with Service

use of com.hortonworks.streamline.streams.cluster.catalog.Service in project streamline by hortonworks.

the class TopologyMetricsContainer method buildAMSTimeSeriesQuerierConfigMap.

private Map<String, String> buildAMSTimeSeriesQuerierConfigMap(Namespace namespace, String timeSeriesDB) {
    // Assuming that a namespace has one mapping of time-series DB service
    Service timeSeriesDBService = getFirstOccurenceServiceForNamespace(namespace, timeSeriesDB);
    if (timeSeriesDBService == null) {
        throw new RuntimeException("Time-series DB " + timeSeriesDB + " is not associated to the namespace " + namespace.getName() + "(" + namespace.getId() + ")");
    }
    Component metricsCollector = getComponent(timeSeriesDBService, COMPONENT_NAME_METRICS_COLLECTOR).orElseThrow(() -> new RuntimeException(timeSeriesDB + " doesn't have " + COMPONENT_NAME_METRICS_COLLECTOR + " as component"));
    Collection<ComponentProcess> metricsCollectorProcesses = environmentService.listComponentProcesses(metricsCollector.getId());
    if (metricsCollectorProcesses.isEmpty()) {
        throw new RuntimeException(timeSeriesDB + " doesn't have any process for " + COMPONENT_NAME_METRICS_COLLECTOR + " as component");
    }
    ComponentProcess metricsCollectorProcess = metricsCollectorProcesses.iterator().next();
    String metricsCollectorHost = metricsCollectorProcess.getHost();
    Integer metricsCollectorPort = metricsCollectorProcess.getPort();
    assertHostAndPort(COMPONENT_NAME_METRICS_COLLECTOR, metricsCollectorHost, metricsCollectorPort);
    Map<String, String> confForTimeSeriesQuerier = new HashMap<>();
    confForTimeSeriesQuerier.put(COLLECTOR_API_URL_KEY, buildAMSCollectorRestApiRootUrl(metricsCollectorHost, metricsCollectorPort));
    return confForTimeSeriesQuerier;
}