Examples with TopologyComponent com.hortonworks.streamline.streams.catalog.TopologyComponent used on opensource projects

Example 1 with TopologyComponent

use of com.hortonworks.streamline.streams.catalog.TopologyComponent in project streamline by hortonworks.

the class MetricsResource method getCompleteLatency.

@GET
@Path("/topologies/{id}/components/{topologyComponentId}/complete_latency")
@Timed
public Response getCompleteLatency(@PathParam("id") Long id, @PathParam("topologyComponentId") Long topologyComponentId, @QueryParam("from") Long from, @QueryParam("to") Long to, @Context SecurityContext securityContext) throws Exception {
    SecurityUtil.checkRoleOrPermissions(authorizer, securityContext, Roles.ROLE_TOPOLOGY_USER, Topology.NAMESPACE, id, READ);
    assertTimeRange(from, to);
    Topology topology = catalogService.getTopology(id);
    TopologyComponent topologyComponent = catalogService.getTopologyComponent(id, topologyComponentId);
    if (topology != null && topologyComponent != null) {
        String asUser = WSUtils.getUserFromSecurityContext(securityContext);
        Map<Long, Double> metrics = metricsService.getCompleteLatency(topology, topologyComponent, from, to, asUser);
        return WSUtils.respondEntity(metrics, OK);
    } else if (topology == null) {
        throw EntityNotFoundException.byId("Topology: " + id.toString());
    } else {
        // topologyComponent == null
        throw EntityNotFoundException.byId("TopologyComponent: " + id.toString());
    }
}

Example 2 with TopologyComponent

use of com.hortonworks.streamline.streams.catalog.TopologyComponent in project streamline by hortonworks.

the class MetricsResource method getKafkaTopicOffsets.

@GET
@Path("/topologies/{id}/components/{topologyComponentId}/kafka_topic_offsets")
@Timed
public Response getKafkaTopicOffsets(@PathParam("id") Long id, @PathParam("topologyComponentId") Long topologyComponentId, @QueryParam("from") Long from, @QueryParam("to") Long to, @Context SecurityContext securityContext) throws IOException {
    SecurityUtil.checkRoleOrPermissions(authorizer, securityContext, Roles.ROLE_TOPOLOGY_USER, Topology.NAMESPACE, id, READ);
    assertTimeRange(from, to);
    Topology topology = catalogService.getTopology(id);
    TopologyComponent topologyComponent = catalogService.getTopologyComponent(id, topologyComponentId);
    if (topology != null && topologyComponent != null) {
        String asUser = WSUtils.getUserFromSecurityContext(securityContext);
        Map<String, Map<Long, Double>> metrics = metricsService.getKafkaTopicOffsets(topology, topologyComponent, from, to, asUser);
        return WSUtils.respondEntity(metrics, OK);
    } else if (topology == null) {
        throw EntityNotFoundException.byId("Topology: " + id.toString());
    } else {
        // topologyComponent == null
        throw EntityNotFoundException.byId("TopologyComponent: " + id.toString());
    }
}

Example 3 with TopologyComponent

use of com.hortonworks.streamline.streams.catalog.TopologyComponent in project streamline by hortonworks.

the class MetricsResource method getComponentStats.

@GET
@Path("/topologies/{id}/components/all/component_stats")
@Timed
public Response getComponentStats(@PathParam("id") Long id, @QueryParam("from") Long from, @QueryParam("to") Long to, @Context SecurityContext securityContext) throws Exception {
    SecurityUtil.checkRoleOrPermissions(authorizer, securityContext, Roles.ROLE_TOPOLOGY_USER, Topology.NAMESPACE, id, READ);
    assertTimeRange(from, to);
    Topology topology = catalogService.getTopology(id);
    if (topology != null) {
        List<TopologyComponent> topologyComponents = new ArrayList<>();
        List<com.hortonworks.registries.common.QueryParam> queryParams = new ArrayList<>();
        queryParams.add(new com.hortonworks.registries.common.QueryParam("topologyId", String.valueOf(topology.getId())));
        queryParams.add(new com.hortonworks.registries.common.QueryParam("versionId", String.valueOf(topology.getVersionId())));
        topologyComponents.addAll(catalogService.listTopologySources(queryParams));
        topologyComponents.addAll(catalogService.listTopologyProcessors(queryParams));
        topologyComponents.addAll(catalogService.listTopologySinks(queryParams));
        Map<String, TopologyTimeSeriesMetrics.TimeSeriesComponentMetric> topologyMetrics = ParallelStreamUtil.execute(() -> topologyComponents.parallelStream().map(c -> {
            try {
                String asUser = WSUtils.getUserFromSecurityContext(securityContext);
                return Pair.of(c, metricsService.getComponentStats(topology, c, from, to, asUser));
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }).collect(toMap(m -> m.getKey().getId().toString(), m -> m.getValue())), forkJoinPool);
        return WSUtils.respondEntity(topologyMetrics, OK);
    }
    throw EntityNotFoundException.byId(id.toString());
}

Example 4 with TopologyComponent

use of com.hortonworks.streamline.streams.catalog.TopologyComponent in project streamline by hortonworks.

the class StreamCatalogService method setReconfigureTarget.

private void setReconfigureTarget(TopologyEdge edge, TopologyStream stream) {
    TopologyComponent component = getTo(edge);
    component.setReconfigure(true);
    dao.addOrUpdate(component);
    // if component is a processor, update any rules in that processor that uses any of the streams
    if (component instanceof TopologyProcessor) {
        setReconfigureRules(Collections.singletonList((TopologyProcessor) component), edge.getStreamGroupings().stream().map(StreamGrouping::getStreamId).map(sid -> getStreamInfo(edge.getTopologyId(), sid, edge.getVersionId())).filter(curStream -> stream == null || curStream.getId().equals(stream.getId())).collect(Collectors.toList()));
    }
}

Example 5 with TopologyComponent

use of com.hortonworks.streamline.streams.catalog.TopologyComponent in project streamline by hortonworks.

the class StreamCatalogService method validateEdge.

// validate from, to and stream ids of the edge
private void validateEdge(TopologyEdge edge) {
    TopologyOutputComponent from = getFrom(edge);
    if ((from == null || !from.getTopologyId().equals(edge.getTopologyId()))) {
        throw new IllegalArgumentException("Invalid source for edge " + edge);
    }
    TopologyComponent to = getTo(edge);
    if ((to == null || !to.getTopologyId().equals(edge.getTopologyId()))) {
        throw new IllegalArgumentException("Invalid destination for edge " + edge);
    }
    Set<Long> outputStreamIds = new HashSet<>();
    if (from.getOutputStreamIds() != null) {
        outputStreamIds.addAll(from.getOutputStreamIds());
    } else if (from.getOutputStreams() != null) {
        outputStreamIds.addAll(Collections2.transform(from.getOutputStreams(), new Function<TopologyStream, Long>() {

            @Override
            public Long apply(TopologyStream input) {
                return input.getId();
            }
        }));
    }
    Collection<Long> edgeStreamIds = Collections2.transform(edge.getStreamGroupings(), new Function<StreamGrouping, Long>() {

        public Long apply(StreamGrouping streamGrouping) {
            return streamGrouping.getStreamId();
        }
    });
    if (!outputStreamIds.containsAll(edgeStreamIds)) {
        throw new IllegalArgumentException("Edge stream Ids " + edgeStreamIds + " must be a subset of outputStreamIds " + outputStreamIds);
    }
    // check the fields specified in the fields grouping is a subset of the stream fields
    for (StreamGrouping streamGrouping : edge.getStreamGroupings()) {
        List<String> fields;
        if ((fields = streamGrouping.getFields()) != null) {
            Set<String> schemaFieldPatterns = getFieldPatterns(getStreamInfo(edge.getTopologyId(), streamGrouping.getStreamId(), edge.getVersionId()).getFields());
            fields.forEach(field -> {
                schemaFieldPatterns.stream().filter(pat -> field.matches(pat)).findAny().orElseThrow(() -> new IllegalArgumentException("Fields in the grouping " + fields + " must be a subset the stream fields " + schemaFieldPatterns));
            });
        }
    }
}