Examples with TopologyStream com.hortonworks.streamline.streams.catalog.TopologyStream used on opensource projects

Example 6 with TopologyStream

use of com.hortonworks.streamline.streams.catalog.TopologyStream in project streamline by hortonworks.

the class StreamCatalogService method getOutputStreams.

private List<TopologyStream> getOutputStreams(Long topologyId, Long versionId, List<Long> outputStreamIds) {
    List<TopologyStream> topologyStreams = new ArrayList<>();
    for (Long outputStreamId : outputStreamIds) {
        TopologyStream topologyStream;
        if ((topologyStream = getStreamInfo(topologyId, outputStreamId, versionId)) == null) {
            throw new IllegalArgumentException("Output stream with id '" + outputStreamId + "' does not exist.");
        }
        topologyStreams.add(topologyStream);
    }
    return topologyStreams;
}

Example 7 with TopologyStream

use of com.hortonworks.streamline.streams.catalog.TopologyStream in project streamline by hortonworks.

the class StreamCatalogService method validateEdge.

// validate from, to and stream ids of the edge
private void validateEdge(TopologyEdge edge) {
    TopologyOutputComponent from = getFrom(edge);
    if ((from == null || !from.getTopologyId().equals(edge.getTopologyId()))) {
        throw new IllegalArgumentException("Invalid source for edge " + edge);
    }
    TopologyComponent to = getTo(edge);
    if ((to == null || !to.getTopologyId().equals(edge.getTopologyId()))) {
        throw new IllegalArgumentException("Invalid destination for edge " + edge);
    }
    Set<Long> outputStreamIds = new HashSet<>();
    if (from.getOutputStreamIds() != null) {
        outputStreamIds.addAll(from.getOutputStreamIds());
    } else if (from.getOutputStreams() != null) {
        outputStreamIds.addAll(Collections2.transform(from.getOutputStreams(), new Function<TopologyStream, Long>() {

            @Override
            public Long apply(TopologyStream input) {
                return input.getId();
            }
        }));
    }
    Collection<Long> edgeStreamIds = Collections2.transform(edge.getStreamGroupings(), new Function<StreamGrouping, Long>() {

        public Long apply(StreamGrouping streamGrouping) {
            return streamGrouping.getStreamId();
        }
    });
    if (!outputStreamIds.containsAll(edgeStreamIds)) {
        throw new IllegalArgumentException("Edge stream Ids " + edgeStreamIds + " must be a subset of outputStreamIds " + outputStreamIds);
    }
    // check the fields specified in the fields grouping is a subset of the stream fields
    for (StreamGrouping streamGrouping : edge.getStreamGroupings()) {
        List<String> fields;
        if ((fields = streamGrouping.getFields()) != null) {
            Set<String> schemaFieldPatterns = getFieldPatterns(getStreamInfo(edge.getTopologyId(), streamGrouping.getStreamId(), edge.getVersionId()).getFields());
            fields.forEach(field -> {
                schemaFieldPatterns.stream().filter(pat -> field.matches(pat)).findAny().orElseThrow(() -> new IllegalArgumentException("Fields in the grouping " + fields + " must be a subset the stream fields " + schemaFieldPatterns));
            });
        }
    }
}

Example 8 with TopologyStream

use of com.hortonworks.streamline.streams.catalog.TopologyStream in project streamline by hortonworks.

the class StreamCatalogService method removeStreamInfo.

public TopologyStream removeStreamInfo(Long topologyId, Long streamId, Long versionId) {
    TopologyStream topologyStream = getStreamInfo(topologyId, streamId, versionId);
    if (topologyStream != null) {
        setReconfigureTarget(topologyStream);
        topologyStream = dao.remove(new StorableKey(STREAMINFO_NAMESPACE, topologyStream.getPrimaryKey()));
        topologyStream.setVersionTimestamp(updateVersionTimestamp(versionId).getTimestamp());
    }
    return topologyStream;
}

Example 9 with TopologyStream

use of com.hortonworks.streamline.streams.catalog.TopologyStream in project streamline by hortonworks.

the class StreamCatalogService method removeTopologyDependencies.

private void removeTopologyDependencies(Long topologyId, Long versionId) throws Exception {
    List<QueryParam> topologyIdVersionIdQueryParams = WSUtils.buildTopologyIdAndVersionIdAwareQueryParams(topologyId, versionId, null);
    // remove topology test histories
    Collection<TopologyTestRunHistory> runHistories = listTopologyTestRunHistory(topologyId, versionId);
    runHistories.forEach(history -> removeTopologyTestRunHistory(history.getId()));
    // remove topology test run case
    Collection<TopologyTestRunCase> runCases = listTopologyTestRunCase(topologyIdVersionIdQueryParams);
    for (TopologyTestRunCase runCase : runCases) {
        Collection<TopologyTestRunCaseSource> runCaseSources = listTopologyTestRunCaseSource(runCase.getId());
        Collection<TopologyTestRunCaseSink> runCaseSinks = listTopologyTestRunCaseSink(runCase.getId());
        // remove topology test run case source
        for (TopologyTestRunCaseSource runCaseSource : runCaseSources) {
            removeTopologyTestRunCaseSource(runCaseSource.getId());
        }
        // remove topology test run case sink
        for (TopologyTestRunCaseSink runCaseSink : runCaseSinks) {
            removeTopologyTestRunCaseSink(runCaseSink.getId());
        }
        removeTestRunCase(topologyId, runCase.getId());
    }
    // remove edges
    Collection<TopologyEdge> edges = listTopologyEdges(topologyIdVersionIdQueryParams);
    for (TopologyEdge edge : edges) {
        removeTopologyEdge(topologyId, edge.getId(), versionId);
    }
    // remove rules
    Collection<TopologyRule> topologyRules = listRules(topologyIdVersionIdQueryParams);
    for (TopologyRule topologyRule : topologyRules) {
        removeRule(topologyId, topologyRule.getId(), versionId);
    }
    // remove windowed rules
    Collection<TopologyWindow> topologyWindows = listWindows(topologyIdVersionIdQueryParams);
    for (TopologyWindow topologyWindow : topologyWindows) {
        removeWindow(topologyId, topologyWindow.getId(), versionId);
    }
    // remove branch rules
    Collection<TopologyBranchRule> topologyBranchRules = listBranchRules(topologyIdVersionIdQueryParams);
    for (TopologyBranchRule topologyBranchRule : topologyBranchRules) {
        removeBranchRule(topologyId, topologyBranchRule.getId(), versionId);
    }
    // remove sinks
    Collection<TopologySink> sinks = listTopologySinks(topologyIdVersionIdQueryParams);
    for (TopologySink sink : sinks) {
        removeTopologySink(topologyId, sink.getId(), versionId, false);
    }
    // remove processors
    Collection<TopologyProcessor> processors = listTopologyProcessors(topologyIdVersionIdQueryParams);
    for (TopologyProcessor processor : processors) {
        removeTopologyProcessor(topologyId, processor.getId(), versionId, false);
    }
    // remove sources
    Collection<TopologySource> sources = listTopologySources(topologyIdVersionIdQueryParams);
    for (TopologySource source : sources) {
        removeTopologySource(topologyId, source.getId(), versionId, false);
    }
    // remove output streams
    Collection<TopologyStream> topologyStreams = listStreamInfos(topologyIdVersionIdQueryParams);
    for (TopologyStream topologyStream : topologyStreams) {
        removeStreamInfo(topologyId, topologyStream.getId(), versionId);
    }
    // remove topology editor metadata
    removeTopologyEditorMetadata(topologyId, versionId);
}

Example 10 with TopologyStream

use of com.hortonworks.streamline.streams.catalog.TopologyStream in project streamline by hortonworks.

the class TopologyStreamCatalogResource method addOrUpdateStreamInfo.

/**
 * <p>Updates a stream in the topology.</p>
 * <p>
 * <b>PUT /api/v1/catalog/topologies/:TOPOLOGY_ID/streams/:STREAM_ID</b>
 * <pre>
 * {
 *   "streamId": "default",
 *   "topologyId": 1,
 *   "fields": [
 *     {"name": "f1", "type": "STRING"},
 *     {"name": "f2", "type": "LONG"},
 *     {"name": "f3", "type": "STRING"}
 *   ]
 * }
 * </pre>
 * <i>Sample success response: </i>
 * <pre>
 * {
 *   "responseCode": 1000,
 *   "responseMessage": "Success",
 *   "entity": {
 *     "id": 1,
 *     "streamId": "default",
 *     "topologyId": 1,
 *     "fields": [
 *       {
 *         "name": "f1",
 *         "type": "STRING",
 *         "optional": false
 *       },
 *       {
 *         "name": "f2",
 *         "type": "LONG",
 *         "optional": false
 *       },
 *       {
 *         "name": "f3",
 *         "type": "STRING",
 *         "optional": false
 *       }
 *     ],
 *     "timestamp": 1463238891203
 *   }
 * }
 * </pre>
 *
 * @param id  the id of the stream to be updated
 * @param topologyStream the updated StreamInfo object
 * @return the response
 */
@PUT
@Path("/topologies/{topologyId}/streams/{id}")
@Timed
public Response addOrUpdateStreamInfo(@PathParam("topologyId") Long topologyId, @PathParam("id") Long id, TopologyStream topologyStream, @Context SecurityContext securityContext) {
    SecurityUtil.checkRoleOrPermissions(authorizer, securityContext, Roles.ROLE_TOPOLOGY_SUPER_ADMIN, Topology.NAMESPACE, topologyId, WRITE);
    TopologyStream newTopologyStream = catalogService.addOrUpdateStreamInfo(topologyId, id, topologyStream);
    return WSUtils.respondEntity(newTopologyStream, OK);
}