Examples with Stream com.hortonworks.streamline.streams.layout.component.Stream used on opensource projects

Example 6 with Stream

use of com.hortonworks.streamline.streams.layout.component.Stream in project streamline by hortonworks.

the class TopologyComponentFactory method createRulesProcessorProvider.

private Provider<StreamlineProcessor> createRulesProcessorProvider(final RuleExtractor ruleExtractor) {
    return new Provider<StreamlineProcessor>() {

        @Override
        public StreamlineProcessor create(TopologyComponent component) {
            RulesProcessor processor = new RulesProcessor();
            ObjectMapper objectMapper = new ObjectMapper();
            if (component instanceof TopologyOutputComponent) {
                Set<Stream> outputStreams = createOutputStreams((TopologyOutputComponent) component);
                processor.addOutputStreams(outputStreams);
            } else {
                throw new IllegalArgumentException("Component " + component + " must be an instance of TopologyOutputComponent");
            }
            boolean processAll = component.getConfig().getBoolean(RulesProcessor.CONFIG_PROCESS_ALL, true);
            processor.setProcessAll(processAll);
            Object ruleList = component.getConfig().getAny(RulesProcessor.CONFIG_KEY_RULES);
            List<Long> ruleIds = objectMapper.convertValue(ruleList, new TypeReference<List<Long>>() {
            });
            try {
                List<Rule> rules = new ArrayList<>();
                for (Long ruleId : ruleIds) {
                    rules.add(ruleExtractor.getRule(component.getTopologyId(), ruleId, component.getVersionId()));
                }
                processor.setRules(rules);
            } catch (Exception ex) {
                throw new RuntimeException(ex);
            }
            return processor;
        }
    };
}

Example 7 with Stream

use of com.hortonworks.streamline.streams.layout.component.Stream in project streamline by hortonworks.

the class TopologyComponentFactory method normalizationProcessorProvider.

private Map.Entry<String, Provider<StreamlineProcessor>> normalizationProcessorProvider() {
    Provider<StreamlineProcessor> provider = new Provider<StreamlineProcessor>() {

        @Override
        public StreamlineProcessor create(TopologyComponent component) {
            Config config = component.getConfig();
            Object typeObj = config.getAny(NormalizationProcessor.CONFIG_KEY_TYPE);
            Object normConfObj = config.getAny(NormalizationProcessor.CONFIG_KEY_NORMALIZATION);
            ObjectMapper objectMapper = new ObjectMapper();
            NormalizationProcessor.Type type = objectMapper.convertValue(typeObj, NormalizationProcessor.Type.class);
            Map<String, NormalizationConfig> normConfig = objectMapper.convertValue(normConfObj, new TypeReference<Map<String, NormalizationConfig>>() {
            });
            updateWithSchemas(component.getTopologyId(), component.getVersionId(), normConfig);
            Set<Stream> outputStreams;
            if (component instanceof TopologyOutputComponent) {
                outputStreams = createOutputStreams((TopologyOutputComponent) component);
            } else {
                throw new IllegalArgumentException("Component " + component + " must be an instance of TopologyOutputComponent");
            }
            if (outputStreams.size() != 1) {
                throw new IllegalArgumentException("Normalization component [" + component + "] must have only one output stream");
            }
            return new NormalizationProcessor(normConfig, outputStreams.iterator().next(), type);
        }
    };
    return new SimpleImmutableEntry<>(NORMALIZATION, provider);
}

Example 8 with Stream

use of com.hortonworks.streamline.streams.layout.component.Stream in project streamline by hortonworks.

the class TopologyComponentFactory method getStreamlineEdge.

public Edge getStreamlineEdge(TopologyEdge topologyEdge) {
    Edge edge = new Edge();
    edge.setId(topologyEdge.getId().toString());
    edge.setFrom(getOutputComponent(topologyEdge));
    edge.setTo(getInputComponent(topologyEdge));
    Set<StreamGrouping> streamGroupings = new HashSet<>();
    for (TopologyEdge.StreamGrouping streamGrouping : topologyEdge.getStreamGroupings()) {
        Stream stream = getStream(catalogService.getStreamInfo(topologyEdge.getTopologyId(), streamGrouping.getStreamId(), topologyEdge.getVersionId()));
        Stream.Grouping grouping = Stream.Grouping.valueOf(streamGrouping.getGrouping().name());
        streamGroupings.add(new StreamGrouping(stream, grouping, streamGrouping.getFields()));
    }
    edge.addStreamGroupings(streamGroupings);
    return edge;
}

Example 9 with Stream

use of com.hortonworks.streamline.streams.layout.component.Stream in project streamline by hortonworks.

the class RuleParserTest method testParseAgg.

@Test
public void testParseAgg() throws Exception {
    final UDF stddevp = new UDF();
    stddevp.setClassName("foo.class.name");
    stddevp.setDescription("stddev p");
    stddevp.setId(100L);
    stddevp.setJarStoragePath("jarstoragepath");
    stddevp.setName("stddevp");
    stddevp.setType(Udf.Type.AGGREGATE);
    new Expectations() {

        {
            mockCatalogService.listStreamInfos(withAny(new ArrayList<QueryParam>()));
            result = mockTopologyStream;
            mockCatalogService.listUDFs();
            result = Collections.singleton(stddevp);
            mockTopologyStream.getStreamId();
            result = "teststream";
            mockTopologyStream.getFields();
            result = Arrays.asList(Schema.Field.of("temperature", Schema.Type.LONG), Schema.Field.of("humidity", Schema.Type.LONG));
        }
    };
    TopologyRule topologyRule = new TopologyRule();
    topologyRule.setId(1L);
    topologyRule.setName("Test");
    topologyRule.setDescription("test rule");
    topologyRule.setTopologyId(1L);
    topologyRule.setVersionId(1L);
    topologyRule.setSql("select stddevp(temperature) from teststream");
    RuleParser ruleParser = new RuleParser(mockCatalogService, topologyRule.getSql(), topologyRule.getTopologyId(), topologyRule.getVersionId());
    ruleParser.parse();
    LOG.info("Projection: [{}]", ruleParser.getProjection());
    assertEquals(1, ruleParser.getStreams().size());
    assertEquals(new Stream("teststream", Arrays.asList(Schema.Field.of("temperature", Schema.Type.LONG), Schema.Field.of("humidity", Schema.Type.LONG))), ruleParser.getStreams().get(0));
    assertNull(ruleParser.getGroupBy());
    assertNull(ruleParser.getHaving());
}

Example 10 with Stream

use of com.hortonworks.streamline.streams.layout.component.Stream in project streamline by hortonworks.

the class RuleParserTest method testParseSimple.

@Test
public void testParseSimple() throws Exception {
    new Expectations() {

        {
            mockCatalogService.listStreamInfos(withAny(new ArrayList<QueryParam>()));
            result = mockTopologyStream;
            mockTopologyStream.getStreamId();
            result = "teststream";
            mockTopologyStream.getFields();
            result = Arrays.asList(Schema.Field.of("temperature", Schema.Type.LONG), Schema.Field.of("humidity", Schema.Type.LONG));
        }
    };
    TopologyRule topologyRule = new TopologyRule();
    topologyRule.setId(1L);
    topologyRule.setName("Test");
    topologyRule.setDescription("test rule");
    topologyRule.setTopologyId(1L);
    topologyRule.setVersionId(1L);
    topologyRule.setSql("select temperature as temp from teststream where humidity > 80");
    RuleParser ruleParser = new RuleParser(mockCatalogService, topologyRule.getSql(), topologyRule.getTopologyId(), topologyRule.getVersionId());
    ruleParser.parse();
    assertEquals(new Condition(new BinaryExpression(Operator.GREATER_THAN, new FieldExpression(Schema.Field.of("humidity", Schema.Type.LONG)), new Literal("80"))), ruleParser.getCondition());
    assertEquals(new Projection(Arrays.asList(new AsExpression(new FieldExpression(Schema.Field.of("temperature", Schema.Type.LONG)), "TEMP"))), ruleParser.getProjection());
    assertEquals(1, ruleParser.getStreams().size());
    assertEquals(new Stream("teststream", Arrays.asList(Schema.Field.of("temperature", Schema.Type.LONG), Schema.Field.of("humidity", Schema.Type.LONG))), ruleParser.getStreams().get(0));
    assertNull(ruleParser.getGroupBy());
    assertNull(ruleParser.getHaving());
    assertTrue(ruleParser.getCondition().getExpression() instanceof BinaryExpression);
    assertTrue(((BinaryExpression) ruleParser.getCondition().getExpression()).getSecond() instanceof Literal);
    Literal literal = ((Literal) ((BinaryExpression) ruleParser.getCondition().getExpression()).getSecond());
    assertEquals("80", literal.getValue());
}