Examples with Builder org.apache.heron.streamlet.Builder used on opensource projects

Example 1 with Builder

use of org.apache.heron.streamlet.Builder in project heron by twitter.

the class StreamletWithMapAndFlatMapAndFilterAndClone method buildTopology.

@Override
protected TestTopologyBuilder buildTopology(TestTopologyBuilder testTopologyBuilder) {
    Builder streamletBuilder = Builder.newBuilder();
    Streamlet<String> streamlet = streamletBuilder.newSource(() -> MONTHS).setName("months-text").flatMap((String m) -> Arrays.asList(m.split(" - "))).setName("months").filter((month) -> SUMMER_MONTHS.contains(month.toLowerCase()) && incomingMonths.add(month.toLowerCase())).setName("summer-months").map((String word) -> word.substring(0, 3)).setName("summer-months-with-short-name");
    List<Streamlet<String>> clonedStreamlet = streamlet.clone(2);
    // Returns Summer Months with year
    clonedStreamlet.get(0).map((String month) -> month + "_2018");
    // Returns Summer Months with Uppercase
    clonedStreamlet.get(1).map((String month) -> month.toUpperCase());
    BuilderImpl streamletBuilderImpl = (BuilderImpl) streamletBuilder;
    TestTopologyBuilder topology = (TestTopologyBuilder) streamletBuilderImpl.build(testTopologyBuilder);
    return topology;
}

Example 2 with Builder

use of org.apache.heron.streamlet.Builder in project heron by twitter.

the class StreamletWithKeybyCountAndReduce method buildTopology.

@Override
protected TestTopologyBuilder buildTopology(TestTopologyBuilder testTopologyBuilder) {
    Builder streamletBuilder = Builder.newBuilder();
    Streamlet<String> monthStreamlet = streamletBuilder.newSource(() -> MONTHS).setName("months-text").flatMap((String m) -> Arrays.asList(m.split(" - "))).setName("months").filter((month) -> incomingMonths.add(month.toLowerCase())).setName("unique-months");
    SerializableFunction<String, String> getSeason = month -> {
        if (SPRING_MONTHS.contains(month)) {
            return "spring";
        } else if (SUMMER_MONTHS.contains(month)) {
            return "summer";
        } else if (FALL_MONTHS.contains(month)) {
            return "fall";
        } else if (WINTER_MONTHS.contains(month)) {
            return "winter";
        } else {
            return "really?";
        }
    };
    SerializableFunction<String, Integer> getNumberOfDays = month -> {
        switch(month) {
            case "january":
                return 31;
            case "february":
                // Dont use this code in real projects
                return 28;
            case "march":
                return 31;
            case "april":
                return 30;
            case "may":
                return 31;
            case "june":
                return 30;
            case "july":
                return 31;
            case "august":
                return 31;
            case "september":
                return 30;
            case "october":
                return 31;
            case "november":
                return 30;
            case "december":
                return 31;
            default:
                // Shouldn't be here
                return -1;
        }
    };
    // Count months per season
    monthStreamlet.keyBy(getSeason, getNumberOfDays).setName("key-by-season").countByKey(x -> x.getKey()).setName("key-by-and-count").map(x -> String.format("%s: %d months", x.getKey(), x.getValue())).setName("to-string");
    // Sum days per season
    monthStreamlet.<String, Integer>reduceByKey(getSeason, getNumberOfDays, StreamletReducers::sum).setName("sum-by-season").map(x -> String.format("%s: %d days", x.getKey(), x.getValue())).setName("to-string-2");
    BuilderImpl streamletBuilderImpl = (BuilderImpl) streamletBuilder;
    TestTopologyBuilder topology = (TestTopologyBuilder) streamletBuilderImpl.build(testTopologyBuilder);
    return topology;
}

Example 3 with Builder

use of org.apache.heron.streamlet.Builder in project heron by twitter.

the class SmartWatchTopology method main.

public static void main(String[] args) throws Exception {
    Builder processingGraphBuilder = Builder.newBuilder();
    processingGraphBuilder.newSource(SmartWatchReading::new).setName("incoming-watch-readings").reduceByKeyAndWindow(// Key extractor
    reading -> reading.getJoggerId(), // Value extractor
    reading -> reading.getFeetRun(), // The time window (1 minute of clock time)
    WindowConfig.TumblingTimeWindow(Duration.ofSeconds(10)), // The reduce function (produces a cumulative sum)
    (cumulative, incoming) -> cumulative + incoming).setName("reduce-to-total-distance-per-jogger").map(keyWindow -> {
        // The per-key result of the previous reduce step
        long totalFeetRun = keyWindow.getValue();
        // The amount of time elapsed
        long startTime = keyWindow.getKey().getWindow().getStartTime();
        long endTime = keyWindow.getKey().getWindow().getEndTime();
        // Cast to float to use as denominator
        long timeLengthMillis = endTime - startTime;
        // The feet-per-minute calculation
        float feetPerMinute = totalFeetRun / (float) (timeLengthMillis / 1000);
        // Reduce to two decimal places
        String paceString = new DecimalFormat("#.##").format(feetPerMinute);
        // Return a per-jogger average pace
        return new KeyValue<>(keyWindow.getKey().getKey(), paceString);
    }).setName("calculate-average-speed").consume(kv -> {
        String logMessage = String.format("(runner: %s, avgFeetPerMinute: %s)", kv.getKey(), kv.getValue());
        LOG.info(logMessage);
    });
    Config config = Config.defaultConfig();
    // Fetches the topology name from the first command-line argument
    String topologyName = StreamletUtils.getTopologyName(args);
    // Finally, the processing graph and configuration are passed to the Runner, which converts
    // the graph into a Heron topology that can be run in a Heron cluster.
    new Runner().run(topologyName, config, processingGraphBuilder);
}

Example 4 with Builder

use of org.apache.heron.streamlet.Builder in project heron by twitter.

the class TransformsTopology method main.

/**
 * All Heron topologies require a main function that defines the topology's behavior
 * at runtime
 */
public static void main(String[] args) throws Exception {
    Builder builder = Builder.newBuilder();
    /**
     * The processing graph consists of a supplier streamlet that emits
     * random integers between 1 and 100. From there, a series of transformers
     * is applied. At the end of the graph, the original value is ultimately
     * unchanged.
     */
    builder.newSource(() -> ThreadLocalRandom.current().nextInt(100)).transform(new DoNothingTransformer<>()).transform(new IncrementTransformer(10)).transform(new IncrementTransformer(-7)).transform(new DoNothingTransformer<>()).transform(new IncrementTransformer(-3)).log();
    Config config = Config.defaultConfig();
    // Fetches the topology name from the first command-line argument
    String topologyName = StreamletUtils.getTopologyName(args);
    // Finally, the processing graph and configuration are passed to the Runner, which converts
    // the graph into a Heron topology that can be run in a Heron cluster.
    new Runner().run(topologyName, config, builder);
}

Example 5 with Builder

use of org.apache.heron.streamlet.Builder in project heron by twitter.

the class WireRequestsTopology method main.

/**
 * All Heron topologies require a main function that defines the topology's behavior
 * at runtime
 */
public static void main(String[] args) throws Exception {
    Builder builder = Builder.newBuilder();
    // Requests from the "quiet" bank branch (high throttling).
    Streamlet<WireRequest> quietBranch = builder.newSource(() -> new WireRequest(20)).setNumPartitions(1).setName("quiet-branch-requests").filter(WireRequestsTopology::checkRequestAmount).setName("quiet-branch-check-balance");
    // Requests from the "medium" bank branch (medium throttling).
    Streamlet<WireRequest> mediumBranch = builder.newSource(() -> new WireRequest(10)).setNumPartitions(2).setName("medium-branch-requests").filter(WireRequestsTopology::checkRequestAmount).setName("medium-branch-check-balance");
    // Requests from the "busy" bank branch (low throttling).
    Streamlet<WireRequest> busyBranch = builder.newSource(() -> new WireRequest(5)).setNumPartitions(4).setName("busy-branch-requests").filter(WireRequestsTopology::checkRequestAmount).setName("busy-branch-check-balance");
    // Here, the streamlets for the three bank branches are united into one. The fraud
    // detection filter then operates on that unified streamlet.
    quietBranch.union(mediumBranch).setNumPartitions(2).setName("union-1").union(busyBranch).setName("union-2").setNumPartitions(4).filter(WireRequestsTopology::fraudDetect).setName("all-branches-fraud-detect").log();
    Config config = Config.newBuilder().setDeliverySemantics(Config.DeliverySemantics.EFFECTIVELY_ONCE).setNumContainers(2).build();
    // Fetches the topology name from the first command-line argument
    String topologyName = StreamletUtils.getTopologyName(args);
    // Finally, the processing graph and configuration are passed to the Runner, which converts
    // the graph into a Heron topology that can be run in a Heron cluster.
    new Runner().run(topologyName, config, builder);
}