Examples with Tuple org.apache.reef.io.Tuple used on opensource projects

Example 1 with Tuple

use of org.apache.reef.io.Tuple in project mist by snuspl.

the class FirstFitRebalancerImpl method makeBinsAndItems.

/**
 * Makes bins (event processors with the available size) and items (groups).
 * @param desirableLoad desirable load (size)
 * @param loadTable load table
 * @param eventProcessors event processors
 * @return bins and items
 */
private Tuple<Map<EventProcessor, Double>, List<Group>> makeBinsAndItems(final double desirableLoad, final Map<EventProcessor, Double> loadTable, final List<EventProcessor> eventProcessors) {
    // Make bins
    final Map<EventProcessor, Double> bins = new HashMap<>(groupAllocationTable.size());
    // Make items
    final List<Group> items = new LinkedList<>();
    for (final EventProcessor eventProcessor : eventProcessors) {
        double load = loadTable.get(eventProcessor);
        if (load > desirableLoad) {
            // Add groups until the load is less than the desirable load
            final Iterator<Group> iterator = groupAllocationTable.getValue(eventProcessor).iterator();
            while (load > desirableLoad && iterator.hasNext()) {
                final Group group = iterator.next();
                items.add(group);
                load -= group.getLoad();
            }
        }
        // Put the bin with the size
        bins.put(eventProcessor, desirableLoad - load);
    }
    return new Tuple<>(bins, items);
}

Example 2 with Tuple

use of org.apache.reef.io.Tuple in project mist by snuspl.

the class StatelessOperatorTest method testMapOperation.

/**
 * Test map operation.
 * It converts string to tuple (string, 1).
 */
@Test
public void testMapOperation() throws InjectionException {
    // input stream
    final List<MistDataEvent> inputStream = ImmutableList.of(new MistDataEvent("a", 1L), new MistDataEvent("b", 2L), new MistDataEvent("d", 3L), new MistDataEvent("b", 4L), new MistDataEvent("c", 5L));
    // expected output
    final List<MistEvent> expectedStream = ImmutableList.of(new MistDataEvent(new Tuple<>("a", 1), 1L), new MistDataEvent(new Tuple<>("b", 1), 2L), new MistDataEvent(new Tuple<>("d", 1), 3L), new MistDataEvent(new Tuple<>("b", 1), 4L), new MistDataEvent(new Tuple<>("c", 1), 5L));
    // map function: convert string to tuple
    final MISTFunction<String, Tuple> mapFunc = (mapInput) -> new Tuple<>(mapInput, 1);
    final MapOperator<String, Tuple> mapOperator = new MapOperator<>(mapFunc);
    testStatelessOperator(inputStream, expectedStream, mapOperator);
}

Example 3 with Tuple

use of org.apache.reef.io.Tuple in project mist by snuspl.

the class DefaultDagGeneratorImplTest method testPlanGenerator.

/**
 * Round-trip test of de-serializing AvroOperatorChainDag.
 * @throws org.apache.reef.tang.exceptions.InjectionException
 */
@Test
public void testPlanGenerator() throws InjectionException, IOException, URISyntaxException, ClassNotFoundException {
    // Generate a query
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).flatMap(s -> Arrays.asList(s.split(" "))).filter(s -> s.startsWith("A")).map(s -> new Tuple2<>(s, 1)).reduceByKey(0, String.class, (Integer x, Integer y) -> x + y).textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    // Generate avro operator chain dag
    final Tuple<List<AvroVertex>, List<Edge>> serializedDag = query.getAvroOperatorDag();
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroChainedDag = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(new ArrayList<>()).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(TaskHostname.class, "127.0.0.1");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    final ConfigDagGenerator configDagGenerator = injector.getInstance(ConfigDagGenerator.class);
    final DagGenerator dagGenerator = injector.getInstance(DagGenerator.class);
    final Tuple<String, AvroDag> tuple = new Tuple<>("query-test", avroChainedDag);
    final DAG<ConfigVertex, MISTEdge> configDag = configDagGenerator.generate(tuple.getValue());
    final ExecutionDag executionDag = dagGenerator.generate(configDag, new LinkedList<>());
    // Test execution dag
    final DAG<ExecutionVertex, MISTEdge> dag = executionDag.getDag();
    final Set<ExecutionVertex> sources = dag.getRootVertices();
    Assert.assertEquals(1, sources.size());
    Assert.assertTrue(sources.iterator().next() instanceof PhysicalSource);
    final PhysicalSource source = (PhysicalSource) sources.iterator().next();
    final Map<ExecutionVertex, MISTEdge> nextOps = dag.getEdges(source);
    Assert.assertEquals(1, nextOps.size());
    final PhysicalOperator flatMapOp = (PhysicalOperator) nextOps.entrySet().iterator().next().getKey();
    final PhysicalOperator filterOp = (PhysicalOperator) dag.getEdges(flatMapOp).entrySet().iterator().next().getKey();
    final PhysicalOperator mapOp = (PhysicalOperator) dag.getEdges(filterOp).entrySet().iterator().next().getKey();
    final PhysicalOperator reduceByKeyOp = (PhysicalOperator) dag.getEdges(mapOp).entrySet().iterator().next().getKey();
    final PhysicalSink sink = (PhysicalSink) dag.getEdges(reduceByKeyOp).entrySet().iterator().next().getKey();
    Assert.assertTrue(flatMapOp.getOperator() instanceof FlatMapOperator);
    Assert.assertTrue(filterOp.getOperator() instanceof FilterOperator);
    Assert.assertTrue(mapOp.getOperator() instanceof MapOperator);
    Assert.assertTrue(reduceByKeyOp.getOperator() instanceof ReduceByKeyOperator);
    Assert.assertTrue(sink.getSink() instanceof NettyTextSink);
}

Example 4 with Tuple

use of org.apache.reef.io.Tuple in project mist by snuspl.

the class MergeAwareQueryRemoverTest method generateSimpleDag.

/**
 * Generate a simple query that has the following structure: src -> operator chain -> sink.
 * @param source source
 * @param physicalOperator operator chain
 * @param sink sink
 * @return dag
 */
private Tuple<DAG<ConfigVertex, MISTEdge>, ExecutionDag> generateSimpleDag(final TestSource source, final PhysicalOperator physicalOperator, final PhysicalSink<String> sink, final ConfigVertex srcVertex, final ConfigVertex ocVertex, final ConfigVertex sinkVertex) throws IOException {
    // Create DAG
    final DAG<ConfigVertex, MISTEdge> dag = new AdjacentListDAG<>();
    dag.addVertex(srcVertex);
    dag.addVertex(ocVertex);
    dag.addVertex(sinkVertex);
    dag.addEdge(srcVertex, ocVertex, new MISTEdge(Direction.LEFT));
    dag.addEdge(ocVertex, sinkVertex, new MISTEdge(Direction.LEFT));
    final DAG<ExecutionVertex, MISTEdge> newDag = new AdjacentListDAG<>();
    newDag.addVertex(source);
    newDag.addVertex(physicalOperator);
    newDag.addVertex(sink);
    newDag.addEdge(source, physicalOperator, new MISTEdge(Direction.LEFT));
    newDag.addEdge(physicalOperator, sink, new MISTEdge(Direction.LEFT));
    final ExecutionDag executionDag = new ExecutionDag(newDag);
    return new Tuple<>(dag, executionDag);
}

Example 5 with Tuple

use of org.apache.reef.io.Tuple in project mist by snuspl.

the class MISTQueryImpl method getAvroOperatorDag.

@Override
public Tuple<List<AvroVertex>, List<Edge>> getAvroOperatorDag() {
    final LogicalDagOptimizer logicalDagOptimizer = new LogicalDagOptimizer(dag);
    final DAG<MISTStream, MISTEdge> optimizedDag = logicalDagOptimizer.getOptimizedDAG();
    final Queue<MISTStream> queue = new LinkedList<>();
    final List<MISTStream> vertices = new ArrayList<>();
    final List<Edge> edges = new ArrayList<>();
    // Put all vertices into a queue
    final Iterator<MISTStream> iterator = GraphUtils.topologicalSort(optimizedDag);
    while (iterator.hasNext()) {
        final MISTStream vertex = iterator.next();
        queue.add(vertex);
        vertices.add(vertex);
    }
    // Visit each vertex and serialize its edges
    while (!queue.isEmpty()) {
        final MISTStream vertex = queue.remove();
        final int fromIndex = vertices.indexOf(vertex);
        final Map<MISTStream, MISTEdge> neighbors = optimizedDag.getEdges(vertex);
        for (final Map.Entry<MISTStream, MISTEdge> neighbor : neighbors.entrySet()) {
            final int toIndex = vertices.indexOf(neighbor.getKey());
            final MISTEdge edgeInfo = neighbor.getValue();
            final Edge.Builder edgeBuilder = Edge.newBuilder().setFrom(fromIndex).setTo(toIndex).setDirection(edgeInfo.getDirection()).setBranchIndex(edgeInfo.getIndex());
            edges.add(edgeBuilder.build());
        }
    }
    final Set<MISTStream> rootVertices = optimizedDag.getRootVertices();
    // Serialize each vertex via avro.
    final List<AvroVertex> serializedVertices = new ArrayList<>();
    for (final MISTStream vertex : vertices) {
        final AvroVertex.Builder vertexBuilder = AvroVertex.newBuilder();
        vertexBuilder.setConfiguration(vertex.getConfiguration());
        vertexBuilder.setVertexId(String.valueOf(vertexIdIndex));
        // Set vertex type
        if (rootVertices.contains(vertex)) {
            // this is a source
            vertexBuilder.setAvroVertexType(AvroVertexTypeEnum.SOURCE);
        } else if (optimizedDag.getEdges(vertex).size() == 0) {
            // this is a sink
            vertexBuilder.setAvroVertexType(AvroVertexTypeEnum.SINK);
        } else {
            vertexBuilder.setAvroVertexType(AvroVertexTypeEnum.OPERATOR);
        }
        serializedVertices.add(vertexBuilder.build());
        vertexIdIndex++;
    }
    return new Tuple<>(serializedVertices, edges);
}