Search in sources :

Example 6 with IOperator

use of edu.uci.ics.texera.api.dataflow.IOperator in project textdb by TextDB.

the class LogicalPlanTest method testGetOutputSchema1.

/*
     * Test getOutputSchema on a valid operator graph.
     *
     * KeywordSource --> RegexMatcher --> TupleSink
     *
     */
@Test
public void testGetOutputSchema1() throws Exception {
    LogicalPlan logicalPlan = getLogicalPlan1();
    Plan queryPlan = logicalPlan.buildQueryPlan();
    HashMap<String, ISink> sinkHashMap = queryPlan.getSinkMap();
    Assert.assertEquals(1, sinkHashMap.size());
    ISink tupleSink = null;
    for (HashMap.Entry<String, ISink> entry : sinkHashMap.entrySet()) {
        tupleSink = entry.getValue();
    }
    Assert.assertNotNull(tupleSink);
    IOperator regexMatcher = ((TupleSink) tupleSink).getInputOperator();
    IOperator keywordSource = ((RegexMatcher) regexMatcher).getInputOperator();
    regexMatcher.open();
    Schema expectedSourceOutputSchema = keywordSource.getOutputSchema();
    Schema expectedMatcherOutputSchema = regexMatcher.getOutputSchema();
    regexMatcher.close();
    Schema sourceOutputSchema = logicalPlan.getOperatorOutputSchema(KEYWORD_SOURCE_ID);
    Schema matcherOutputSchema = logicalPlan.getOperatorOutputSchema(REGEX_ID);
    Assert.assertEquals(expectedSourceOutputSchema, sourceOutputSchema);
    Assert.assertEquals(expectedMatcherOutputSchema, matcherOutputSchema);
}
Also used : ISink(edu.uci.ics.texera.api.dataflow.ISink) TupleSink(edu.uci.ics.texera.dataflow.sink.tuple.TupleSink) HashMap(java.util.HashMap) IOperator(edu.uci.ics.texera.api.dataflow.IOperator) Schema(edu.uci.ics.texera.api.schema.Schema) RegexMatcher(edu.uci.ics.texera.dataflow.regexmatcher.RegexMatcher) Plan(edu.uci.ics.texera.api.engine.Plan) Test(org.junit.Test)

Example 7 with IOperator

use of edu.uci.ics.texera.api.dataflow.IOperator in project textdb by TextDB.

the class LogicalPlan method findSinkOperator.

/*
     * Finds the sink operator in the operator graph.
     * 
     * This function assumes that the graph is valid and there is only one sink in the graph.
     */
private ISink findSinkOperator(HashMap<String, IOperator> operatorObjectMap) throws PlanGenException {
    IOperator sinkOperator = adjacencyList.keySet().stream().filter(operator -> operatorPredicateMap.get(operator).getClass().toString().toLowerCase().contains("sink")).map(operator -> operatorObjectMap.get(operator)).findFirst().orElse(null);
    PlanGenUtils.planGenAssert(sinkOperator != null, "Error: sink operator doesn't exist.");
    PlanGenUtils.planGenAssert(sinkOperator instanceof ISink, "Error: sink operator's type doesn't match.");
    return (ISink) sinkOperator;
}
Also used : Plan(edu.uci.ics.texera.api.engine.Plan) JsonProperty(com.fasterxml.jackson.annotation.JsonProperty) Join(edu.uci.ics.texera.dataflow.join.Join) PropertyNameConstants(edu.uci.ics.texera.dataflow.common.PropertyNameConstants) PlanGenException(edu.uci.ics.texera.api.exception.PlanGenException) HashMap(java.util.HashMap) PredicateBase(edu.uci.ics.texera.dataflow.common.PredicateBase) InvocationTargetException(java.lang.reflect.InvocationTargetException) ArrayList(java.util.ArrayList) OneToNBroadcastConnector(edu.uci.ics.texera.dataflow.connector.OneToNBroadcastConnector) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) List(java.util.List) IOperator(edu.uci.ics.texera.api.dataflow.IOperator) ISink(edu.uci.ics.texera.api.dataflow.ISink) JsonCreator(com.fasterxml.jackson.annotation.JsonCreator) Schema(edu.uci.ics.texera.api.schema.Schema) LinkedHashSet(java.util.LinkedHashSet) ISink(edu.uci.ics.texera.api.dataflow.ISink) IOperator(edu.uci.ics.texera.api.dataflow.IOperator)

Example 8 with IOperator

use of edu.uci.ics.texera.api.dataflow.IOperator in project textdb by TextDB.

the class LogicalPlanTest method testGetOutputSchema2.

/*
     * Test getOutputSchema on a valid operator graph.
     *                  -> RegexMatcher -->
     * KeywordSource --<                     >-- Join --> TupleSink
     *                  -> NlpEntityOperator -->
     *
     */
@Test
public void testGetOutputSchema2() throws Exception {
    LogicalPlan logicalPlan = getLogicalPlan2();
    Plan queryPlan = logicalPlan.buildQueryPlan();
    HashMap<String, ISink> sinkHashMap = queryPlan.getSinkMap();
    Assert.assertEquals(1, sinkHashMap.size());
    ISink tupleSink = null;
    for (HashMap.Entry<String, ISink> entry : sinkHashMap.entrySet()) {
        tupleSink = entry.getValue();
    }
    Assert.assertNotNull(tupleSink);
    IOperator join = ((TupleSink) tupleSink).getInputOperator();
    IOperator joinInput1 = ((Join) join).getInnerInputOperator();
    IOperator joinInput2 = ((Join) join).getOuterInputOperator();
    IOperator connectorOut1 = ((RegexMatcher) joinInput1).getInputOperator();
    IOperator connectorOut2 = ((NlpEntityOperator) joinInput2).getInputOperator();
    OneToNBroadcastConnector connector1 = ((ConnectorOutputOperator) connectorOut1).getOwnerConnector();
    OneToNBroadcastConnector connector2 = ((ConnectorOutputOperator) connectorOut2).getOwnerConnector();
    IOperator keywordSource = connector1.getInputOperator();
    join.open();
    Schema expectedJoinOutputSchema = join.getOutputSchema();
    Schema expectedSourceOutputSchema = keywordSource.getOutputSchema();
    Schema expectedMatcherOutputSchema = joinInput1.getOutputSchema();
    Schema expectedNlpEntityOutputSchema = joinInput2.getOutputSchema();
    join.close();
    Schema joinOutputSchema = logicalPlan.getOperatorOutputSchema(JOIN_DISTANCE_ID);
    Schema sourceOutputSchema = logicalPlan.getOperatorOutputSchema(KEYWORD_SOURCE_ID);
    Schema matcherOutputSchema = logicalPlan.getOperatorOutputSchema(REGEX_ID);
    Schema nlpEntityOutputSchema = logicalPlan.getOperatorOutputSchema(NLP_ENTITY_ID);
    Assert.assertEquals(expectedJoinOutputSchema, joinOutputSchema);
    Assert.assertEquals(expectedSourceOutputSchema, sourceOutputSchema);
    Assert.assertEquals(expectedMatcherOutputSchema, matcherOutputSchema);
    Assert.assertEquals(expectedNlpEntityOutputSchema, nlpEntityOutputSchema);
}
Also used : TupleSink(edu.uci.ics.texera.dataflow.sink.tuple.TupleSink) HashMap(java.util.HashMap) IOperator(edu.uci.ics.texera.api.dataflow.IOperator) Schema(edu.uci.ics.texera.api.schema.Schema) Join(edu.uci.ics.texera.dataflow.join.Join) Plan(edu.uci.ics.texera.api.engine.Plan) ISink(edu.uci.ics.texera.api.dataflow.ISink) ConnectorOutputOperator(edu.uci.ics.texera.dataflow.connector.OneToNBroadcastConnector.ConnectorOutputOperator) NlpEntityOperator(edu.uci.ics.texera.dataflow.nlp.entity.NlpEntityOperator) RegexMatcher(edu.uci.ics.texera.dataflow.regexmatcher.RegexMatcher) OneToNBroadcastConnector(edu.uci.ics.texera.dataflow.connector.OneToNBroadcastConnector) Test(org.junit.Test)

Example 9 with IOperator

use of edu.uci.ics.texera.api.dataflow.IOperator in project textdb by TextDB.

the class LogicalPlanTest method testLogicalPlan2.

/*
     * Test a valid operator graph.
     *                  -> RegexMatcher -->
     * KeywordSource --<                     >-- Join --> TupleSink
     *                  -> NlpEntityOperator -->
     *
     */
@Test
public void testLogicalPlan2() throws Exception {
    LogicalPlan logicalPlan = getLogicalPlan2();
    Plan queryPlan = logicalPlan.buildQueryPlan();
    HashMap<String, ISink> sinkHashMap = queryPlan.getSinkMap();
    Assert.assertEquals(1, sinkHashMap.size());
    ISink tupleSink = null;
    for (HashMap.Entry<String, ISink> entry : sinkHashMap.entrySet()) {
        tupleSink = entry.getValue();
    }
    Assert.assertTrue(tupleSink instanceof TupleSink);
    IOperator join = ((TupleSink) tupleSink).getInputOperator();
    Assert.assertTrue(join instanceof Join);
    IOperator joinInput1 = ((Join) join).getInnerInputOperator();
    Assert.assertTrue(joinInput1 instanceof RegexMatcher);
    IOperator joinInput2 = ((Join) join).getOuterInputOperator();
    Assert.assertTrue(joinInput2 instanceof NlpEntityOperator);
    IOperator connectorOut1 = ((RegexMatcher) joinInput1).getInputOperator();
    Assert.assertTrue(connectorOut1 instanceof ConnectorOutputOperator);
    IOperator connectorOut2 = ((NlpEntityOperator) joinInput2).getInputOperator();
    Assert.assertTrue(connectorOut2 instanceof ConnectorOutputOperator);
    HashSet<Integer> connectorIndices = new HashSet<>();
    connectorIndices.add(((ConnectorOutputOperator) connectorOut1).getOutputIndex());
    connectorIndices.add(((ConnectorOutputOperator) connectorOut2).getOutputIndex());
    Assert.assertEquals(connectorIndices.size(), 2);
    OneToNBroadcastConnector connector1 = ((ConnectorOutputOperator) connectorOut1).getOwnerConnector();
    OneToNBroadcastConnector connector2 = ((ConnectorOutputOperator) connectorOut2).getOwnerConnector();
    Assert.assertSame(connector1, connector2);
    IOperator keywordSource = connector1.getInputOperator();
    Assert.assertTrue(keywordSource instanceof KeywordMatcherSourceOperator);
}
Also used : TupleSink(edu.uci.ics.texera.dataflow.sink.tuple.TupleSink) HashMap(java.util.HashMap) IOperator(edu.uci.ics.texera.api.dataflow.IOperator) Join(edu.uci.ics.texera.dataflow.join.Join) Plan(edu.uci.ics.texera.api.engine.Plan) KeywordMatcherSourceOperator(edu.uci.ics.texera.dataflow.keywordmatcher.KeywordMatcherSourceOperator) ISink(edu.uci.ics.texera.api.dataflow.ISink) ConnectorOutputOperator(edu.uci.ics.texera.dataflow.connector.OneToNBroadcastConnector.ConnectorOutputOperator) NlpEntityOperator(edu.uci.ics.texera.dataflow.nlp.entity.NlpEntityOperator) RegexMatcher(edu.uci.ics.texera.dataflow.regexmatcher.RegexMatcher) OneToNBroadcastConnector(edu.uci.ics.texera.dataflow.connector.OneToNBroadcastConnector) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 10 with IOperator

use of edu.uci.ics.texera.api.dataflow.IOperator in project textdb by TextDB.

the class LogicalPlanTest method testLogicalPlan3.

/*
     * Test a valid operator graph.
     *
     *                  --> RegexMatcher -->
     *                  |                    >-- Join1
     * KeywordSource --< -> NlpEntityOperator -->          >-- Join2 --> TupleSink
     *                  |                           /
     *                  --> FuzzyTokenMatcher ----->
     *
     */
@Test
public void testLogicalPlan3() throws Exception {
    LogicalPlan logicalPlan = getLogicalPlan3();
    Plan queryPlan = logicalPlan.buildQueryPlan();
    HashMap<String, ISink> sinkHashMap = queryPlan.getSinkMap();
    Assert.assertEquals(1, sinkHashMap.size());
    ISink tupleSink = null;
    for (HashMap.Entry<String, ISink> entry : sinkHashMap.entrySet()) {
        tupleSink = entry.getValue();
    }
    Assert.assertTrue(tupleSink instanceof TupleSink);
    IOperator join2 = ((TupleSink) tupleSink).getInputOperator();
    Assert.assertTrue(join2 instanceof Join);
    IOperator join2Input1 = ((Join) join2).getOuterInputOperator();
    Assert.assertTrue(join2Input1 instanceof Join);
    IOperator join2Input2 = ((Join) join2).getInnerInputOperator();
    Assert.assertTrue(join2Input2 instanceof FuzzyTokenMatcher);
    IOperator join1Input1 = ((Join) join2Input1).getInnerInputOperator();
    Assert.assertTrue(join1Input1 instanceof RegexMatcher);
    IOperator join1Input2 = ((Join) join2Input1).getOuterInputOperator();
    Assert.assertTrue(join1Input2 instanceof NlpEntityOperator);
    IOperator connectorOut1 = ((RegexMatcher) join1Input1).getInputOperator();
    Assert.assertTrue(connectorOut1 instanceof ConnectorOutputOperator);
    IOperator connectorOut2 = ((NlpEntityOperator) join1Input2).getInputOperator();
    Assert.assertTrue(connectorOut2 instanceof ConnectorOutputOperator);
    IOperator connectorOut3 = ((FuzzyTokenMatcher) join2Input2).getInputOperator();
    Assert.assertTrue(connectorOut3 instanceof ConnectorOutputOperator);
    HashSet<Integer> connectorIndices = new HashSet<>();
    connectorIndices.add(((ConnectorOutputOperator) connectorOut1).getOutputIndex());
    connectorIndices.add(((ConnectorOutputOperator) connectorOut2).getOutputIndex());
    connectorIndices.add(((ConnectorOutputOperator) connectorOut3).getOutputIndex());
    Assert.assertEquals(connectorIndices.size(), 3);
    OneToNBroadcastConnector connector1 = ((ConnectorOutputOperator) connectorOut1).getOwnerConnector();
    OneToNBroadcastConnector connector2 = ((ConnectorOutputOperator) connectorOut2).getOwnerConnector();
    OneToNBroadcastConnector connector3 = ((ConnectorOutputOperator) connectorOut3).getOwnerConnector();
    Assert.assertSame(connector1, connector2);
    Assert.assertSame(connector1, connector3);
    IOperator keywordSource = connector1.getInputOperator();
    Assert.assertTrue(keywordSource instanceof KeywordMatcherSourceOperator);
}
Also used : TupleSink(edu.uci.ics.texera.dataflow.sink.tuple.TupleSink) HashMap(java.util.HashMap) IOperator(edu.uci.ics.texera.api.dataflow.IOperator) Join(edu.uci.ics.texera.dataflow.join.Join) Plan(edu.uci.ics.texera.api.engine.Plan) FuzzyTokenMatcher(edu.uci.ics.texera.dataflow.fuzzytokenmatcher.FuzzyTokenMatcher) KeywordMatcherSourceOperator(edu.uci.ics.texera.dataflow.keywordmatcher.KeywordMatcherSourceOperator) ISink(edu.uci.ics.texera.api.dataflow.ISink) ConnectorOutputOperator(edu.uci.ics.texera.dataflow.connector.OneToNBroadcastConnector.ConnectorOutputOperator) NlpEntityOperator(edu.uci.ics.texera.dataflow.nlp.entity.NlpEntityOperator) RegexMatcher(edu.uci.ics.texera.dataflow.regexmatcher.RegexMatcher) OneToNBroadcastConnector(edu.uci.ics.texera.dataflow.connector.OneToNBroadcastConnector) HashSet(java.util.HashSet) Test(org.junit.Test)

Aggregations

IOperator (edu.uci.ics.texera.api.dataflow.IOperator)18 Schema (edu.uci.ics.texera.api.schema.Schema)12 Test (org.junit.Test)11 ISink (edu.uci.ics.texera.api.dataflow.ISink)8 Plan (edu.uci.ics.texera.api.engine.Plan)8 HashMap (java.util.HashMap)8 RegexMatcher (edu.uci.ics.texera.dataflow.regexmatcher.RegexMatcher)7 TupleSink (edu.uci.ics.texera.dataflow.sink.tuple.TupleSink)7 ArrayList (java.util.ArrayList)7 Tuple (edu.uci.ics.texera.api.tuple.Tuple)6 IField (edu.uci.ics.texera.api.field.IField)5 TextField (edu.uci.ics.texera.api.field.TextField)5 OneToNBroadcastConnector (edu.uci.ics.texera.dataflow.connector.OneToNBroadcastConnector)5 DateField (edu.uci.ics.texera.api.field.DateField)4 DoubleField (edu.uci.ics.texera.api.field.DoubleField)4 IntegerField (edu.uci.ics.texera.api.field.IntegerField)4 StringField (edu.uci.ics.texera.api.field.StringField)4 Attribute (edu.uci.ics.texera.api.schema.Attribute)4 Join (edu.uci.ics.texera.dataflow.join.Join)4 ConnectorOutputOperator (edu.uci.ics.texera.dataflow.connector.OneToNBroadcastConnector.ConnectorOutputOperator)3