Examples with LegacyOperatorTestBuilder org.apache.drill.test.LegacyOperatorTestBuilder used on opensource projects

Example 16 with LegacyOperatorTestBuilder

use of org.apache.drill.test.LegacyOperatorTestBuilder in project drill by apache.

the class TestOutputBatchSize method testNestedLoopJoinLowerLimit.

@Test
public void testNestedLoopJoinLowerLimit() throws Exception {
    // test the lower limit of at least one batch
    LogicalExpression functionCallExpr = new FunctionCall("equal", ImmutableList.of((LogicalExpression) new FieldReference("c1", ExpressionPosition.UNKNOWN), (LogicalExpression) new FieldReference("c2", ExpressionPosition.UNKNOWN)), ExpressionPosition.UNKNOWN);
    NestedLoopJoinPOP nestedLoopJoin = new NestedLoopJoinPOP(null, null, JoinRelType.INNER, functionCallExpr);
    numRows = 10;
    // create left input rows like this.
    // "a1" : 5, "b1" : wideString, "c1" : <id>
    List<String> leftJsonBatches = Lists.newArrayList();
    StringBuilder leftBatchString = new StringBuilder();
    leftBatchString.append("[");
    for (int i = 0; i < numRows; i++) {
        leftBatchString.append("{\"a1\": 5, " + "\"b1\" : " + "\"" + wideString + "\"," + "\"c1\" : " + i + "},");
    }
    leftBatchString.append("{\"a1\": 5, " + "\"b1\" : " + "\"" + wideString + "\"," + "\"c1\" : " + numRows + "}");
    leftBatchString.append("]");
    leftJsonBatches.add(leftBatchString.toString());
    // create right input rows like this.
    // "a2" : 6, "b2" : wideString, "c2" : <id>
    List<String> rightJsonBatches = Lists.newArrayList();
    StringBuilder rightBatchString = new StringBuilder();
    rightBatchString.append("[");
    for (int i = 0; i < numRows; i++) {
        rightBatchString.append("{\"a2\": 6, " + "\"b2\" : " + "\"" + wideString + "\"," + "\"c2\" : " + i + "},");
    }
    rightBatchString.append("{\"a2\": 6, " + "\"b2\" : " + "\"" + wideString + "\"," + "\"c2\" : " + numRows + "}");
    rightBatchString.append("]");
    rightJsonBatches.add(rightBatchString.toString());
    // output rows will be like this.
    // "a1" : 5, "b1" : wideString, "c1" : 1, "a2":6, "b2" : wideString, "c2": 1
    // "a1" : 5, "b1" : wideString, "c1" : 2, "a2":6, "b2" : wideString, "c2": 2
    // "a1" : 5, "b1" : wideString, "c1" : 3, "a2":6, "b2" : wideString, "c2": 3
    // set very low value of output batch size so we can do only one row per batch.
    fragContext.getOptions().setLocalOption("drill.exec.memory.operator.output_batch_size", 128);
    LegacyOperatorTestBuilder opTestBuilder = legacyOpTestBuilder().physicalOperator(nestedLoopJoin).baselineColumns("a1", "b1", "c1", "a2", "b2", "c2").expectedNumBatches(// verify number of batches
    10).inputDataStreamsJson(Lists.newArrayList(leftJsonBatches, rightJsonBatches));
    for (long i = 0; i < numRows + 1; i++) {
        opTestBuilder.baselineValues(5l, wideString, i, 6l, wideString, i);
    }
    opTestBuilder.go();
}

Example 17 with LegacyOperatorTestBuilder

use of org.apache.drill.test.LegacyOperatorTestBuilder in project drill by apache.

the class TestOutputBatchSize method testFlattenFixedWidth.

@Test
public void testFlattenFixedWidth() throws Exception {
    PhysicalOperator flatten = new FlattenPOP(null, SchemaPath.getSimplePath("c"));
    mockOpContext(flatten, initReservation, maxAllocation);
    // create input rows like this.
    // "a" : 5, "b" : wideString, "c" : [6,7,8,9]
    List<String> inputJsonBatches = Lists.newArrayList();
    StringBuilder batchString = new StringBuilder();
    batchString.append("[");
    for (int i = 0; i < numRows; i++) {
        batchString.append("{\"a\": 5, " + "\"b\" : " + "\"" + wideString + "\"," + "\"c\" : [6, 7, 8, 9]},");
    }
    batchString.append("{\"a\": 5, " + "\"b\" : " + "\"" + wideString + "\"," + "\"c\" : [6, 7, 8, 9]}");
    batchString.append("]");
    inputJsonBatches.add(batchString.toString());
    // Figure out what will be approximate total output size out of flatten for input above
    // We will use this sizing information to set output batch size so we can produce desired
    // number of batches that can be verified.
    // output rows will be like this.
    // "a" : 5, "b" : wideString, "c" : 6
    // "a" : 5, "b" : wideString, "c" : 7
    // "a" : 5, "b" : wideString, "c" : 8
    // "a" : 5, "b" : wideString, "c" : 9
    List<String> expectedJsonBatches = Lists.newArrayList();
    StringBuilder expectedBatchString = new StringBuilder();
    expectedBatchString.append("[");
    for (int i = 0; i < numRows; i++) {
        expectedBatchString.append("{\"a\": 5, " + "\"b\" : " + "\"" + wideString + "\"," + "\"c\" : 6},");
        expectedBatchString.append("{\"a\": 5, " + "\"b\" : " + "\"" + wideString + "\"," + "\"c\" : 7},");
        expectedBatchString.append("{\"a\": 5, " + "\"b\" : " + "\"" + wideString + "\"," + "\"c\" : 8},");
        expectedBatchString.append("{\"a\": 5, " + "\"b\" : " + "\"" + wideString + "\"," + "\"c\" : 9},");
    }
    expectedBatchString.append("{\"a\": 5, " + "\"b\" : " + "\"" + wideString + "\"," + "\"c\" : 6},");
    expectedBatchString.append("{\"a\": 5, " + "\"b\" : " + "\"" + wideString + "\"," + "\"c\" : 7},");
    expectedBatchString.append("{\"a\": 5, " + "\"b\" : " + "\"" + wideString + "\"," + "\"c\" : 8},");
    expectedBatchString.append("{\"a\": 5, " + "\"b\" : " + "\"" + wideString + "\"," + "\"c\" : 9}");
    expectedBatchString.append("]");
    expectedJsonBatches.add(expectedBatchString.toString());
    long totalSize = getExpectedSize(expectedJsonBatches);
    // set the output batch size to 1/2 of total size expected.
    // We will get approximately get 2 batches and max of 4.
    fragContext.getOptions().setLocalOption("drill.exec.memory.operator.output_batch_size", totalSize / 2);
    LegacyOperatorTestBuilder opTestBuilder = legacyOpTestBuilder().physicalOperator(flatten).inputDataStreamJson(inputJsonBatches).baselineColumns("a", "b", "c").expectedNumBatches(// verify number of batches
    2).expectedBatchSize(// verify batch size.
    totalSize / 2);
    for (int i = 0; i < numRows + 1; i++) {
        opTestBuilder.baselineValues(5l, wideString, 6l);
        opTestBuilder.baselineValues(5l, wideString, 7l);
        opTestBuilder.baselineValues(5l, wideString, 8l);
        opTestBuilder.baselineValues(5l, wideString, 9l);
    }
    opTestBuilder.go();
}

Example 18 with LegacyOperatorTestBuilder

use of org.apache.drill.test.LegacyOperatorTestBuilder in project drill by apache.

the class BasicPhysicalOpUnitTest method externalSortLowMemoryHelper.

private void externalSortLowMemoryHelper(int batchSize, int numberOfBatches, long initReservation, long maxAllocation) {
    ExternalSort sortConf = new ExternalSort(null, Lists.newArrayList(ordering("b", RelFieldCollation.Direction.ASCENDING, RelFieldCollation.NullDirection.FIRST)), false);
    List<String> inputJsonBatches = Lists.newArrayList();
    StringBuilder batchString = new StringBuilder();
    for (int j = 0; j < numberOfBatches; j++) {
        batchString.append("[");
        for (int i = 0; i < batchSize; i++) {
            batchString.append("{\"a\": 5, \"b\" : 5},{\"a\": 3, \"b\" : 8},");
        }
        batchString.append("{\"a\": 5, \"b\" : 1 }");
        batchString.append("]");
        inputJsonBatches.add(batchString.toString());
    }
    LegacyOperatorTestBuilder opTestBuilder = legacyOpTestBuilder().initReservation(initReservation).maxAllocation(maxAllocation).physicalOperator(sortConf).inputDataStreamJson(inputJsonBatches).baselineColumns("a", "b");
    for (int i = 0; i < numberOfBatches; i++) {
        opTestBuilder.baselineValues(5l, 1l);
    }
    for (int i = 0; i < batchSize * numberOfBatches; i++) {
        opTestBuilder.baselineValues(5l, 5l);
    }
    for (int i = 0; i < batchSize * numberOfBatches; i++) {
        opTestBuilder.baselineValues(3l, 8l);
    }
    opTestBuilder.go();
}

Example 19 with LegacyOperatorTestBuilder

use of org.apache.drill.test.LegacyOperatorTestBuilder in project drill by apache.

the class BasicPhysicalOpUnitTest method testFlatten.

@Test
public void testFlatten() {
    final PhysicalOperator flatten = new FlattenPOP(null, SchemaPath.getSimplePath("b"));
    List<String> inputJsonBatches = Lists.newArrayList();
    StringBuilder batchString = new StringBuilder();
    for (int j = 0; j < 1; j++) {
        batchString.append("[");
        for (int i = 0; i < 1; i++) {
            batchString.append("{\"a\": 5, \"b\" : [5, 6, 7]}");
        }
        batchString.append("]");
        inputJsonBatches.add(batchString.toString());
    }
    LegacyOperatorTestBuilder opTestBuilder = legacyOpTestBuilder().physicalOperator(flatten).inputDataStreamJson(inputJsonBatches).baselineColumns("a", "b").baselineValues(5l, 5l).baselineValues(5l, 6l).baselineValues(5l, 7l);
    opTestBuilder.go();
}

Example 20 with LegacyOperatorTestBuilder

use of org.apache.drill.test.LegacyOperatorTestBuilder in project drill by apache.

the class TestOutputBatchSize method testNestedLoopJoinUpperLimit.

@Test
public void testNestedLoopJoinUpperLimit() throws Exception {
    // test the upper limit of 65535 records per batch.
    LogicalExpression functionCallExpr = new FunctionCall("<", ImmutableList.of((LogicalExpression) new FieldReference("c1", ExpressionPosition.UNKNOWN), (LogicalExpression) new FieldReference("c2", ExpressionPosition.UNKNOWN)), ExpressionPosition.UNKNOWN);
    NestedLoopJoinPOP nestedLoopJoin = new NestedLoopJoinPOP(null, null, JoinRelType.INNER, functionCallExpr);
    numRows = 500;
    // create left input rows like this.
    // "a1" : 5,  "c1" : <id>
    List<String> leftJsonBatches = Lists.newArrayList();
    StringBuilder leftBatchString = new StringBuilder();
    leftBatchString.append("[");
    for (int i = 0; i < numRows; i++) {
        leftBatchString.append("{\"a1\": 5, " + "\"c1\" : " + i + "},");
    }
    leftBatchString.append("{\"a1\": 5, " + "\"c1\" : " + numRows + "}");
    leftBatchString.append("]");
    leftJsonBatches.add(leftBatchString.toString());
    // create right input rows like this.
    // "a2" : 6, "c2" : <id>
    List<String> rightJsonBatches = Lists.newArrayList();
    StringBuilder rightBatchString = new StringBuilder();
    rightBatchString.append("[");
    for (int i = 0; i < numRows; i++) {
        rightBatchString.append("{\"a2\": 6, " + "\"c2\" : " + i + "},");
    }
    rightBatchString.append("{\"a2\": 6, " + "\"c2\" : " + numRows + "}");
    rightBatchString.append("]");
    rightJsonBatches.add(rightBatchString.toString());
    // output rows will be like this.
    // "a1" : 5,  "c1" : 1, "a2":6,  "c2": 1
    // "a1" : 5,  "c1" : 2, "a2":6,  "c2": 2
    // "a1" : 5,  "c1" : 3, "a2":6,  "c2": 3
    // we expect n(n+1)/2 number of records i.e. (500 * 501)/2 = 125250
    // expect two batches, batch limited by 65535 records
    LegacyOperatorTestBuilder opTestBuilder = legacyOpTestBuilder().physicalOperator(nestedLoopJoin).baselineColumns("a1", "c1", "a2", "c2").expectedNumBatches(// verify number of batches
    2).inputDataStreamsJson(Lists.newArrayList(leftJsonBatches, rightJsonBatches));
    for (long i = 0; i < numRows + 1; i++) {
        for (long j = i + 1; j < numRows + 1; j++) {
            opTestBuilder.baselineValues(5l, i, 6l, j);
        }
    }
    opTestBuilder.go();
}