Examples with InputOperatorControlTuple org.apache.apex.malhar.kudu.InputOperatorControlTuple used on opensource projects

Example 1 with InputOperatorControlTuple

use of org.apache.apex.malhar.kudu.InputOperatorControlTuple in project apex-malhar by apache.

the class KuduPartitionScannerCallableTest method testRowScansForAllDataAcrossAllPartitions.

@KuduClusterTestContext(kuduClusterBasedTest = true)
@Test
public void testRowScansForAllDataAcrossAllPartitions() throws Exception {
    partitonScanStrategy = KuduPartitionScanStrategy.MANY_TABLETS_PER_OPERATOR;
    numberOfKuduInputOperatorPartitions = 1;
    initOperatorState();
    AbstractKuduPartitionScanner<UnitTestTablePojo, InputOperatorControlTuple> currentScanner = unitTestStepwiseScanInputOperator.getScanner();
    // truncate and add some data to the unit test table
    truncateTable();
    // This is per partition and there are 12 partitions
    addTestDataRows(10);
    assertEquals((KuduClientTestCommons.TOTAL_KUDU_TABLETS_FOR_UNITTEST_TABLE * 10), countNumRowsInTable());
    SQLToKuduPredicatesTranslator translator = new SQLToKuduPredicatesTranslator("select * from unittests", new ArrayList<ColumnSchema>(columnDefs.values()));
    List<KuduPartitionScanAssignmentMeta> scansForThisQuery = currentScanner.preparePlanForScanners(translator);
    // Now scan for exact match of counts
    long totalRowsRead = 0;
    unitTestStepwiseScanInputOperator.getBuffer().clear();
    for (KuduPartitionScanAssignmentMeta aSegmentToScan : scansForThisQuery) {
        KuduPartitionScannerCallable<UnitTestTablePojo, InputOperatorControlTuple> threadToScan = new KuduPartitionScannerCallable<>(unitTestStepwiseScanInputOperator, aSegmentToScan, currentScanner.getConnectionPoolForThreads().get(0), unitTestStepwiseScanInputOperator.extractSettersForResultObject(translator), translator);
        totalRowsRead += threadToScan.call();
    }
    // 144 = 120 records + 12 * 2 markers
    int expectedCount = (10 * KuduClientTestCommons.TOTAL_KUDU_TABLETS_FOR_UNITTEST_TABLE) + (2 * KuduClientTestCommons.TOTAL_KUDU_TABLETS_FOR_UNITTEST_TABLE);
    assertEquals(expectedCount, unitTestStepwiseScanInputOperator.getBuffer().size());
    // revert all configs to default
    partitonScanStrategy = KuduPartitionScanStrategy.ONE_TABLET_PER_OPERATOR;
    numberOfKuduInputOperatorPartitions = 5;
}

Example 2 with InputOperatorControlTuple

use of org.apache.apex.malhar.kudu.InputOperatorControlTuple in project apex-malhar by apache.

the class AbstractKuduPartitionScannerTest method textPrepaprePlanForScanners.

@KuduClusterTestContext(kuduClusterBasedTest = true)
@Test
public void textPrepaprePlanForScanners() throws Exception {
    // no predicates test
    initOperatorState();
    AbstractKuduPartitionScanner<UnitTestTablePojo, InputOperatorControlTuple> scanner = unitTestStepwiseScanInputOperator.getScanner();
    SQLToKuduPredicatesTranslator translator = new SQLToKuduPredicatesTranslator("select introwkey as intColumn from unittests", new ArrayList<ColumnSchema>(columnDefs.values()));
    List<KuduPartitionScanAssignmentMeta> scansForThisQuery = scanner.preparePlanForScanners(translator);
    // No predicates and hence a full scan and 1-1 implies one partition for this attempt
    assertEquals(1, scansForThisQuery.size());
    // many to one partitioner and no predicates
    numberOfKuduInputOperatorPartitions = 4;
    partitonScanStrategy = KuduPartitionScanStrategy.MANY_TABLETS_PER_OPERATOR;
    initOperatorState();
    translator = new SQLToKuduPredicatesTranslator("select introwkey as intColumn from unittests", new ArrayList<ColumnSchema>(columnDefs.values()));
    scanner = unitTestStepwiseScanInputOperator.getScanner();
    scansForThisQuery = scanner.preparePlanForScanners(translator);
    // 12 kudu partitions over 4 Apex partitions = 3
    assertEquals(3, scansForThisQuery.size());
    // many to one partitioner and no predicates
    numberOfKuduInputOperatorPartitions = 4;
    partitonScanStrategy = KuduPartitionScanStrategy.MANY_TABLETS_PER_OPERATOR;
    initOperatorState();
    translator = new SQLToKuduPredicatesTranslator("select introwkey as intColumn from unittests where introwkey = 1", new ArrayList<ColumnSchema>(columnDefs.values()));
    scanner = unitTestStepwiseScanInputOperator.getScanner();
    scansForThisQuery = scanner.preparePlanForScanners(translator);
    // This query will actually result in two tablet scans as there is a hash partition involved as well apart from
    // range partitioning. However this operator is getting one scan as part of the pie assignment. Hence assert 1
    assertEquals(1, scansForThisQuery.size());
    // revert all changes back for subsequent tests
    numberOfKuduInputOperatorPartitions = 5;
    partitonScanStrategy = KuduPartitionScanStrategy.ONE_TABLET_PER_OPERATOR;
}

Example 3 with InputOperatorControlTuple

use of org.apache.apex.malhar.kudu.InputOperatorControlTuple in project apex-malhar by apache.

the class KuduPartitionScannerCallableTest method testRowScansForAllDataInSinglePartition.

@KuduClusterTestContext(kuduClusterBasedTest = true)
@Test
public void testRowScansForAllDataInSinglePartition() throws Exception {
    partitonScanStrategy = KuduPartitionScanStrategy.MANY_TABLETS_PER_OPERATOR;
    initOperatorState();
    AbstractKuduPartitionScanner<UnitTestTablePojo, InputOperatorControlTuple> currentScanner = unitTestStepwiseScanInputOperator.getScanner();
    // truncate and add some data to the unit test table
    truncateTable();
    // This is per partition and there are 12 partitions
    addTestDataRows(10);
    assertEquals((10 * KuduClientTestCommons.TOTAL_KUDU_TABLETS_FOR_UNITTEST_TABLE), countNumRowsInTable());
    // 5 to allow ofr scan to fall in lower
    int intRowBoundary = Integer.MAX_VALUE / SPLIT_COUNT_FOR_INT_ROW_KEY;
    SQLToKuduPredicatesTranslator translator = new SQLToKuduPredicatesTranslator("select * from unittests where introwkey < " + intRowBoundary, new ArrayList<ColumnSchema>(columnDefs.values()));
    List<KuduPartitionScanAssignmentMeta> scansForThisQuery = currentScanner.preparePlanForScanners(translator);
    // Now scan for exact match of counts
    long totalRowsRead = 0;
    unitTestStepwiseScanInputOperator.getBuffer().clear();
    for (KuduPartitionScanAssignmentMeta aSegmentToScan : scansForThisQuery) {
        KuduPartitionScannerCallable<UnitTestTablePojo, InputOperatorControlTuple> threadToScan = new KuduPartitionScannerCallable<>(unitTestStepwiseScanInputOperator, aSegmentToScan, currentScanner.getConnectionPoolForThreads().get(0), unitTestStepwiseScanInputOperator.extractSettersForResultObject(translator), translator);
        totalRowsRead += threadToScan.call();
    }
    // 23  because of the hash distributions and the scan markers. 21 are data records and 2 are end of scan markers
    assertEquals(23L, unitTestStepwiseScanInputOperator.getBuffer().size());
    // revert all configs to default
    partitonScanStrategy = KuduPartitionScanStrategy.ONE_TABLET_PER_OPERATOR;
    numberOfKuduInputOperatorPartitions = 5;
}

Example 4 with InputOperatorControlTuple

use of org.apache.apex.malhar.kudu.InputOperatorControlTuple in project apex-malhar by apache.

the class KuduPartitionScannerCallableTest method testSettersForPojo.

@KuduClusterTestContext(kuduClusterBasedTest = true)
@Test
public void testSettersForPojo() throws Exception {
    initOperatorState();
    AbstractKuduPartitionScanner<UnitTestTablePojo, InputOperatorControlTuple> currentScanner = unitTestStepwiseScanInputOperator.getScanner();
    SQLToKuduPredicatesTranslator translator = new SQLToKuduPredicatesTranslator("select introwkey as intColumn from unittests", new ArrayList<ColumnSchema>(columnDefs.values()));
    List<KuduPartitionScanAssignmentMeta> scansForThisQuery = currentScanner.preparePlanForScanners(translator);
    KuduPartitionScannerCallable<UnitTestTablePojo, InputOperatorControlTuple> threadToScan = new KuduPartitionScannerCallable<>(unitTestStepwiseScanInputOperator, scansForThisQuery.get(0), currentScanner.getConnectionPoolForThreads().get(0), unitTestStepwiseScanInputOperator.extractSettersForResultObject(translator), translator);
    long countOfScan = threadToScan.call();
}