Examples with KuduClusterTestContext org.apache.apex.malhar.kudu.test.KuduClusterTestContext used on opensource projects

Example 1 with KuduClusterTestContext

use of org.apache.apex.malhar.kudu.test.KuduClusterTestContext in project apex-malhar by apache.

the class KuduPartitionScannerCallableTest method testRowScansForAllDataAcrossAllPartitions.

@KuduClusterTestContext(kuduClusterBasedTest = true)
@Test
public void testRowScansForAllDataAcrossAllPartitions() throws Exception {
    partitonScanStrategy = KuduPartitionScanStrategy.MANY_TABLETS_PER_OPERATOR;
    numberOfKuduInputOperatorPartitions = 1;
    initOperatorState();
    AbstractKuduPartitionScanner<UnitTestTablePojo, InputOperatorControlTuple> currentScanner = unitTestStepwiseScanInputOperator.getScanner();
    // truncate and add some data to the unit test table
    truncateTable();
    // This is per partition and there are 12 partitions
    addTestDataRows(10);
    assertEquals((KuduClientTestCommons.TOTAL_KUDU_TABLETS_FOR_UNITTEST_TABLE * 10), countNumRowsInTable());
    SQLToKuduPredicatesTranslator translator = new SQLToKuduPredicatesTranslator("select * from unittests", new ArrayList<ColumnSchema>(columnDefs.values()));
    List<KuduPartitionScanAssignmentMeta> scansForThisQuery = currentScanner.preparePlanForScanners(translator);
    // Now scan for exact match of counts
    long totalRowsRead = 0;
    unitTestStepwiseScanInputOperator.getBuffer().clear();
    for (KuduPartitionScanAssignmentMeta aSegmentToScan : scansForThisQuery) {
        KuduPartitionScannerCallable<UnitTestTablePojo, InputOperatorControlTuple> threadToScan = new KuduPartitionScannerCallable<>(unitTestStepwiseScanInputOperator, aSegmentToScan, currentScanner.getConnectionPoolForThreads().get(0), unitTestStepwiseScanInputOperator.extractSettersForResultObject(translator), translator);
        totalRowsRead += threadToScan.call();
    }
    // 144 = 120 records + 12 * 2 markers
    int expectedCount = (10 * KuduClientTestCommons.TOTAL_KUDU_TABLETS_FOR_UNITTEST_TABLE) + (2 * KuduClientTestCommons.TOTAL_KUDU_TABLETS_FOR_UNITTEST_TABLE);
    assertEquals(expectedCount, unitTestStepwiseScanInputOperator.getBuffer().size());
    // revert all configs to default
    partitonScanStrategy = KuduPartitionScanStrategy.ONE_TABLET_PER_OPERATOR;
    numberOfKuduInputOperatorPartitions = 5;
}

Example 2 with KuduClusterTestContext

use of org.apache.apex.malhar.kudu.test.KuduClusterTestContext in project apex-malhar by apache.

the class IncrementalStepScanInputOperatorTest method testInit.

@KuduClusterTestContext(kuduClusterBasedTest = true)
@Test
public void testInit() throws Exception {
    Attribute.AttributeMap.DefaultAttributeMap attributeMapForInputOperator = new Attribute.AttributeMap.DefaultAttributeMap();
    attributeMapForInputOperator.put(DAG.APPLICATION_ID, APP_ID);
    operatorContext = mockOperatorContext(OPERATOR_ID_FOR_ONE_TO_ONE_PARTITIONER, attributeMapForInputOperator);
    Attribute.AttributeMap.DefaultAttributeMap portAttributesForInputOperator = new Attribute.AttributeMap.DefaultAttributeMap();
    portAttributesForInputOperator.put(Context.PortContext.TUPLE_CLASS, UnitTestTablePojo.class);
    testPortContext = new TestPortContext(portAttributesForInputOperator);
    incrementalStepScanInputOperator = new IncrementalStepScanInputOperator(UnitTestTablePojo.class, "kuduincrementalstepscaninputoperator.properties");
    incrementalStepScanInputOperator.setNumberOfPartitions(numberOfKuduInputOperatorPartitions);
    incrementalStepScanInputOperator.setPartitionScanStrategy(partitonScanStrategy);
    incrementalStepScanInputOperator.setScanOrderStrategy(scanOrderStrategy);
    partitioningContext = new Partitioner.PartitioningContext() {

        @Override
        public int getParallelPartitionCount() {
            return numberOfKuduInputOperatorPartitions;
        }

        @Override
        public List<Operator.InputPort<?>> getInputPorts() {
            return null;
        }
    };
    partitions = incrementalStepScanInputOperator.definePartitions(new ArrayList(), partitioningContext);
    Iterator<Partitioner.Partition<AbstractKuduInputOperator>> iteratorForMeta = partitions.iterator();
    IncrementalStepScanInputOperator actualOperator = (IncrementalStepScanInputOperator) iteratorForMeta.next().getPartitionedInstance();
    // Adjust the bindings as if apex has completed the partioning.The runtime of the framework does this in reality
    incrementalStepScanInputOperator = actualOperator;
    incrementalStepScanInputOperator.setup(operatorContext);
    incrementalStepScanInputOperator.activate(operatorContext);
    // rewire parent operator to enable proper unit testing method calls
    incrementalStepScanInputOperator.getPartitioner().setPrototypeKuduInputOperator(incrementalStepScanInputOperator);
    incrementalStepScanInputOperator.getScanner().setParentOperator(incrementalStepScanInputOperator);
}

Example 3 with KuduClusterTestContext

use of org.apache.apex.malhar.kudu.test.KuduClusterTestContext in project apex-malhar by apache.

the class KuduCreateUpdateDeleteOutputOperatorTest method setUpKuduOutputOperatorContext.

@KuduClusterTestContext(kuduClusterBasedTest = true)
@Before
public void setUpKuduOutputOperatorContext() throws Exception {
    Attribute.AttributeMap.DefaultAttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
    attributeMap.put(DAG.APPLICATION_ID, APP_ID);
    contextForKuduOutputOperator = mockOperatorContext(OPERATOR_ID_FOR_KUDU_CRUD, attributeMap);
    simpleKuduOutputOperator = new BaseKuduOutputOperator();
    Attribute.AttributeMap.DefaultAttributeMap portAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
    portAttributes.put(Context.PortContext.TUPLE_CLASS, UnitTestTablePojo.class);
    testPortContextForKuduOutput = new TestPortContext(portAttributes);
    simpleKuduOutputOperator.setup(contextForKuduOutputOperator);
    simpleKuduOutputOperator.activate(contextForKuduOutputOperator);
    simpleKuduOutputOperator.input.setup(testPortContextForKuduOutput);
}

Example 4 with KuduClusterTestContext

use of org.apache.apex.malhar.kudu.test.KuduClusterTestContext in project apex-malhar by apache.

the class AbstractKuduPartitionScannerTest method textPrepaprePlanForScanners.

@KuduClusterTestContext(kuduClusterBasedTest = true)
@Test
public void textPrepaprePlanForScanners() throws Exception {
    // no predicates test
    initOperatorState();
    AbstractKuduPartitionScanner<UnitTestTablePojo, InputOperatorControlTuple> scanner = unitTestStepwiseScanInputOperator.getScanner();
    SQLToKuduPredicatesTranslator translator = new SQLToKuduPredicatesTranslator("select introwkey as intColumn from unittests", new ArrayList<ColumnSchema>(columnDefs.values()));
    List<KuduPartitionScanAssignmentMeta> scansForThisQuery = scanner.preparePlanForScanners(translator);
    // No predicates and hence a full scan and 1-1 implies one partition for this attempt
    assertEquals(1, scansForThisQuery.size());
    // many to one partitioner and no predicates
    numberOfKuduInputOperatorPartitions = 4;
    partitonScanStrategy = KuduPartitionScanStrategy.MANY_TABLETS_PER_OPERATOR;
    initOperatorState();
    translator = new SQLToKuduPredicatesTranslator("select introwkey as intColumn from unittests", new ArrayList<ColumnSchema>(columnDefs.values()));
    scanner = unitTestStepwiseScanInputOperator.getScanner();
    scansForThisQuery = scanner.preparePlanForScanners(translator);
    // 12 kudu partitions over 4 Apex partitions = 3
    assertEquals(3, scansForThisQuery.size());
    // many to one partitioner and no predicates
    numberOfKuduInputOperatorPartitions = 4;
    partitonScanStrategy = KuduPartitionScanStrategy.MANY_TABLETS_PER_OPERATOR;
    initOperatorState();
    translator = new SQLToKuduPredicatesTranslator("select introwkey as intColumn from unittests where introwkey = 1", new ArrayList<ColumnSchema>(columnDefs.values()));
    scanner = unitTestStepwiseScanInputOperator.getScanner();
    scansForThisQuery = scanner.preparePlanForScanners(translator);
    // This query will actually result in two tablet scans as there is a hash partition involved as well apart from
    // range partitioning. However this operator is getting one scan as part of the pie assignment. Hence assert 1
    assertEquals(1, scansForThisQuery.size());
    // revert all changes back for subsequent tests
    numberOfKuduInputOperatorPartitions = 5;
    partitonScanStrategy = KuduPartitionScanStrategy.ONE_TABLET_PER_OPERATOR;
}

Example 5 with KuduClusterTestContext

use of org.apache.apex.malhar.kudu.test.KuduClusterTestContext in project apex-malhar by apache.

the class KuduPartitionScannerCallableTest method testRowScansForAllDataInSinglePartition.

@KuduClusterTestContext(kuduClusterBasedTest = true)
@Test
public void testRowScansForAllDataInSinglePartition() throws Exception {
    partitonScanStrategy = KuduPartitionScanStrategy.MANY_TABLETS_PER_OPERATOR;
    initOperatorState();
    AbstractKuduPartitionScanner<UnitTestTablePojo, InputOperatorControlTuple> currentScanner = unitTestStepwiseScanInputOperator.getScanner();
    // truncate and add some data to the unit test table
    truncateTable();
    // This is per partition and there are 12 partitions
    addTestDataRows(10);
    assertEquals((10 * KuduClientTestCommons.TOTAL_KUDU_TABLETS_FOR_UNITTEST_TABLE), countNumRowsInTable());
    // 5 to allow ofr scan to fall in lower
    int intRowBoundary = Integer.MAX_VALUE / SPLIT_COUNT_FOR_INT_ROW_KEY;
    SQLToKuduPredicatesTranslator translator = new SQLToKuduPredicatesTranslator("select * from unittests where introwkey < " + intRowBoundary, new ArrayList<ColumnSchema>(columnDefs.values()));
    List<KuduPartitionScanAssignmentMeta> scansForThisQuery = currentScanner.preparePlanForScanners(translator);
    // Now scan for exact match of counts
    long totalRowsRead = 0;
    unitTestStepwiseScanInputOperator.getBuffer().clear();
    for (KuduPartitionScanAssignmentMeta aSegmentToScan : scansForThisQuery) {
        KuduPartitionScannerCallable<UnitTestTablePojo, InputOperatorControlTuple> threadToScan = new KuduPartitionScannerCallable<>(unitTestStepwiseScanInputOperator, aSegmentToScan, currentScanner.getConnectionPoolForThreads().get(0), unitTestStepwiseScanInputOperator.extractSettersForResultObject(translator), translator);
        totalRowsRead += threadToScan.call();
    }
    // 23  because of the hash distributions and the scan markers. 21 are data records and 2 are end of scan markers
    assertEquals(23L, unitTestStepwiseScanInputOperator.getBuffer().size());
    // revert all configs to default
    partitonScanStrategy = KuduPartitionScanStrategy.ONE_TABLET_PER_OPERATOR;
    numberOfKuduInputOperatorPartitions = 5;
}