Examples with SQLToKuduPredicatesTranslator org.apache.apex.malhar.kudu.sqltranslator.SQLToKuduPredicatesTranslator used on opensource projects

Example 1 with SQLToKuduPredicatesTranslator

use of org.apache.apex.malhar.kudu.sqltranslator.SQLToKuduPredicatesTranslator in project apex-malhar by apache.

the class KuduPartitionScannerCallableTest method testRowScansForAllDataAcrossAllPartitions.

@KuduClusterTestContext(kuduClusterBasedTest = true)
@Test
public void testRowScansForAllDataAcrossAllPartitions() throws Exception {
    partitonScanStrategy = KuduPartitionScanStrategy.MANY_TABLETS_PER_OPERATOR;
    numberOfKuduInputOperatorPartitions = 1;
    initOperatorState();
    AbstractKuduPartitionScanner<UnitTestTablePojo, InputOperatorControlTuple> currentScanner = unitTestStepwiseScanInputOperator.getScanner();
    // truncate and add some data to the unit test table
    truncateTable();
    // This is per partition and there are 12 partitions
    addTestDataRows(10);
    assertEquals((KuduClientTestCommons.TOTAL_KUDU_TABLETS_FOR_UNITTEST_TABLE * 10), countNumRowsInTable());
    SQLToKuduPredicatesTranslator translator = new SQLToKuduPredicatesTranslator("select * from unittests", new ArrayList<ColumnSchema>(columnDefs.values()));
    List<KuduPartitionScanAssignmentMeta> scansForThisQuery = currentScanner.preparePlanForScanners(translator);
    // Now scan for exact match of counts
    long totalRowsRead = 0;
    unitTestStepwiseScanInputOperator.getBuffer().clear();
    for (KuduPartitionScanAssignmentMeta aSegmentToScan : scansForThisQuery) {
        KuduPartitionScannerCallable<UnitTestTablePojo, InputOperatorControlTuple> threadToScan = new KuduPartitionScannerCallable<>(unitTestStepwiseScanInputOperator, aSegmentToScan, currentScanner.getConnectionPoolForThreads().get(0), unitTestStepwiseScanInputOperator.extractSettersForResultObject(translator), translator);
        totalRowsRead += threadToScan.call();
    }
    // 144 = 120 records + 12 * 2 markers
    int expectedCount = (10 * KuduClientTestCommons.TOTAL_KUDU_TABLETS_FOR_UNITTEST_TABLE) + (2 * KuduClientTestCommons.TOTAL_KUDU_TABLETS_FOR_UNITTEST_TABLE);
    assertEquals(expectedCount, unitTestStepwiseScanInputOperator.getBuffer().size());
    // revert all configs to default
    partitonScanStrategy = KuduPartitionScanStrategy.ONE_TABLET_PER_OPERATOR;
    numberOfKuduInputOperatorPartitions = 5;
}

Example 2 with SQLToKuduPredicatesTranslator

use of org.apache.apex.malhar.kudu.sqltranslator.SQLToKuduPredicatesTranslator in project apex-malhar by apache.

the class AbstractKuduPartitionScannerTest method textPrepaprePlanForScanners.

@KuduClusterTestContext(kuduClusterBasedTest = true)
@Test
public void textPrepaprePlanForScanners() throws Exception {
    // no predicates test
    initOperatorState();
    AbstractKuduPartitionScanner<UnitTestTablePojo, InputOperatorControlTuple> scanner = unitTestStepwiseScanInputOperator.getScanner();
    SQLToKuduPredicatesTranslator translator = new SQLToKuduPredicatesTranslator("select introwkey as intColumn from unittests", new ArrayList<ColumnSchema>(columnDefs.values()));
    List<KuduPartitionScanAssignmentMeta> scansForThisQuery = scanner.preparePlanForScanners(translator);
    // No predicates and hence a full scan and 1-1 implies one partition for this attempt
    assertEquals(1, scansForThisQuery.size());
    // many to one partitioner and no predicates
    numberOfKuduInputOperatorPartitions = 4;
    partitonScanStrategy = KuduPartitionScanStrategy.MANY_TABLETS_PER_OPERATOR;
    initOperatorState();
    translator = new SQLToKuduPredicatesTranslator("select introwkey as intColumn from unittests", new ArrayList<ColumnSchema>(columnDefs.values()));
    scanner = unitTestStepwiseScanInputOperator.getScanner();
    scansForThisQuery = scanner.preparePlanForScanners(translator);
    // 12 kudu partitions over 4 Apex partitions = 3
    assertEquals(3, scansForThisQuery.size());
    // many to one partitioner and no predicates
    numberOfKuduInputOperatorPartitions = 4;
    partitonScanStrategy = KuduPartitionScanStrategy.MANY_TABLETS_PER_OPERATOR;
    initOperatorState();
    translator = new SQLToKuduPredicatesTranslator("select introwkey as intColumn from unittests where introwkey = 1", new ArrayList<ColumnSchema>(columnDefs.values()));
    scanner = unitTestStepwiseScanInputOperator.getScanner();
    scansForThisQuery = scanner.preparePlanForScanners(translator);
    // This query will actually result in two tablet scans as there is a hash partition involved as well apart from
    // range partitioning. However this operator is getting one scan as part of the pie assignment. Hence assert 1
    assertEquals(1, scansForThisQuery.size());
    // revert all changes back for subsequent tests
    numberOfKuduInputOperatorPartitions = 5;
    partitonScanStrategy = KuduPartitionScanStrategy.ONE_TABLET_PER_OPERATOR;
}

Example 3 with SQLToKuduPredicatesTranslator

use of org.apache.apex.malhar.kudu.sqltranslator.SQLToKuduPredicatesTranslator in project apex-malhar by apache.

the class AbstractKuduInputOperator method processForQueryString.

/**
 * Implements  the logic to process a given query. Note that this method is invoked from two states.
 *  One state when the Operator is resuming from a checkpointed state and the second when a new query needs to be
 *   processed as part of the normal execution processing.
 */
protected boolean processForQueryString(String queryExpression) {
    LOG.info("Processing query " + queryExpression);
    SQLToKuduPredicatesTranslator parsedQuery = null;
    try {
        parsedQuery = new SQLToKuduPredicatesTranslator(queryExpression, new ArrayList<ColumnSchema>(kuduColNameToSchemaMapping.values()));
        parsedQuery.parseKuduExpression();
    } catch (Exception e) {
        LOG.error("Could not parse the SQL expression/query " + e.getMessage(), e);
        errorPort.emit(queryExpression);
        return false;
    }
    if (parsedQuery.getErrorListener().isSyntaxError()) {
        LOG.error(" Query is an invalid query " + queryExpression);
        errorPort.emit(queryExpression);
        return false;
    }
    if (!parsedQuery.getKuduSQLParseTreeListener().isSuccessfullyParsed()) {
        LOG.error(" Query could not be successfully parsed instead of being syntactically correct " + queryExpression);
        errorPort.emit(queryExpression);
        return false;
    }
    Map<String, Object> setters = extractSettersForResultObject(parsedQuery);
    try {
        currentQueryBeingProcessed = queryExpression;
        optionsEnabledForCurrentQuery.clear();
        optionsEnabledForCurrentQuery.putAll(parsedQuery.getKuduSQLParseTreeListener().getOptionsUsed());
        plannedSegmentsForCurrentQuery = scanner.scanAllRecords(parsedQuery, setters);
    } catch (IOException e) {
        LOG.error("Error while scanning the kudu segments " + e.getMessage(), e);
        errorPort.emit(queryExpression);
        return false;
    }
    LOG.info("Query" + queryExpression + " submitted for scanning");
    return true;
}

Example 4 with SQLToKuduPredicatesTranslator

use of org.apache.apex.malhar.kudu.sqltranslator.SQLToKuduPredicatesTranslator in project apex-malhar by apache.

the class KuduPartitionScannerCallableTest method testRowScansForAllDataInSinglePartition.

@KuduClusterTestContext(kuduClusterBasedTest = true)
@Test
public void testRowScansForAllDataInSinglePartition() throws Exception {
    partitonScanStrategy = KuduPartitionScanStrategy.MANY_TABLETS_PER_OPERATOR;
    initOperatorState();
    AbstractKuduPartitionScanner<UnitTestTablePojo, InputOperatorControlTuple> currentScanner = unitTestStepwiseScanInputOperator.getScanner();
    // truncate and add some data to the unit test table
    truncateTable();
    // This is per partition and there are 12 partitions
    addTestDataRows(10);
    assertEquals((10 * KuduClientTestCommons.TOTAL_KUDU_TABLETS_FOR_UNITTEST_TABLE), countNumRowsInTable());
    // 5 to allow ofr scan to fall in lower
    int intRowBoundary = Integer.MAX_VALUE / SPLIT_COUNT_FOR_INT_ROW_KEY;
    SQLToKuduPredicatesTranslator translator = new SQLToKuduPredicatesTranslator("select * from unittests where introwkey < " + intRowBoundary, new ArrayList<ColumnSchema>(columnDefs.values()));
    List<KuduPartitionScanAssignmentMeta> scansForThisQuery = currentScanner.preparePlanForScanners(translator);
    // Now scan for exact match of counts
    long totalRowsRead = 0;
    unitTestStepwiseScanInputOperator.getBuffer().clear();
    for (KuduPartitionScanAssignmentMeta aSegmentToScan : scansForThisQuery) {
        KuduPartitionScannerCallable<UnitTestTablePojo, InputOperatorControlTuple> threadToScan = new KuduPartitionScannerCallable<>(unitTestStepwiseScanInputOperator, aSegmentToScan, currentScanner.getConnectionPoolForThreads().get(0), unitTestStepwiseScanInputOperator.extractSettersForResultObject(translator), translator);
        totalRowsRead += threadToScan.call();
    }
    // 23  because of the hash distributions and the scan markers. 21 are data records and 2 are end of scan markers
    assertEquals(23L, unitTestStepwiseScanInputOperator.getBuffer().size());
    // revert all configs to default
    partitonScanStrategy = KuduPartitionScanStrategy.ONE_TABLET_PER_OPERATOR;
    numberOfKuduInputOperatorPartitions = 5;
}

Example 5 with SQLToKuduPredicatesTranslator

use of org.apache.apex.malhar.kudu.sqltranslator.SQLToKuduPredicatesTranslator in project apex-malhar by apache.

the class KuduPartitionScannerCallableTest method testSettersForPojo.

@KuduClusterTestContext(kuduClusterBasedTest = true)
@Test
public void testSettersForPojo() throws Exception {
    initOperatorState();
    AbstractKuduPartitionScanner<UnitTestTablePojo, InputOperatorControlTuple> currentScanner = unitTestStepwiseScanInputOperator.getScanner();
    SQLToKuduPredicatesTranslator translator = new SQLToKuduPredicatesTranslator("select introwkey as intColumn from unittests", new ArrayList<ColumnSchema>(columnDefs.values()));
    List<KuduPartitionScanAssignmentMeta> scansForThisQuery = currentScanner.preparePlanForScanners(translator);
    KuduPartitionScannerCallable<UnitTestTablePojo, InputOperatorControlTuple> threadToScan = new KuduPartitionScannerCallable<>(unitTestStepwiseScanInputOperator, scansForThisQuery.get(0), currentScanner.getConnectionPoolForThreads().get(0), unitTestStepwiseScanInputOperator.extractSettersForResultObject(translator), translator);
    long countOfScan = threadToScan.call();
}