Examples with QueryMeasure org.apache.carbondata.core.scan.model.QueryMeasure used on opensource projects

Example 6 with QueryMeasure

use of org.apache.carbondata.core.scan.model.QueryMeasure in project carbondata by apache.

the class CarbonInputFormatUtil method addQueryMeasure.

private static void addQueryMeasure(CarbonQueryPlan plan, int order, CarbonMeasure measure) {
    QueryMeasure queryMeasure = new QueryMeasure(measure.getColName());
    queryMeasure.setQueryOrder(order);
    queryMeasure.setMeasure(measure);
    plan.addMeasure(queryMeasure);
}

Example 7 with QueryMeasure

use of org.apache.carbondata.core.scan.model.QueryMeasure in project carbondata by apache.

the class RestructureUtilTest method testToGetAggregatorInfos.

@Test
public void testToGetAggregatorInfos() {
    ColumnSchema columnSchema1 = new ColumnSchema();
    columnSchema1.setColumnName("Id");
    columnSchema1.setDataType(DataType.STRING);
    columnSchema1.setColumnUniqueId(UUID.randomUUID().toString());
    ColumnSchema columnSchema2 = new ColumnSchema();
    columnSchema2.setColumnName("Name");
    columnSchema2.setDataType(DataType.STRING);
    columnSchema2.setColumnUniqueId(UUID.randomUUID().toString());
    ColumnSchema columnSchema3 = new ColumnSchema();
    columnSchema3.setColumnName("Age");
    columnSchema3.setDataType(DataType.STRING);
    columnSchema3.setColumnUniqueId(UUID.randomUUID().toString());
    CarbonMeasure carbonMeasure1 = new CarbonMeasure(columnSchema1, 1);
    CarbonMeasure carbonMeasure2 = new CarbonMeasure(columnSchema2, 2);
    CarbonMeasure carbonMeasure3 = new CarbonMeasure(columnSchema3, 3);
    carbonMeasure3.getColumnSchema().setDefaultValue("3".getBytes());
    List<CarbonMeasure> currentBlockMeasures = Arrays.asList(carbonMeasure1, carbonMeasure2);
    QueryMeasure queryMeasure1 = new QueryMeasure("Id");
    queryMeasure1.setMeasure(carbonMeasure1);
    QueryMeasure queryMeasure2 = new QueryMeasure("Name");
    queryMeasure2.setMeasure(carbonMeasure2);
    QueryMeasure queryMeasure3 = new QueryMeasure("Age");
    queryMeasure3.setMeasure(carbonMeasure3);
    List<QueryMeasure> queryMeasures = Arrays.asList(queryMeasure1, queryMeasure2, queryMeasure3);
    BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
    RestructureUtil.createMeasureInfoAndGetCurrentBlockQueryMeasures(blockExecutionInfo, queryMeasures, currentBlockMeasures);
    MeasureInfo measureInfo = blockExecutionInfo.getMeasureInfo();
    boolean[] measuresExist = { true, true, false };
    assertThat(measureInfo.getMeasureExists(), is(equalTo(measuresExist)));
    Object[] defaultValues = { null, null, 3.0 };
    assertThat(measureInfo.getDefaultValues(), is(equalTo(defaultValues)));
}

Example 8 with QueryMeasure

use of org.apache.carbondata.core.scan.model.QueryMeasure in project carbondata by apache.

the class VectorizedCarbonRecordReader method initBatch.

/**
   * Returns the ColumnarBatch object that will be used for all rows returned by this reader.
   * This object is reused. Calling this enables the vectorized reader. This should be called
   * before any calls to nextKeyValue/nextBatch.
   */
private void initBatch(MemoryMode memMode) {
    List<QueryDimension> queryDimension = queryModel.getQueryDimension();
    List<QueryMeasure> queryMeasures = queryModel.getQueryMeasures();
    StructField[] fields = new StructField[queryDimension.size() + queryMeasures.size()];
    for (int i = 0; i < queryDimension.size(); i++) {
        QueryDimension dim = queryDimension.get(i);
        if (dim.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
            DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(dim.getDimension().getDataType());
            fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(generator.getReturnType()), true, null);
        } else if (!dim.getDimension().hasEncoding(Encoding.DICTIONARY)) {
            fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true, null);
        } else if (dim.getDimension().isComplex()) {
            fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true, null);
        } else {
            fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(DataType.INT), true, null);
        }
    }
    for (int i = 0; i < queryMeasures.size(); i++) {
        QueryMeasure msr = queryMeasures.get(i);
        switch(msr.getMeasure().getDataType()) {
            case SHORT:
            case INT:
            case LONG:
                fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(msr.getMeasure().getDataType()), true, null);
                break;
            case DECIMAL:
                fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(), new DecimalType(msr.getMeasure().getPrecision(), msr.getMeasure().getScale()), true, null);
                break;
            default:
                fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(DataType.DOUBLE), true, null);
        }
    }
    columnarBatch = ColumnarBatch.allocate(new StructType(fields), memMode);
    CarbonColumnVector[] vectors = new CarbonColumnVector[fields.length];
    boolean[] filteredRows = new boolean[columnarBatch.capacity()];
    for (int i = 0; i < fields.length; i++) {
        vectors[i] = new ColumnarVectorWrapper(columnarBatch.column(i), filteredRows);
    }
    carbonColumnarBatch = new CarbonColumnarBatch(vectors, columnarBatch.capacity(), filteredRows);
}

Example 9 with QueryMeasure

use of org.apache.carbondata.core.scan.model.QueryMeasure in project carbondata by apache.

the class CarbonCompactionExecutor method prepareQueryModel.

/**
   * Preparing of the query model.
   *
   * @param blockList
   * @return
   */
private QueryModel prepareQueryModel(List<TableBlockInfo> blockList) {
    QueryModel model = new QueryModel();
    model.setTableBlockInfos(blockList);
    model.setForcedDetailRawQuery(true);
    model.setFilterExpressionResolverTree(null);
    List<QueryDimension> dims = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(carbonTable.getFactTableName());
    for (CarbonDimension dim : dimensions) {
        // check if dimension is deleted
        QueryDimension queryDimension = new QueryDimension(dim.getColName());
        queryDimension.setDimension(dim);
        dims.add(queryDimension);
    }
    model.setQueryDimension(dims);
    List<QueryMeasure> msrs = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(carbonTable.getFactTableName());
    for (CarbonMeasure carbonMeasure : measures) {
        // check if measure is deleted
        QueryMeasure queryMeasure = new QueryMeasure(carbonMeasure.getColName());
        queryMeasure.setMeasure(carbonMeasure);
        msrs.add(queryMeasure);
    }
    model.setQueryMeasures(msrs);
    model.setQueryId(System.nanoTime() + "");
    model.setAbsoluteTableIdentifier(carbonTable.getAbsoluteTableIdentifier());
    model.setTable(carbonTable);
    return model;
}

Example 10 with QueryMeasure

use of org.apache.carbondata.core.scan.model.QueryMeasure in project carbondata by apache.

the class RestructureUtil method createMeasureInfoAndGetCurrentBlockQueryMeasures.

/**
   * Below method will be used to prepare the measure info object
   * in this method some of the properties which will be extracted
   * from query measure and current block measures will be set
   *
   * @param blockExecutionInfo
   * @param queryMeasures        measures present in query
   * @param currentBlockMeasures current block measures
   * @return measures present in the block
   */
public static List<QueryMeasure> createMeasureInfoAndGetCurrentBlockQueryMeasures(BlockExecutionInfo blockExecutionInfo, List<QueryMeasure> queryMeasures, List<CarbonMeasure> currentBlockMeasures) {
    MeasureInfo measureInfo = new MeasureInfo();
    List<QueryMeasure> presentMeasure = new ArrayList<>(queryMeasures.size());
    int numberOfMeasureInQuery = queryMeasures.size();
    List<Integer> measureOrdinalList = new ArrayList<>(numberOfMeasureInQuery);
    Object[] defaultValues = new Object[numberOfMeasureInQuery];
    boolean[] measureExistsInCurrentBlock = new boolean[numberOfMeasureInQuery];
    int index = 0;
    for (QueryMeasure queryMeasure : queryMeasures) {
        // otherwise adding a default value of a measure
        for (CarbonMeasure carbonMeasure : currentBlockMeasures) {
            if (carbonMeasure.getColumnId().equals(queryMeasure.getMeasure().getColumnId())) {
                QueryMeasure currentBlockMeasure = new QueryMeasure(carbonMeasure.getColName());
                carbonMeasure.getColumnSchema().setDataType(queryMeasure.getMeasure().getDataType());
                carbonMeasure.getColumnSchema().setPrecision(queryMeasure.getMeasure().getPrecision());
                carbonMeasure.getColumnSchema().setScale(queryMeasure.getMeasure().getScale());
                carbonMeasure.getColumnSchema().setDefaultValue(queryMeasure.getMeasure().getDefaultValue());
                currentBlockMeasure.setMeasure(carbonMeasure);
                currentBlockMeasure.setQueryOrder(queryMeasure.getQueryOrder());
                presentMeasure.add(currentBlockMeasure);
                measureOrdinalList.add(carbonMeasure.getOrdinal());
                measureExistsInCurrentBlock[index] = true;
                break;
            }
        }
        if (!measureExistsInCurrentBlock[index]) {
            defaultValues[index] = getMeasureDefaultValue(queryMeasure.getMeasure().getColumnSchema(), queryMeasure.getMeasure().getDefaultValue());
            blockExecutionInfo.setRestructuredBlock(true);
        }
        index++;
    }
    int[] measureOrdinals = ArrayUtils.toPrimitive(measureOrdinalList.toArray(new Integer[measureOrdinalList.size()]));
    measureInfo.setDefaultValues(defaultValues);
    measureInfo.setMeasureOrdinals(measureOrdinals);
    measureInfo.setMeasureExists(measureExistsInCurrentBlock);
    blockExecutionInfo.setMeasureInfo(measureInfo);
    return presentMeasure;
}