Examples with TypeInfoFactory.shortTypeInfo org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.shortTypeInfo used on opensource projects

Example 1 with TypeInfoFactory.shortTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.shortTypeInfo in project hive by apache.

the class FullOuterMapJoinOptimization method removeFilterMap.

public static void removeFilterMap(MapJoinDesc mapJoinDesc) throws SemanticException {
    int[][] filterMaps = mapJoinDesc.getFilterMap();
    if (filterMaps == null) {
        return;
    }
    final byte posBigTable = (byte) mapJoinDesc.getPosBigTable();
    final int numAliases = mapJoinDesc.getExprs().size();
    List<TableDesc> valueFilteredTblDescs = mapJoinDesc.getValueFilteredTblDescs();
    for (byte pos = 0; pos < numAliases; pos++) {
        if (pos != posBigTable) {
            int[] filterMap = filterMaps[pos];
            TableDesc tableDesc = valueFilteredTblDescs.get(pos);
            Properties properties = tableDesc.getProperties();
            String columnNameProperty = properties.getProperty(serdeConstants.LIST_COLUMNS);
            String columnNameDelimiter = properties.containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ? properties.getProperty(serdeConstants.COLUMN_NAME_DELIMITER) : String.valueOf(SerDeUtils.COMMA);
            String columnTypeProperty = properties.getProperty(serdeConstants.LIST_COLUMN_TYPES);
            List<String> columnNameList;
            if (columnNameProperty.length() == 0) {
                columnNameList = new ArrayList<String>();
            } else {
                columnNameList = Arrays.asList(columnNameProperty.split(columnNameDelimiter));
            }
            List<String> truncatedColumnNameList = columnNameList.subList(0, columnNameList.size() - 1);
            String truncatedColumnNameProperty = Joiner.on(columnNameDelimiter).join(truncatedColumnNameList);
            List<TypeInfo> columnTypeList;
            if (columnTypeProperty.length() == 0) {
                columnTypeList = new ArrayList<TypeInfo>();
            } else {
                columnTypeList = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
            }
            if (!columnTypeList.get(columnTypeList.size() - 1).equals(TypeInfoFactory.shortTypeInfo)) {
                throw new SemanticException("Expecting filterTag smallint as last column type");
            }
            List<TypeInfo> truncatedColumnTypeList = columnTypeList.subList(0, columnTypeList.size() - 1);
            String truncatedColumnTypeProperty = Joiner.on(",").join(truncatedColumnTypeList);
            properties.setProperty(serdeConstants.LIST_COLUMNS, truncatedColumnNameProperty);
            properties.setProperty(serdeConstants.LIST_COLUMN_TYPES, truncatedColumnTypeProperty);
        }
    }
    mapJoinDesc.setFilterMap(null);
}

Example 2 with TypeInfoFactory.shortTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.shortTypeInfo in project hive by apache.

the class TestGenericUDFOPPlus method testBytePlusShort.

@Test
public void testBytePlusShort() throws HiveException {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();
    // Byte
    ByteWritable left = new ByteWritable((byte) 4);
    ShortWritable right = new ShortWritable((short) 6);
    ObjectInspector[] inputOIs = { PrimitiveObjectInspectorFactory.writableByteObjectInspector, PrimitiveObjectInspectorFactory.writableShortObjectInspector };
    DeferredObject[] args = { new DeferredJavaObject(left), new DeferredJavaObject(right) };
    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(oi.getTypeInfo(), TypeInfoFactory.shortTypeInfo);
    ShortWritable res = (ShortWritable) udf.evaluate(args);
    Assert.assertEquals(10, res.get());
}

Example 3 with TypeInfoFactory.shortTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.shortTypeInfo in project hive by apache.

the class TestGenericUDFOPPositive method testShort.

@Test
public void testShort() throws HiveException {
    GenericUDFOPPositive udf = new GenericUDFOPPositive();
    ShortWritable input = new ShortWritable((short) 74);
    ObjectInspector[] inputOIs = { PrimitiveObjectInspectorFactory.writableShortObjectInspector };
    DeferredObject[] args = { new DeferredJavaObject(input) };
    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.shortTypeInfo, oi.getTypeInfo());
    ShortWritable res = (ShortWritable) udf.evaluate(args);
    Assert.assertEquals((short) 74, res.get());
}

Example 4 with TypeInfoFactory.shortTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.shortTypeInfo in project hive by apache.

the class TestVectorAggregation method doTests.

private void doTests(Random random, String aggregationName, TypeInfo typeInfo, boolean isCountStar, boolean tryDecimal64) throws Exception {
    List<GenerationSpec> dataAggrGenerationSpecList = new ArrayList<GenerationSpec>();
    List<DataTypePhysicalVariation> explicitDataTypePhysicalVariationList = new ArrayList<DataTypePhysicalVariation>();
    TypeInfo keyTypeInfo = TypeInfoFactory.shortTypeInfo;
    GenerationSpec keyGenerationSpec = GenerationSpec.createOmitGeneration(keyTypeInfo);
    dataAggrGenerationSpecList.add(keyGenerationSpec);
    explicitDataTypePhysicalVariationList.add(DataTypePhysicalVariation.NONE);
    final boolean decimal64Enable = checkDecimal64(tryDecimal64, typeInfo);
    GenerationSpec generationSpec = GenerationSpec.createSameType(typeInfo);
    dataAggrGenerationSpecList.add(generationSpec);
    explicitDataTypePhysicalVariationList.add(decimal64Enable ? DataTypePhysicalVariation.DECIMAL_64 : DataTypePhysicalVariation.NONE);
    List<String> columns = new ArrayList<String>();
    columns.add("col0");
    columns.add("col1");
    ExprNodeColumnDesc dataAggrCol1Expr = new ExprNodeColumnDesc(typeInfo, "col1", "table", false);
    List<ExprNodeDesc> dataAggrParameters = new ArrayList<ExprNodeDesc>();
    if (!isCountStar) {
        dataAggrParameters.add(dataAggrCol1Expr);
    }
    final int dataAggrParameterCount = dataAggrParameters.size();
    ObjectInspector[] dataAggrParameterObjectInspectors = new ObjectInspector[dataAggrParameterCount];
    for (int i = 0; i < dataAggrParameterCount; i++) {
        TypeInfo paramTypeInfo = dataAggrParameters.get(i).getTypeInfo();
        dataAggrParameterObjectInspectors[i] = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(paramTypeInfo);
    }
    String[] columnNames = columns.toArray(new String[0]);
    final int dataAggrMaxKeyCount = 20000;
    final int reductionFactor = 16;
    ObjectInspector keyObjectInspector = VectorRandomRowSource.getObjectInspector(keyTypeInfo);
    /*
     * PARTIAL1.
     */
    VectorRandomRowSource partial1RowSource = new VectorRandomRowSource();
    boolean allowNull = !aggregationName.equals("bloom_filter");
    partial1RowSource.initGenerationSpecSchema(random, dataAggrGenerationSpecList, /* maxComplexDepth */
    0, allowNull, /* isUnicodeOk */
    true, explicitDataTypePhysicalVariationList);
    Object[][] partial1RandomRows = partial1RowSource.randomRows(TEST_ROW_COUNT);
    final int partial1RowCount = partial1RandomRows.length;
    for (int i = 0; i < partial1RowCount; i++) {
        final short shortKey = (short) getLinearRandomNumber(random, dataAggrMaxKeyCount);
        partial1RandomRows[i][0] = ((WritableShortObjectInspector) keyObjectInspector).create((short) shortKey);
    }
    VectorRandomBatchSource partial1BatchSource = VectorRandomBatchSource.createInterestingBatches(random, partial1RowSource, partial1RandomRows, null);
    GenericUDAFEvaluator partial1Evaluator = getEvaluator(aggregationName, typeInfo);
    if (isCountStar) {
        Assert.assertTrue(partial1Evaluator instanceof GenericUDAFCountEvaluator);
        GenericUDAFCountEvaluator countEvaluator = (GenericUDAFCountEvaluator) partial1Evaluator;
        countEvaluator.setCountAllColumns(true);
    }
    /*
    System.out.println(
        "*DEBUG* GenericUDAFEvaluator for " + aggregationName + ", " + typeInfo.getTypeName() + ": " +
            partial1Evaluator.getClass().getSimpleName());
    */
    // The only way to get the return object inspector (and its return type) is to
    // initialize it...
    final GenericUDAFEvaluator.Mode partial1UdafEvaluatorMode = GenericUDAFEvaluator.Mode.PARTIAL1;
    ObjectInspector partial1ReturnOI = partial1Evaluator.init(partial1UdafEvaluatorMode, dataAggrParameterObjectInspectors);
    TypeInfo partial1OutputTypeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(partial1ReturnOI);
    Object[] partial1ResultsArray = new Object[AggregationTestMode.count];
    executeAggregationTests(aggregationName, typeInfo, partial1Evaluator, partial1OutputTypeInfo, partial1UdafEvaluatorMode, dataAggrMaxKeyCount, columns, columnNames, dataAggrParameters, partial1RandomRows, partial1RowSource, partial1BatchSource, tryDecimal64, partial1ResultsArray);
    verifyAggregationResults(typeInfo, partial1OutputTypeInfo, dataAggrMaxKeyCount, partial1UdafEvaluatorMode, partial1ResultsArray);
    final boolean hasDifferentCompleteExpr;
    if (varianceNames.contains(aggregationName)) {
        hasDifferentCompleteExpr = true;
    } else {
        switch(aggregationName) {
            case "avg":
                hasDifferentCompleteExpr = true;
                break;
            case "bloom_filter":
            case "count":
            case "max":
            case "min":
            case "sum":
                hasDifferentCompleteExpr = false;
                break;
            default:
                throw new RuntimeException("Unexpected aggregation name " + aggregationName);
        }
    }
    if (hasDifferentCompleteExpr) {
        /*
       * COMPLETE.
       */
        VectorRandomRowSource completeRowSource = new VectorRandomRowSource();
        completeRowSource.initGenerationSpecSchema(random, dataAggrGenerationSpecList, /* maxComplexDepth */
        0, /* allowNull */
        true, /* isUnicodeOk */
        true, explicitDataTypePhysicalVariationList);
        Object[][] completeRandomRows = completeRowSource.randomRows(TEST_ROW_COUNT);
        final int completeRowCount = completeRandomRows.length;
        for (int i = 0; i < completeRowCount; i++) {
            final short shortKey = (short) getLinearRandomNumber(random, dataAggrMaxKeyCount);
            completeRandomRows[i][0] = ((WritableShortObjectInspector) keyObjectInspector).create((short) shortKey);
        }
        VectorRandomBatchSource completeBatchSource = VectorRandomBatchSource.createInterestingBatches(random, completeRowSource, completeRandomRows, null);
        GenericUDAFEvaluator completeEvaluator = getEvaluator(aggregationName, typeInfo);
        /*
      System.out.println(
          "*DEBUG* GenericUDAFEvaluator for " + aggregationName + ", " + typeInfo.getTypeName() + ": " +
              completeEvaluator.getClass().getSimpleName());
      */
        // The only way to get the return object inspector (and its return type) is to
        // initialize it...
        final GenericUDAFEvaluator.Mode completeUdafEvaluatorMode = GenericUDAFEvaluator.Mode.COMPLETE;
        ObjectInspector completeReturnOI = completeEvaluator.init(completeUdafEvaluatorMode, dataAggrParameterObjectInspectors);
        TypeInfo completeOutputTypeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(completeReturnOI);
        Object[] completeResultsArray = new Object[AggregationTestMode.count];
        executeAggregationTests(aggregationName, typeInfo, completeEvaluator, completeOutputTypeInfo, completeUdafEvaluatorMode, dataAggrMaxKeyCount, columns, columnNames, dataAggrParameters, completeRandomRows, completeRowSource, completeBatchSource, tryDecimal64, completeResultsArray);
        verifyAggregationResults(typeInfo, completeOutputTypeInfo, dataAggrMaxKeyCount, completeUdafEvaluatorMode, completeResultsArray);
    }
    final boolean hasDifferentPartial2Expr;
    if (varianceNames.contains(aggregationName)) {
        hasDifferentPartial2Expr = true;
    } else {
        switch(aggregationName) {
            case "avg":
                hasDifferentPartial2Expr = true;
                break;
            case "bloom_filter":
            case "count":
            case "max":
            case "min":
            case "sum":
                hasDifferentPartial2Expr = false;
                break;
            default:
                throw new RuntimeException("Unexpected aggregation name " + aggregationName);
        }
    }
    if (hasDifferentPartial2Expr) {
        /*
       * PARTIAL2.
       */
        final GenericUDAFEvaluator.Mode mergeUdafEvaluatorMode = GenericUDAFEvaluator.Mode.PARTIAL2;
        doMerge(mergeUdafEvaluatorMode, random, aggregationName, typeInfo, keyGenerationSpec, columns, columnNames, dataAggrMaxKeyCount, reductionFactor, partial1OutputTypeInfo, partial1ResultsArray);
    }
    final boolean hasDifferentFinalExpr;
    if (varianceNames.contains(aggregationName)) {
        hasDifferentFinalExpr = true;
    } else {
        switch(aggregationName) {
            case "avg":
                hasDifferentFinalExpr = true;
                break;
            case "bloom_filter":
            case "count":
                hasDifferentFinalExpr = true;
                break;
            case "max":
            case "min":
            case "sum":
                hasDifferentFinalExpr = false;
                break;
            default:
                throw new RuntimeException("Unexpected aggregation name " + aggregationName);
        }
    }
    if (hasDifferentFinalExpr) {
        /*
       * FINAL.
       */
        final GenericUDAFEvaluator.Mode mergeUdafEvaluatorMode = GenericUDAFEvaluator.Mode.FINAL;
        doMerge(mergeUdafEvaluatorMode, random, aggregationName, typeInfo, keyGenerationSpec, columns, columnNames, dataAggrMaxKeyCount, reductionFactor, partial1OutputTypeInfo, partial1ResultsArray);
    }
}

Example 5 with TypeInfoFactory.shortTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.shortTypeInfo in project hive by apache.

the class TestMapJoinOperator method doTestLong2.

public boolean doTestLong2(long seed, int rowCount, int hiveConfVariation, VectorMapJoinVariation vectorMapJoinVariation, MapJoinPlanVariation mapJoinPlanVariation) throws Exception {
    HiveConf hiveConf = new HiveConf();
    if (!addLongHiveConfVariation(hiveConfVariation, hiveConf)) {
        return true;
    }
    TypeInfo[] bigTableTypeInfos = null;
    int[] bigTableKeyColumnNums = null;
    TypeInfo[] smallTableValueTypeInfos = null;
    int[] smallTableRetainKeyColumnNums = null;
    SmallTableGenerationParameters smallTableGenerationParameters = new SmallTableGenerationParameters();
    MapJoinTestDescription testDesc = null;
    MapJoinTestData testData = null;
    // Big Table: short key, no value; Small Table: key retained, timestamp value
    bigTableTypeInfos = new TypeInfo[] { TypeInfoFactory.shortTypeInfo };
    bigTableKeyColumnNums = new int[] { 0 };
    smallTableRetainKeyColumnNums = new int[] { 0 };
    smallTableValueTypeInfos = new TypeInfo[] { TypeInfoFactory.timestampTypeInfo };
    testDesc = new MapJoinTestDescription(hiveConf, vectorMapJoinVariation, bigTableTypeInfos, bigTableKeyColumnNums, smallTableValueTypeInfos, smallTableRetainKeyColumnNums, smallTableGenerationParameters, mapJoinPlanVariation);
    if (!goodTestVariation(testDesc)) {
        return false;
    }
    testData = new MapJoinTestData(rowCount, testDesc, seed);
    executeTest(testDesc, testData, "testLong2");
    return false;
}