Examples with PrimitiveCategory org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory used on opensource projects

Example 46 with PrimitiveCategory

use of org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory in project hive by apache.

the class VectorizationContext method getGenericUDFBridgeVectorExpression.

/**
 * Invoke special handling for expressions that can't be vectorized by regular
 * descriptor based lookup.
 */
private VectorExpression getGenericUDFBridgeVectorExpression(GenericUDFBridge udf, List<ExprNodeDesc> childExpr, VectorExpressionDescriptor.Mode mode, TypeInfo returnType) throws HiveException {
    Class<? extends UDF> cl = udf.getUdfClass();
    VectorExpression ve = null;
    if (isCastToIntFamily(cl)) {
        PrimitiveCategory integerPrimitiveCategory = getAnyIntegerPrimitiveCategoryFromUdfClass(cl);
        ve = getCastToLongExpression(childExpr, integerPrimitiveCategory);
    } else if (cl.equals(UDFToBoolean.class)) {
        ve = getCastToBoolean(childExpr);
    } else if (isCastToFloatFamily(cl)) {
        ve = getCastToDoubleExpression(cl, childExpr, returnType);
    } else if (cl.equals(UDFToString.class)) {
        ve = getCastToString(childExpr, returnType);
    }
    if (ve == null && childExpr instanceof ExprNodeGenericFuncDesc) {
        ve = getCustomUDFExpression((ExprNodeGenericFuncDesc) childExpr, mode);
    }
    return ve;
}

Example 47 with PrimitiveCategory

use of org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory in project hive by apache.

the class MapJoinTestDescription method computeDerived.

public void computeDerived() {
    bigTableColumnNamesList = Arrays.asList(bigTableColumnNames);
    bigTableKeyColumnNames = new String[bigTableKeyColumnNums.length];
    bigTableKeyTypeInfos = new TypeInfo[bigTableKeyColumnNums.length];
    for (int i = 0; i < bigTableKeyColumnNums.length; i++) {
        bigTableKeyColumnNames[i] = bigTableColumnNames[bigTableKeyColumnNums[i]];
        bigTableKeyTypeInfos[i] = bigTableTypeInfos[bigTableKeyColumnNums[i]];
    }
    smallTableValueColumnNamesList = Arrays.asList(smallTableValueColumnNames);
    bigTableObjectInspectors = new ObjectInspector[bigTableTypeInfos.length];
    for (int i = 0; i < bigTableTypeInfos.length; i++) {
        bigTableObjectInspectors[i] = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector((PrimitiveTypeInfo) bigTableTypeInfos[i]);
    }
    bigTableObjectInspectorsList = Arrays.asList(bigTableObjectInspectors);
    smallTableObjectInspectors = new ObjectInspector[smallTableValueTypeInfos.length];
    smallTablePrimitiveCategories = new PrimitiveCategory[smallTableValueTypeInfos.length];
    smallTableValuePrimitiveTypeInfos = new PrimitiveTypeInfo[smallTableValueTypeInfos.length];
    for (int i = 0; i < smallTableValueTypeInfos.length; i++) {
        PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) smallTableValueTypeInfos[i];
        smallTableObjectInspectors[i] = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(primitiveTypeInfo);
        smallTablePrimitiveCategories[i] = primitiveTypeInfo.getPrimitiveCategory();
        smallTableValuePrimitiveTypeInfos[i] = primitiveTypeInfo;
    }
    smallTableObjectInspectorsList = Arrays.asList(smallTableObjectInspectors);
    bigTableStandardObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(bigTableColumnNamesList, Arrays.asList((ObjectInspector[]) bigTableObjectInspectors));
    smallTableStandardObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(smallTableValueColumnNamesList, Arrays.asList((ObjectInspector[]) smallTableObjectInspectors));
    inputObjectInspectors = new ObjectInspector[] { bigTableStandardObjectInspector, smallTableStandardObjectInspector };
    int outputLength = bigTableRetainColumnNums.length + smallTableRetainKeyColumnNums.length + smallTableRetainValueColumnNums.length;
    outputColumnNames = createOutputColumnNames(outputLength);
    outputTypeInfos = new TypeInfo[outputLength];
    int outputIndex = 0;
    for (int i = 0; i < bigTableRetainColumnNums.length; i++) {
        outputTypeInfos[outputIndex++] = bigTableTypeInfos[bigTableRetainColumnNums[i]];
    }
    // }
    for (int i = 0; i < smallTableRetainValueColumnNums.length; i++) {
        outputTypeInfos[outputIndex++] = smallTableValueTypeInfos[smallTableRetainValueColumnNums[i]];
    }
    outputObjectInspectors = new ObjectInspector[outputLength];
    for (int i = 0; i < outputLength; i++) {
        PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) outputTypeInfos[i];
        outputObjectInspectors[i] = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(primitiveTypeInfo);
    }
}

Example 48 with PrimitiveCategory

use of org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory in project hive by apache.

the class TestVectorMapJoinFastRowHashMap method addAndVerifyRows.

private void addAndVerifyRows(VectorRandomRowSource valueSource, Object[][] rows, VectorMapJoinFastHashTable map, HashTableKeyType hashTableKeyType, VerifyFastRowHashMap verifyTable, String[] keyTypeNames, boolean doClipping, boolean useExactBytes) throws HiveException, IOException, SerDeException {
    final int keyCount = keyTypeNames.length;
    PrimitiveTypeInfo[] keyPrimitiveTypeInfos = new PrimitiveTypeInfo[keyCount];
    PrimitiveCategory[] keyPrimitiveCategories = new PrimitiveCategory[keyCount];
    ArrayList<ObjectInspector> keyPrimitiveObjectInspectorList = new ArrayList<ObjectInspector>(keyCount);
    for (int i = 0; i < keyCount; i++) {
        PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromTypeString(keyTypeNames[i]);
        keyPrimitiveTypeInfos[i] = primitiveTypeInfo;
        PrimitiveCategory primitiveCategory = primitiveTypeInfo.getPrimitiveCategory();
        keyPrimitiveCategories[i] = primitiveCategory;
        keyPrimitiveObjectInspectorList.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(primitiveTypeInfo));
    }
    boolean[] keyColumnSortOrderIsDesc = new boolean[keyCount];
    Arrays.fill(keyColumnSortOrderIsDesc, false);
    byte[] keyColumnNullMarker = new byte[keyCount];
    Arrays.fill(keyColumnNullMarker, BinarySortableSerDe.ZERO);
    byte[] keyColumnNotNullMarker = new byte[keyCount];
    Arrays.fill(keyColumnNotNullMarker, BinarySortableSerDe.ONE);
    BinarySortableSerializeWrite keySerializeWrite = new BinarySortableSerializeWrite(keyColumnSortOrderIsDesc, keyColumnNullMarker, keyColumnNotNullMarker);
    TypeInfo[] valueTypeInfos = valueSource.typeInfos();
    final int columnCount = valueTypeInfos.length;
    SerializeWrite valueSerializeWrite = new LazyBinarySerializeWrite(columnCount);
    final int count = rows.length;
    for (int i = 0; i < count; i++) {
        Object[] valueRow = rows[i];
        Output valueOutput = new Output();
        ((LazyBinarySerializeWrite) valueSerializeWrite).set(valueOutput);
        for (int index = 0; index < columnCount; index++) {
            VerifyFastRow.serializeWrite(valueSerializeWrite, valueTypeInfos[index], valueRow[index]);
        }
        byte[] value = Arrays.copyOf(valueOutput.getData(), valueOutput.getLength());
        // Add a new key or add a value to an existing key?
        byte[] key;
        if (random.nextBoolean() || verifyTable.getCount() == 0) {
            Object[] keyRow = VectorRandomRowSource.randomWritablePrimitiveRow(keyCount, random, keyPrimitiveTypeInfos);
            Output keyOutput = new Output();
            keySerializeWrite.set(keyOutput);
            for (int index = 0; index < keyCount; index++) {
                VerifyFastRow.serializeWrite(keySerializeWrite, keyPrimitiveTypeInfos[index], keyRow[index]);
            }
            key = Arrays.copyOf(keyOutput.getData(), keyOutput.getLength());
            verifyTable.add(key, keyRow, value, valueRow);
        } else {
            key = verifyTable.addRandomExisting(value, valueRow, random);
        }
        // Serialize keyRow into key bytes.
        BytesWritable keyWritable = new BytesWritable(key);
        BytesWritable valueWritable = new BytesWritable(value);
        map.putRow(keyWritable, valueWritable);
    // verifyTable.verify(map);
    }
    verifyTable.verify(map, hashTableKeyType, valueTypeInfos, doClipping, useExactBytes, random);
}

Example 49 with PrimitiveCategory

use of org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory in project hive by apache.

the class TestVectorGenericDateExpressions method testDateAddColScalar.

@Test
public void testDateAddColScalar() throws HiveException {
    for (PrimitiveCategory colType1 : dateTimestampStringTypes) testDateAddColScalar(colType1, true);
    VectorExpression udf = new VectorUDFDateAddColScalar(0, 0, 1);
    udf.setInputTypeInfos(new TypeInfo[] { TypeInfoFactory.stringTypeInfo, TypeInfoFactory.timestampTypeInfo });
    udf.transientInit();
    VectorizedRowBatch batch = new VectorizedRowBatch(2, 1);
    batch.cols[0] = new BytesColumnVector(1);
    batch.cols[1] = new LongColumnVector(1);
    BytesColumnVector bcv = (BytesColumnVector) batch.cols[0];
    byte[] bytes = "error".getBytes(utf8);
    bcv.vector[0] = bytes;
    bcv.start[0] = 0;
    bcv.length[0] = bytes.length;
    udf.evaluate(batch);
    Assert.assertEquals(batch.cols[1].isNull[0], true);
}

Example 50 with PrimitiveCategory

use of org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory in project hive by apache.

the class TestVectorGenericDateExpressions method testDateDiffColScalar.

@Test
public void testDateDiffColScalar() throws HiveException {
    for (PrimitiveCategory colType1 : dateTimestampStringTypes) {
        for (PrimitiveCategory scalarType2 : dateTimestampStringTypes) {
            LongColumnVector date1 = newRandomLongColumnVector(10000, size);
            LongColumnVector output = new LongColumnVector(size);
            VectorizedRowBatch batch = new VectorizedRowBatch(2, size);
            batch.cols[0] = castTo(date1, colType1);
            batch.cols[1] = output;
            long scalar2 = newRandom(1000);
            validateDateDiff(batch, date1, scalar2, colType1, scalarType2);
            TestVectorizedRowBatch.addRandomNulls(date1);
            batch.cols[0] = castTo(date1, colType1);
            validateDateDiff(batch, date1, scalar2, colType1, scalarType2);
        }
    }
    VectorExpression udf;
    byte[] bytes = "error".getBytes(utf8);
    VectorizedRowBatch batch = new VectorizedRowBatch(2, 1);
    udf = new VectorUDFDateDiffColScalar(0, 0L, 1);
    udf.setInputTypeInfos(new TypeInfo[] { TypeInfoFactory.timestampTypeInfo, TypeInfoFactory.stringTypeInfo });
    batch.cols[0] = new BytesColumnVector(1);
    batch.cols[1] = new LongColumnVector(1);
    BytesColumnVector bcv = (BytesColumnVector) batch.cols[0];
    bcv.vector[0] = bytes;
    bcv.start[0] = 0;
    bcv.length[0] = bytes.length;
    udf.evaluate(batch);
    Assert.assertEquals(batch.cols[1].isNull[0], true);
    udf = new VectorUDFDateDiffColScalar(0, bytes, 1);
    udf.setInputTypeInfos(new TypeInfo[] { TypeInfoFactory.timestampTypeInfo, TypeInfoFactory.stringTypeInfo });
    udf.transientInit();
    batch.cols[0] = new LongColumnVector(1);
    batch.cols[1] = new LongColumnVector(1);
    udf.evaluate(batch);
    Assert.assertEquals(batch.cols[1].isNull[0], true);
}