Examples with HiveShim org.apache.flink.table.catalog.hive.client.HiveShim used on opensource projects

Example 11 with HiveShim

use of org.apache.flink.table.catalog.hive.client.HiveShim in project flink by apache.

the class HiveSimpleUDF method openInternal.

@Override
public void openInternal() {
    LOG.info("Opening HiveSimpleUDF as '{}'", hiveFunctionWrapper.getClassName());
    function = hiveFunctionWrapper.createFunction();
    List<TypeInfo> typeInfos = new ArrayList<>();
    for (DataType arg : argTypes) {
        typeInfos.add(HiveTypeUtil.toHiveTypeInfo(arg, false));
    }
    try {
        method = function.getResolver().getEvalMethod(typeInfos);
        returnInspector = ObjectInspectorFactory.getReflectionObjectInspector(method.getGenericReturnType(), ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
        ObjectInspector[] argInspectors = new ObjectInspector[typeInfos.size()];
        for (int i = 0; i < argTypes.length; i++) {
            argInspectors[i] = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfos.get(i));
        }
        conversionHelper = new GenericUDFUtils.ConversionHelper(method, argInspectors);
        conversions = new HiveObjectConversion[argInspectors.length];
        for (int i = 0; i < argInspectors.length; i++) {
            conversions[i] = HiveInspectors.getConversion(argInspectors[i], argTypes[i].getLogicalType(), hiveShim);
        }
        allIdentityConverter = Arrays.stream(conversions).allMatch(conv -> conv instanceof IdentityConversion);
    } catch (Exception e) {
        throw new FlinkHiveUDFException(String.format("Failed to open HiveSimpleUDF from %s", hiveFunctionWrapper.getClassName()), e);
    }
}

Example 12 with HiveShim

use of org.apache.flink.table.catalog.hive.client.HiveShim in project flink by apache.

the class HiveTablePartition method ofPartition.

/**
 * Creates a HiveTablePartition to represent a hive partition.
 *
 * @param hiveConf the HiveConf used to connect to HMS
 * @param hiveVersion the version of hive in use, if it's null the version will be automatically
 *     detected
 * @param dbName name of the database
 * @param tableName name of the table
 * @param partitionSpec map from each partition column to its value. The map should contain
 *     exactly all the partition columns and in the order in which the partition columns are
 *     defined
 */
public static HiveTablePartition ofPartition(HiveConf hiveConf, @Nullable String hiveVersion, String dbName, String tableName, LinkedHashMap<String, String> partitionSpec) {
    HiveShim hiveShim = getHiveShim(hiveVersion);
    try (HiveMetastoreClientWrapper client = new HiveMetastoreClientWrapper(hiveConf, hiveShim)) {
        Table hiveTable = client.getTable(dbName, tableName);
        Partition hivePartition = client.getPartition(dbName, tableName, new ArrayList<>(partitionSpec.values()));
        return new HiveTablePartition(hivePartition.getSd(), partitionSpec, HiveReflectionUtils.getTableMetadata(hiveShim, hiveTable));
    } catch (TException e) {
        throw new FlinkHiveException(String.format("Failed to create HiveTablePartition for partition %s of hive table %s.%s", partitionSpec, dbName, tableName), e);
    }
}

Example 13 with HiveShim

use of org.apache.flink.table.catalog.hive.client.HiveShim in project flink by apache.

the class HiveStatsUtil method createTableColumnStats.

/**
 * Create Flink ColumnStats from Hive ColumnStatisticsData.
 */
private static CatalogColumnStatisticsDataBase createTableColumnStats(DataType colType, ColumnStatisticsData stats, String hiveVersion) {
    HiveShim hiveShim = HiveShimLoader.loadHiveShim(hiveVersion);
    if (stats.isSetBinaryStats()) {
        BinaryColumnStatsData binaryStats = stats.getBinaryStats();
        return new CatalogColumnStatisticsDataBinary(binaryStats.isSetMaxColLen() ? binaryStats.getMaxColLen() : null, binaryStats.isSetAvgColLen() ? binaryStats.getAvgColLen() : null, binaryStats.isSetNumNulls() ? binaryStats.getNumNulls() : null);
    } else if (stats.isSetBooleanStats()) {
        BooleanColumnStatsData booleanStats = stats.getBooleanStats();
        return new CatalogColumnStatisticsDataBoolean(booleanStats.isSetNumTrues() ? booleanStats.getNumTrues() : null, booleanStats.isSetNumFalses() ? booleanStats.getNumFalses() : null, booleanStats.isSetNumNulls() ? booleanStats.getNumNulls() : null);
    } else if (hiveShim.isDateStats(stats)) {
        return hiveShim.toFlinkDateColStats(stats);
    } else if (stats.isSetDoubleStats()) {
        DoubleColumnStatsData doubleStats = stats.getDoubleStats();
        return new CatalogColumnStatisticsDataDouble(doubleStats.isSetLowValue() ? doubleStats.getLowValue() : null, doubleStats.isSetHighValue() ? doubleStats.getHighValue() : null, doubleStats.isSetNumDVs() ? doubleStats.getNumDVs() : null, doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null);
    } else if (stats.isSetLongStats()) {
        LongColumnStatsData longColStats = stats.getLongStats();
        return new CatalogColumnStatisticsDataLong(longColStats.isSetLowValue() ? longColStats.getLowValue() : null, longColStats.isSetHighValue() ? longColStats.getHighValue() : null, longColStats.isSetNumDVs() ? longColStats.getNumDVs() : null, longColStats.isSetNumNulls() ? longColStats.getNumNulls() : null);
    } else if (stats.isSetStringStats()) {
        StringColumnStatsData stringStats = stats.getStringStats();
        return new CatalogColumnStatisticsDataString(stringStats.isSetMaxColLen() ? stringStats.getMaxColLen() : null, stringStats.isSetAvgColLen() ? stringStats.getAvgColLen() : null, stringStats.isSetNumDVs() ? stringStats.getNumDVs() : null, stringStats.isSetNumDVs() ? stringStats.getNumNulls() : null);
    } else if (stats.isSetDecimalStats()) {
        DecimalColumnStatsData decimalStats = stats.getDecimalStats();
        // for now, just return CatalogColumnStatisticsDataDouble for decimal columns
        Double max = null;
        if (decimalStats.isSetHighValue()) {
            max = toHiveDecimal(decimalStats.getHighValue()).doubleValue();
        }
        Double min = null;
        if (decimalStats.isSetLowValue()) {
            min = toHiveDecimal(decimalStats.getLowValue()).doubleValue();
        }
        Long ndv = decimalStats.isSetNumDVs() ? decimalStats.getNumDVs() : null;
        Long nullCount = decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null;
        return new CatalogColumnStatisticsDataDouble(min, max, ndv, nullCount);
    } else {
        LOG.warn("Flink does not support converting ColumnStatisticsData '{}' for Hive column type '{}' yet.", stats, colType);
        return null;
    }
}