Examples with FieldSchema org.apache.hadoop.hive.metastore.api.FieldSchema used on opensource projects

Example 31 with FieldSchema

use of org.apache.hadoop.hive.metastore.api.FieldSchema in project hive by apache.

the class PlanModifierForReturnPath method introduceProjectIfNeeded.

private static RelNode introduceProjectIfNeeded(RelNode optimizedOptiqPlan, List<FieldSchema> resultSchema) throws CalciteSemanticException {
    List<String> fieldNames = new ArrayList<>();
    for (FieldSchema fieldSchema : resultSchema) {
        fieldNames.add(fieldSchema.getName());
    }
    RelNode newRoot = null;
    List<RexNode> projectList = null;
    if (!(optimizedOptiqPlan instanceof Project)) {
        projectList = HiveCalciteUtil.getProjsFromBelowAsInputRef(optimizedOptiqPlan);
        newRoot = HiveProject.create(optimizedOptiqPlan, projectList, fieldNames);
    } else {
        HiveProject project = (HiveProject) optimizedOptiqPlan;
        newRoot = HiveProject.create(project.getInput(0), project.getChildExps(), fieldNames);
    }
    return newRoot;
}

Example 32 with FieldSchema

use of org.apache.hadoop.hive.metastore.api.FieldSchema in project hive by apache.

the class PlanModifierUtil method generateInvalidSchemaMessage.

protected static String generateInvalidSchemaMessage(Project topLevelProj, List<FieldSchema> resultSchema, int fieldsForOB) {
    String errorDesc = "Result Schema didn't match Calcite Optimized Op Tree; schema: ";
    for (FieldSchema fs : resultSchema) {
        errorDesc += "[" + fs.getName() + ":" + fs.getType() + "], ";
    }
    errorDesc += " projection fields: ";
    for (RexNode exp : topLevelProj.getChildExps()) {
        errorDesc += "[" + exp.toString() + ":" + exp.getType() + "], ";
    }
    if (fieldsForOB != 0) {
        errorDesc += fieldsForOB + " fields removed due to ORDER BY  ";
    }
    return errorDesc.substring(0, errorDesc.length() - 2);
}

Example 33 with FieldSchema

use of org.apache.hadoop.hive.metastore.api.FieldSchema in project hive by apache.

the class HBaseUtils method hashStorageDescriptor.

/**
   * Produce a hash for the storage descriptor
   * @param sd storage descriptor to hash
   * @param md message descriptor to use to generate the hash
   * @return the hash as a byte array
   */
static byte[] hashStorageDescriptor(StorageDescriptor sd, MessageDigest md) {
    // Note all maps and lists have to be absolutely sorted.  Otherwise we'll produce different
    // results for hashes based on the OS or JVM being used.
    md.reset();
    for (FieldSchema fs : sd.getCols()) {
        md.update(fs.getName().getBytes(ENCODING));
        md.update(fs.getType().getBytes(ENCODING));
        if (fs.getComment() != null)
            md.update(fs.getComment().getBytes(ENCODING));
    }
    if (sd.getInputFormat() != null) {
        md.update(sd.getInputFormat().getBytes(ENCODING));
    }
    if (sd.getOutputFormat() != null) {
        md.update(sd.getOutputFormat().getBytes(ENCODING));
    }
    md.update(sd.isCompressed() ? "true".getBytes(ENCODING) : "false".getBytes(ENCODING));
    md.update(Integer.toString(sd.getNumBuckets()).getBytes(ENCODING));
    if (sd.getSerdeInfo() != null) {
        SerDeInfo serde = sd.getSerdeInfo();
        if (serde.getName() != null) {
            md.update(serde.getName().getBytes(ENCODING));
        }
        if (serde.getSerializationLib() != null) {
            md.update(serde.getSerializationLib().getBytes(ENCODING));
        }
        if (serde.getParameters() != null) {
            SortedMap<String, String> params = new TreeMap<>(serde.getParameters());
            for (Map.Entry<String, String> param : params.entrySet()) {
                md.update(param.getKey().getBytes(ENCODING));
                md.update(param.getValue().getBytes(ENCODING));
            }
        }
    }
    if (sd.getBucketCols() != null) {
        SortedSet<String> bucketCols = new TreeSet<>(sd.getBucketCols());
        for (String bucket : bucketCols) md.update(bucket.getBytes(ENCODING));
    }
    if (sd.getSortCols() != null) {
        SortedSet<Order> orders = new TreeSet<>(sd.getSortCols());
        for (Order order : orders) {
            md.update(order.getCol().getBytes(ENCODING));
            md.update(Integer.toString(order.getOrder()).getBytes(ENCODING));
        }
    }
    if (sd.getSkewedInfo() != null) {
        SkewedInfo skewed = sd.getSkewedInfo();
        if (skewed.getSkewedColNames() != null) {
            SortedSet<String> colnames = new TreeSet<>(skewed.getSkewedColNames());
            for (String colname : colnames) md.update(colname.getBytes(ENCODING));
        }
        if (skewed.getSkewedColValues() != null) {
            SortedSet<String> sortedOuterList = new TreeSet<>();
            for (List<String> innerList : skewed.getSkewedColValues()) {
                SortedSet<String> sortedInnerList = new TreeSet<>(innerList);
                sortedOuterList.add(StringUtils.join(sortedInnerList, "."));
            }
            for (String colval : sortedOuterList) md.update(colval.getBytes(ENCODING));
        }
        if (skewed.getSkewedColValueLocationMaps() != null) {
            SortedMap<String, String> sortedMap = new TreeMap<>();
            for (Map.Entry<List<String>, String> smap : skewed.getSkewedColValueLocationMaps().entrySet()) {
                SortedSet<String> sortedKey = new TreeSet<>(smap.getKey());
                sortedMap.put(StringUtils.join(sortedKey, "."), smap.getValue());
            }
            for (Map.Entry<String, String> e : sortedMap.entrySet()) {
                md.update(e.getKey().getBytes(ENCODING));
                md.update(e.getValue().getBytes(ENCODING));
            }
        }
    }
    return md.digest();
}

Example 34 with FieldSchema

use of org.apache.hadoop.hive.metastore.api.FieldSchema in project hive by apache.

the class SharedStorageDescriptor method copyCols.

private void copyCols() {
    if (!colsCopied) {
        colsCopied = true;
        if (super.getCols() != null) {
            List<FieldSchema> cols = new ArrayList<FieldSchema>(super.getColsSize());
            for (FieldSchema fs : super.getCols()) cols.add(new FieldSchema(fs));
            super.setCols(cols);
        }
    }
}

Example 35 with FieldSchema

use of org.apache.hadoop.hive.metastore.api.FieldSchema in project hive by apache.

the class HBaseStore method getPartitionNamesPrunedByExprNoTxn.

/**
   * Gets the partition names from a table, pruned using an expression.
   * @param table Table.
   * @param expr Expression.
   * @param defaultPartName Default partition name from job config, if any.
   * @param maxParts Maximum number of partition names to return.
   * @param result The resulting names.
   * @return Whether the result contains any unknown partitions.
   * @throws NoSuchObjectException
   */
private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr, String defaultPartName, short maxParts, List<String> result) throws MetaException, NoSuchObjectException {
    List<Partition> parts = getPartitions(table.getDbName(), table.getTableName(), maxParts);
    for (Partition part : parts) {
        result.add(Warehouse.makePartName(table.getPartitionKeys(), part.getValues()));
    }
    List<String> columnNames = new ArrayList<String>();
    List<PrimitiveTypeInfo> typeInfos = new ArrayList<PrimitiveTypeInfo>();
    for (FieldSchema fs : table.getPartitionKeys()) {
        columnNames.add(fs.getName());
        typeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(fs.getType()));
    }
    if (defaultPartName == null || defaultPartName.isEmpty()) {
        defaultPartName = HiveConf.getVar(getConf(), HiveConf.ConfVars.DEFAULTPARTITIONNAME);
    }
    return expressionProxy.filterPartitionsByExpr(columnNames, typeInfos, expr, defaultPartName, result);
}