Examples with PartitionTransformSpec org.apache.hadoop.hive.ql.parse.PartitionTransformSpec used on opensource projects

Example 1 with PartitionTransformSpec

use of org.apache.hadoop.hive.ql.parse.PartitionTransformSpec in project hive by apache.

the class HiveIcebergStorageHandler method getPartitionTransformSpec.

@Override
public List<PartitionTransformSpec> getPartitionTransformSpec(org.apache.hadoop.hive.ql.metadata.Table hmsTable) {
    List<PartitionTransformSpec> result = new ArrayList<>();
    TableDesc tableDesc = Utilities.getTableDesc(hmsTable);
    Table table = IcebergTableUtil.getTable(conf, tableDesc.getProperties());
    return table.spec().fields().stream().map(f -> {
        PartitionTransformSpec spec = new PartitionTransformSpec();
        spec.setColumnName(table.schema().findColumnName(f.sourceId()));
        // right now the only way to fetch the transform type and its params is through the toString() call
        String transformName = f.transform().toString().toUpperCase();
        // if the transform name contains '[' it means it has some config params
        if (transformName.contains("[")) {
            spec.setTransformType(PartitionTransformSpec.TransformType.valueOf(transformName.substring(0, transformName.indexOf("["))));
            spec.setTransformParam(Optional.of(Integer.valueOf(transformName.substring(transformName.indexOf("[") + 1, transformName.indexOf("]")))));
        } else {
            spec.setTransformType(PartitionTransformSpec.TransformType.valueOf(transformName));
            spec.setTransformParam(Optional.empty());
        }
        return spec;
    }).collect(Collectors.toList());
}

Example 2 with PartitionTransformSpec

use of org.apache.hadoop.hive.ql.parse.PartitionTransformSpec in project hive by apache.

the class DDLPlanUtils method getPartitionsBySpec.

private String getPartitionsBySpec(Table table) {
    if (table.isNonNative() && table.getStorageHandler() != null && table.getStorageHandler().supportsPartitionTransform()) {
        List<PartitionTransformSpec> specs = table.getStorageHandler().getPartitionTransformSpec(table);
        if (specs.isEmpty()) {
            return "";
        }
        List<String> partitionTransforms = new ArrayList<>();
        for (PartitionTransformSpec spec : specs) {
            if (spec.getTransformType() == PartitionTransformSpec.TransformType.IDENTITY) {
                partitionTransforms.add(spec.getColumnName());
            } else {
                partitionTransforms.add(spec.getTransformType().name() + "(" + (spec.getTransformParam().isPresent() ? spec.getTransformParam().get() + ", " : "") + spec.getColumnName() + ")");
            }
        }
        return "PARTITIONED BY SPEC ( \n" + StringUtils.join(partitionTransforms, ", \n") + ")";
    }
    return "";
}

Example 3 with PartitionTransformSpec

use of org.apache.hadoop.hive.ql.parse.PartitionTransformSpec in project hive by apache.

the class TextDescTableFormatter method addPartitionTransformData.

private void addPartitionTransformData(DataOutputStream out, Table table, boolean isOutputPadded) throws IOException {
    String partitionTransformOutput = "";
    if (table.isNonNative() && table.getStorageHandler() != null && table.getStorageHandler().supportsPartitionTransform()) {
        List<PartitionTransformSpec> partSpecs = table.getStorageHandler().getPartitionTransformSpec(table);
        if (partSpecs != null && !partSpecs.isEmpty()) {
            TextMetaDataTable metaDataTable = new TextMetaDataTable();
            partitionTransformOutput += LINE_DELIM + "# Partition Transform Information" + LINE_DELIM + "# ";
            metaDataTable.addRow(DescTableDesc.PARTITION_TRANSFORM_SPEC_SCHEMA.split("#")[0].split(","));
            for (PartitionTransformSpec spec : partSpecs) {
                String[] row = new String[2];
                row[0] = spec.getColumnName();
                if (spec.getTransformType() != null) {
                    row[1] = spec.getTransformParam().isPresent() ? spec.getTransformType().name() + "[" + spec.getTransformParam().get() + "]" : spec.getTransformType().name();
                }
                metaDataTable.addRow(row);
            }
            partitionTransformOutput += metaDataTable.renderTable(isOutputPadded);
        }
    }
    out.write(partitionTransformOutput.getBytes(StandardCharsets.UTF_8));
}

Example 4 with PartitionTransformSpec

use of org.apache.hadoop.hive.ql.parse.PartitionTransformSpec in project hive by apache.

the class CreateTableDesc method toTable.

public Table toTable(HiveConf conf) throws HiveException {
    Table tbl = new Table(tableName.getDb(), tableName.getTable());
    if (getTblProps() != null) {
        tbl.getTTable().getParameters().putAll(getTblProps());
    }
    if (getNumBuckets() != -1) {
        tbl.setNumBuckets(getNumBuckets());
    }
    if (getStorageHandler() != null) {
        tbl.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, getStorageHandler());
    }
    HiveStorageHandler storageHandler = tbl.getStorageHandler();
    /*
     * If the user didn't specify a SerDe, we use the default.
     */
    String serDeClassName;
    if (getSerName() == null) {
        if (storageHandler == null) {
            serDeClassName = PlanUtils.getDefaultSerDe().getName();
            LOG.info("Default to " + serDeClassName + " for table " + tableName);
        } else {
            serDeClassName = storageHandler.getSerDeClass().getName();
            LOG.info("Use StorageHandler-supplied " + serDeClassName + " for table " + tableName);
        }
    } else {
        // let's validate that the serde exists
        serDeClassName = getSerName();
        DDLUtils.validateSerDe(serDeClassName, conf);
    }
    tbl.setSerializationLib(serDeClassName);
    if (getFieldDelim() != null) {
        tbl.setSerdeParam(serdeConstants.FIELD_DELIM, getFieldDelim());
        tbl.setSerdeParam(serdeConstants.SERIALIZATION_FORMAT, getFieldDelim());
    }
    if (getFieldEscape() != null) {
        tbl.setSerdeParam(serdeConstants.ESCAPE_CHAR, getFieldEscape());
    }
    if (getCollItemDelim() != null) {
        tbl.setSerdeParam(serdeConstants.COLLECTION_DELIM, getCollItemDelim());
    }
    if (getMapKeyDelim() != null) {
        tbl.setSerdeParam(serdeConstants.MAPKEY_DELIM, getMapKeyDelim());
    }
    if (getLineDelim() != null) {
        tbl.setSerdeParam(serdeConstants.LINE_DELIM, getLineDelim());
    }
    if (getNullFormat() != null) {
        tbl.setSerdeParam(serdeConstants.SERIALIZATION_NULL_FORMAT, getNullFormat());
    }
    if (getSerdeProps() != null) {
        Iterator<Map.Entry<String, String>> iter = getSerdeProps().entrySet().iterator();
        while (iter.hasNext()) {
            Map.Entry<String, String> m = iter.next();
            tbl.setSerdeParam(m.getKey(), m.getValue());
        }
    }
    Optional<List<FieldSchema>> cols = Optional.ofNullable(getCols());
    Optional<List<FieldSchema>> partCols = Optional.ofNullable(getPartCols());
    if (storageHandler != null && storageHandler.alwaysUnpartitioned()) {
        tbl.getSd().setCols(new ArrayList<>());
        cols.ifPresent(c -> tbl.getSd().getCols().addAll(c));
        if (partCols.isPresent() && !partCols.get().isEmpty()) {
            // Add the partition columns to the normal columns and save the transform to the session state
            tbl.getSd().getCols().addAll(partCols.get());
            List<PartitionTransformSpec> spec = PartitionTransform.getPartitionTransformSpec(partCols.get());
            if (!SessionStateUtil.addResource(conf, hive_metastoreConstants.PARTITION_TRANSFORM_SPEC, spec)) {
                throw new HiveException("Query state attached to Session state must be not null. " + "Partition transform metadata cannot be saved.");
            }
        }
    } else {
        cols.ifPresent(c -> tbl.setFields(c));
        partCols.ifPresent(c -> tbl.setPartCols(c));
    }
    if (getBucketCols() != null) {
        tbl.setBucketCols(getBucketCols());
    }
    if (getSortCols() != null) {
        tbl.setSortCols(getSortCols());
    }
    if (getComment() != null) {
        tbl.setProperty("comment", getComment());
    }
    if (getLocation() != null) {
        tbl.setDataLocation(new Path(getLocation()));
    }
    if (getSkewedColNames() != null) {
        tbl.setSkewedColNames(getSkewedColNames());
    }
    if (getSkewedColValues() != null) {
        tbl.setSkewedColValues(getSkewedColValues());
    }
    tbl.getTTable().setTemporary(isTemporary());
    tbl.setStoredAsSubDirectories(isStoredAsSubDirectories());
    tbl.setInputFormatClass(getInputFormat());
    tbl.setOutputFormatClass(getOutputFormat());
    // Otherwise, load lazily via StorageHandler at query time.
    if (getInputFormat() != null && !getInputFormat().isEmpty()) {
        tbl.getTTable().getSd().setInputFormat(tbl.getInputFormatClass().getName());
    }
    if (getOutputFormat() != null && !getOutputFormat().isEmpty()) {
        tbl.getTTable().getSd().setOutputFormat(tbl.getOutputFormatClass().getName());
    }
    if (CreateTableOperation.doesTableNeedLocation(tbl)) {
        // If location is specified - ensure that it is a full qualified name
        CreateTableOperation.makeLocationQualified(tbl, conf);
    }
    if (isExternal()) {
        tbl.setProperty("EXTERNAL", "TRUE");
        tbl.setTableType(TableType.EXTERNAL_TABLE);
    }
    // 'n' columns where 'n' is the length of the bucketed columns.
    if ((tbl.getBucketCols() != null) && (tbl.getSortCols() != null)) {
        List<String> bucketCols = tbl.getBucketCols();
        List<Order> sortCols = tbl.getSortCols();
        if ((sortCols.size() > 0) && (sortCols.size() >= bucketCols.size())) {
            boolean found = true;
            Iterator<String> iterBucketCols = bucketCols.iterator();
            while (iterBucketCols.hasNext()) {
                String bucketCol = iterBucketCols.next();
                boolean colFound = false;
                for (int i = 0; i < bucketCols.size(); i++) {
                    if (bucketCol.equals(sortCols.get(i).getCol())) {
                        colFound = true;
                        break;
                    }
                }
                if (colFound == false) {
                    found = false;
                    break;
                }
            }
            if (found) {
                tbl.setProperty("SORTBUCKETCOLSPREFIX", "TRUE");
            }
        }
    }
    if (colStats != null) {
        ColumnStatisticsDesc colStatsDesc = new ColumnStatisticsDesc(colStats.getStatsDesc());
        colStatsDesc.setCatName(tbl.getCatName());
        colStatsDesc.setDbName(tbl.getDbName());
        colStatsDesc.setTableName(tbl.getTableName());
        String engine = colStats.getEngine();
        if (engine == null) {
            engine = org.apache.hadoop.hive.conf.Constants.HIVE_ENGINE;
        }
        ColumnStatistics columnStatistics = new ColumnStatistics(colStatsDesc, colStats.getStatsObj());
        columnStatistics.setEngine(engine);
        tbl.getTTable().setColStats(columnStatistics);
        // update column statistics.
        if (replWriteId > 0) {
            tbl.getTTable().setWriteId(replWriteId);
        }
    }
    // reset it on replica.
    if (replicationSpec == null || !replicationSpec.isInReplicationScope()) {
        if (!this.isCTAS && (tbl.getPath() == null || (!isExternal() && tbl.isEmpty()))) {
            if (!tbl.isPartitioned() && conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
                StatsSetupConst.setStatsStateForCreateTable(tbl.getTTable().getParameters(), MetaStoreUtils.getColumnNames(tbl.getCols()), StatsSetupConst.TRUE);
            }
        } else {
            StatsSetupConst.setStatsStateForCreateTable(tbl.getTTable().getParameters(), null, StatsSetupConst.FALSE);
        }
    }
    if (ownerName != null) {
        tbl.setOwner(ownerName);
    }
    return tbl;
}

Example 5 with PartitionTransformSpec

use of org.apache.hadoop.hive.ql.parse.PartitionTransformSpec in project hive by apache.

the class AlterTableSetPartitionSpecAnalyzer method analyzeCommand.

@Override
protected void analyzeCommand(TableName tableName, Map<String, String> partitionSpec, ASTNode command) throws SemanticException {
    Table table = getTable(tableName);
    validateAlterTableType(table, AlterTableType.SETPARTITIONSPEC, false);
    inputs.add(new ReadEntity(table));
    List<PartitionTransformSpec> partitionTransformSpec = PartitionTransform.getPartitionTransformSpec(command);
    if (!SessionStateUtil.addResource(conf, hive_metastoreConstants.PARTITION_TRANSFORM_SPEC, partitionTransformSpec)) {
        throw new SemanticException("Query state attached to Session state must be not null. " + "Partition transform metadata cannot be saved.");
    }
    AlterTableSetPartitionSpecDesc desc = new AlterTableSetPartitionSpecDesc(tableName, partitionSpec);
    rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc)));
}