Examples with PartitionIterable org.apache.hadoop.hive.ql.metadata.PartitionIterable used on opensource projects

Example 6 with PartitionIterable

use of org.apache.hadoop.hive.ql.metadata.PartitionIterable in project hive by apache.

the class TableExport method getAuthEntities.

public AuthEntities getAuthEntities() throws SemanticException {
    AuthEntities authEntities = new AuthEntities();
    try {
        // Return if metadata-only
        if (replicationSpec.isMetadataOnly() || replicationSpec.isMetadataOnlyForExternalTables()) {
            return authEntities;
        }
        PartitionIterable partitions = getPartitions();
        if (tableSpec != null) {
            if (tableSpec.tableHandle.isPartitioned()) {
                if (partitions == null) {
                    throw new IllegalStateException("partitions cannot be null for partitionTable :" + tableSpec.getTableName().getTable());
                }
                for (Partition partition : partitions) {
                    authEntities.inputs.add(new ReadEntity(partition));
                }
            } else {
                authEntities.inputs.add(new ReadEntity(tableSpec.tableHandle));
            }
        }
        authEntities.outputs.add(toWriteEntity(paths.metadataExportRootDir(), conf));
    } catch (Exception e) {
        throw new SemanticException(e);
    }
    return authEntities;
}

Example 7 with PartitionIterable

use of org.apache.hadoop.hive.ql.metadata.PartitionIterable in project hive by apache.

the class ExportSemanticAnalyzer method prepareExport.

// FIXME : Move to EximUtil - it's okay for this to stay here for a little while more till we finalize the statics
public static void prepareExport(ASTNode ast, URI toURI, TableSpec ts, ReplicationSpec replicationSpec, Hive db, HiveConf conf, Context ctx, List<Task<? extends Serializable>> rootTasks, HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs, Logger LOG) throws SemanticException {
    if (ts != null) {
        try {
            EximUtil.validateTable(ts.tableHandle);
            if (replicationSpec.isInReplicationScope() && ts.tableHandle.isTemporary()) {
                // No replication for temporary tables either
                ts = null;
            } else if (ts.tableHandle.isView()) {
                replicationSpec.setIsMetadataOnly(true);
            }
        } catch (SemanticException e) {
            // ignore for replication, error if not.
            if (replicationSpec.isInReplicationScope()) {
                // null out ts so we can't use it.
                ts = null;
            } else {
                throw e;
            }
        }
    }
    try {
        FileSystem fs = FileSystem.get(toURI, conf);
        Path toPath = new Path(toURI.getScheme(), toURI.getAuthority(), toURI.getPath());
        try {
            FileStatus tgt = fs.getFileStatus(toPath);
            // target exists
            if (!tgt.isDir()) {
                throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast, "Target is not a directory : " + toURI));
            } else {
                FileStatus[] files = fs.listStatus(toPath, FileUtils.HIDDEN_FILES_PATH_FILTER);
                if (files != null && files.length != 0) {
                    throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast, "Target is not an empty directory : " + toURI));
                }
            }
        } catch (FileNotFoundException e) {
        }
    } catch (IOException e) {
        throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast), e);
    }
    PartitionIterable partitions = null;
    try {
        replicationSpec.setCurrentReplicationState(String.valueOf(db.getMSC().getCurrentNotificationEventId().getEventId()));
        if ((ts != null) && (ts.tableHandle.isPartitioned())) {
            if (ts.specType == TableSpec.SpecType.TABLE_ONLY) {
                // TABLE-ONLY, fetch partitions if regular export, don't if metadata-only
                if (replicationSpec.isMetadataOnly()) {
                    partitions = null;
                } else {
                    partitions = new PartitionIterable(db, ts.tableHandle, null, conf.getIntVar(HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX));
                }
            } else {
                // PARTITIONS specified - partitions inside tableSpec
                partitions = new PartitionIterable(ts.partitions);
            }
        } else {
            // Either tableHandle isn't partitioned => null, or repl-export after ts becomes null => null.
            // or this is a noop-replication export, so we can skip looking at ptns.
            partitions = null;
        }
        Path path = new Path(ctx.getLocalTmpPath(), EximUtil.METADATA_NAME);
        EximUtil.createExportDump(FileSystem.getLocal(conf), path, (ts != null ? ts.tableHandle : null), partitions, replicationSpec);
        Task<? extends Serializable> rTask = ReplCopyTask.getDumpCopyTask(replicationSpec, path, new Path(toURI), conf);
        rootTasks.add(rTask);
        LOG.debug("_metadata file written into " + path.toString() + " and then copied to " + toURI.toString());
    } catch (Exception e) {
        throw new SemanticException(ErrorMsg.IO_ERROR.getMsg("Exception while writing out the local file"), e);
    }
    if (!(replicationSpec.isMetadataOnly() || (ts == null))) {
        Path parentPath = new Path(toURI);
        if (ts.tableHandle.isPartitioned()) {
            for (Partition partition : partitions) {
                Path fromPath = partition.getDataLocation();
                Path toPartPath = new Path(parentPath, partition.getName());
                Task<? extends Serializable> rTask = ReplCopyTask.getDumpCopyTask(replicationSpec, fromPath, toPartPath, conf);
                rootTasks.add(rTask);
                inputs.add(new ReadEntity(partition));
            }
        } else {
            Path fromPath = ts.tableHandle.getDataLocation();
            Path toDataPath = new Path(parentPath, EximUtil.DATA_PATH_NAME);
            Task<? extends Serializable> rTask = ReplCopyTask.getDumpCopyTask(replicationSpec, fromPath, toDataPath, conf);
            rootTasks.add(rTask);
            inputs.add(new ReadEntity(ts.tableHandle));
        }
        outputs.add(toWriteEntity(parentPath, conf));
    }
}

Example 8 with PartitionIterable

use of org.apache.hadoop.hive.ql.metadata.PartitionIterable in project hive by apache.

the class DescTableOperation method getColumnsNoColumnPath.

private void getColumnsNoColumnPath(Table table, Partition partition, List<FieldSchema> cols) throws HiveException {
    cols.addAll(partition == null || table.getTableType() == TableType.VIRTUAL_VIEW ? table.getCols() : partition.getCols());
    if (!desc.isFormatted()) {
        cols.addAll(table.getPartCols());
    }
    // Fetch partition statistics only for describe extended or formatted.
    if (desc.isExtended() || desc.isFormatted()) {
        boolean disablePartitionStats = HiveConf.getBoolVar(context.getConf(), HiveConf.ConfVars.HIVE_DESCRIBE_PARTITIONED_TABLE_IGNORE_STATS);
        if (table.isPartitioned() && partition == null && !disablePartitionStats) {
            // No partition specified for partitioned table, lets fetch all.
            Map<String, String> tblProps = table.getParameters() == null ? new HashMap<String, String>() : table.getParameters();
            Map<String, Long> valueMap = new HashMap<>();
            Map<String, Boolean> stateMap = new HashMap<>();
            for (String stat : StatsSetupConst.SUPPORTED_STATS) {
                valueMap.put(stat, 0L);
                stateMap.put(stat, true);
            }
            PartitionIterable partitions = new PartitionIterable(context.getDb(), table, null, MetastoreConf.getIntVar(context.getConf(), MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX));
            int numParts = 0;
            for (Partition p : partitions) {
                Map<String, String> partitionProps = p.getParameters();
                Boolean state = StatsSetupConst.areBasicStatsUptoDate(partitionProps);
                for (String stat : StatsSetupConst.SUPPORTED_STATS) {
                    stateMap.put(stat, stateMap.get(stat) && state);
                    if (partitionProps != null && partitionProps.get(stat) != null) {
                        valueMap.put(stat, valueMap.get(stat) + Long.parseLong(partitionProps.get(stat)));
                    }
                }
                numParts++;
            }
            tblProps.put(StatsSetupConst.NUM_PARTITIONS, Integer.toString(numParts));
            for (String stat : StatsSetupConst.SUPPORTED_STATS) {
                StatsSetupConst.setBasicStatsState(tblProps, Boolean.toString(stateMap.get(stat)));
                tblProps.put(stat, valueMap.get(stat).toString());
            }
            table.setParameters(tblProps);
        }
    }
}

Example 9 with PartitionIterable

use of org.apache.hadoop.hive.ql.metadata.PartitionIterable in project hive by apache.

the class DescTableOperation method getColumnDataForPartitionKeyColumn.

private void getColumnDataForPartitionKeyColumn(Table table, List<FieldSchema> cols, List<ColumnStatisticsObj> colStats, List<String> colNames, Map<String, String> tableProps) throws HiveException, MetaException {
    FieldSchema partCol = table.getPartColByName(colNames.get(0));
    cols.add(partCol);
    PartitionIterable parts = new PartitionIterable(context.getDb(), table, null, MetastoreConf.getIntVar(context.getConf(), MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX));
    ColumnInfo ci = new ColumnInfo(partCol.getName(), TypeInfoUtils.getTypeInfoFromTypeString(partCol.getType()), null, false);
    ColStatistics cs = StatsUtils.getColStatsForPartCol(ci, parts, context.getConf());
    ColumnStatisticsData data = new ColumnStatisticsData();
    ColStatistics.Range r = cs.getRange();
    StatObjectConverter.fillColumnStatisticsData(partCol.getType(), data, r == null ? null : r.minValue, r == null ? null : r.maxValue, r == null ? null : r.minValue, r == null ? null : r.maxValue, r == null ? null : r.minValue.toString(), r == null ? null : r.maxValue.toString(), cs.getNumNulls(), cs.getCountDistint(), null, cs.getAvgColLen(), cs.getAvgColLen(), cs.getNumTrues(), cs.getNumFalses());
    ColumnStatisticsObj cso = new ColumnStatisticsObj(partCol.getName(), partCol.getType(), data);
    colStats.add(cso);
    StatsSetupConst.setColumnStatsState(tableProps, colNames);
}

Example 10 with PartitionIterable

use of org.apache.hadoop.hive.ql.metadata.PartitionIterable in project hive by apache.

the class AlterTableSetPropertiesOperation method doAlteration.

@Override
protected void doAlteration(Table table, Partition partition) throws HiveException {
    if (StatsSetupConst.USER.equals(environmentContext.getProperties().get(StatsSetupConst.STATS_GENERATED))) {
        environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS);
    }
    if (partition != null) {
        partition.getTPartition().getParameters().putAll(desc.getProps());
    } else {
        boolean isFromMmTable = AcidUtils.isInsertOnlyTable(table.getParameters());
        Boolean isToMmTable = AcidUtils.isToInsertOnlyTable(table, desc.getProps());
        if (!isFromMmTable && BooleanUtils.isTrue(isToMmTable)) {
            if (!HiveConf.getBoolVar(context.getConf(), ConfVars.HIVE_MM_ALLOW_ORIGINALS)) {
                List<Task<?>> mmTasks = generateAddMmTasks(table, desc.getWriteId());
                for (Task<?> mmTask : mmTasks) {
                    context.getTask().addDependentTask(mmTask);
                }
            } else {
                if (!table.getPartitionKeys().isEmpty()) {
                    PartitionIterable parts = new PartitionIterable(context.getDb(), table, null, MetastoreConf.getIntVar(context.getConf(), MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX));
                    for (Partition part : parts) {
                        checkMmLb(part);
                    }
                } else {
                    checkMmLb(table);
                }
            }
        } else if (isFromMmTable && BooleanUtils.isFalse(isToMmTable)) {
            throw new HiveException("Cannot convert an ACID table to non-ACID");
        }
        // Converting to/from external table
        String externalProp = desc.getProps().get("EXTERNAL");
        if (externalProp != null) {
            if (Boolean.parseBoolean(externalProp) && table.getTableType() == TableType.MANAGED_TABLE) {
                table.setTableType(TableType.EXTERNAL_TABLE);
            } else if (!Boolean.parseBoolean(externalProp) && table.getTableType() == TableType.EXTERNAL_TABLE) {
                table.setTableType(TableType.MANAGED_TABLE);
            }
        }
        table.getTTable().getParameters().putAll(desc.getProps());
    }
}