Examples with AcidDirectory org.apache.hadoop.hive.ql.io.AcidDirectory used on opensource projects

Example 1 with AcidDirectory

use of org.apache.hadoop.hive.ql.io.AcidDirectory in project hive by apache.

the class Initiator method checkForCompaction.

private CompactionType checkForCompaction(final CompactionInfo ci, final ValidWriteIdList writeIds, final StorageDescriptor sd, final Map<String, String> tblproperties, final String runAs) throws IOException, InterruptedException {
    // If it's marked as too many aborted, we already know we need to compact
    if (ci.tooManyAborts) {
        LOG.debug("Found too many aborted transactions for " + ci.getFullPartitionName() + ", " + "initiating major compaction");
        return CompactionType.MAJOR;
    }
    if (ci.hasOldAbort) {
        HiveConf.ConfVars oldAbortedTimeoutProp = HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_TIME_THRESHOLD;
        LOG.debug("Found an aborted transaction for " + ci.getFullPartitionName() + " with age older than threshold " + oldAbortedTimeoutProp + ": " + conf.getTimeVar(oldAbortedTimeoutProp, TimeUnit.HOURS) + " hours. " + "Initiating minor compaction.");
        return CompactionType.MINOR;
    }
    AcidDirectory acidDirectory = getAcidDirectory(sd, writeIds);
    long baseSize = getBaseSize(acidDirectory);
    FileSystem fs = acidDirectory.getFs();
    Map<Path, Long> deltaSizes = new HashMap<>();
    for (AcidUtils.ParsedDelta delta : acidDirectory.getCurrentDirectories()) {
        deltaSizes.put(delta.getPath(), getDirSize(fs, delta));
    }
    long deltaSize = deltaSizes.values().stream().reduce(0L, Long::sum);
    AcidMetricService.updateMetricsFromInitiator(ci.dbname, ci.tableName, ci.partName, conf, txnHandler, baseSize, deltaSizes, acidDirectory.getObsolete());
    if (runJobAsSelf(runAs)) {
        return determineCompactionType(ci, acidDirectory, tblproperties, baseSize, deltaSize);
    } else {
        LOG.info("Going to initiate as user " + runAs + " for " + ci.getFullPartitionName());
        UserGroupInformation ugi = UserGroupInformation.createProxyUser(runAs, UserGroupInformation.getLoginUser());
        CompactionType compactionType;
        try {
            compactionType = ugi.doAs(new PrivilegedExceptionAction<CompactionType>() {

                @Override
                public CompactionType run() throws Exception {
                    return determineCompactionType(ci, acidDirectory, tblproperties, baseSize, deltaSize);
                }
            });
        } finally {
            try {
                FileSystem.closeAllForUGI(ugi);
            } catch (IOException exception) {
                LOG.error("Could not clean up file-system handles for UGI: " + ugi + " for " + ci.getFullPartitionName(), exception);
            }
        }
        return compactionType;
    }
}

Example 2 with AcidDirectory

use of org.apache.hadoop.hive.ql.io.AcidDirectory in project hive by apache.

the class Cleaner method removeFiles.

/**
 * @return true if any files were removed
 */
private boolean removeFiles(String location, ValidWriteIdList writeIdList, CompactionInfo ci) throws IOException, NoSuchObjectException, MetaException {
    Path path = new Path(location);
    FileSystem fs = path.getFileSystem(conf);
    AcidDirectory dir = AcidUtils.getAcidState(fs, path, conf, writeIdList, Ref.from(false), false);
    List<Path> obsoleteDirs = dir.getObsolete();
    /**
     * add anything in 'dir'  that only has data from aborted transactions - no one should be
     * trying to read anything in that dir (except getAcidState() that only reads the name of
     * this dir itself)
     * So this may run ahead of {@link CompactionInfo#highestWriteId} but it's ok (suppose there
     * are no active txns when cleaner runs).  The key is to not delete metadata about aborted
     * txns with write IDs > {@link CompactionInfo#highestWriteId}.
     * See {@link TxnStore#markCleaned(CompactionInfo)}
     */
    Table table = getMSForConf(conf).getTable(getDefaultCatalog(conf), ci.dbname, ci.tableName);
    if (isDynPartAbort(table, ci) || dir.hasUncompactedAborts()) {
        ci.setWriteIds(dir.hasUncompactedAborts(), dir.getAbortedWriteIds());
    }
    obsoleteDirs.addAll(dir.getAbortedDirectories());
    if (isDynPartAbort(table, ci)) {
        // In the event of an aborted DP operation, we should only consider the aborted directories for cleanup.
        // Including obsolete directories for partitioned tables can result in data loss.
        obsoleteDirs = dir.getAbortedDirectories();
    }
    if (obsoleteDirs.isEmpty() && !hasDataBelowWatermark(fs, path, writeIdList.getHighWatermark())) {
        LOG.info(idWatermark(ci) + " nothing to remove below watermark " + writeIdList.getHighWatermark() + ", ");
        return true;
    }
    StringBuilder extraDebugInfo = new StringBuilder("[").append(obsoleteDirs.stream().map(Path::getName).collect(Collectors.joining(",")));
    boolean success = remove(location, ci, obsoleteDirs, true, fs, extraDebugInfo);
    if (dir.getObsolete().size() > 0) {
        AcidMetricService.updateMetricsFromCleaner(ci.dbname, ci.tableName, ci.partName, dir.getObsolete(), conf, txnHandler);
    }
    return success;
}

Example 3 with AcidDirectory

use of org.apache.hadoop.hive.ql.io.AcidDirectory in project hive by apache.

the class Worker method findNextCompactionAndExecute.

/**
 * Finds the next compaction and executes it. The main thread might interrupt the execution of this method
 * in case of timeout.
 * @param collectGenericStats If true then for both MR and Query based compaction the stats are regenerated
 * @param collectMrStats If true then for MR compaction the stats are regenerated
 * @return Returns true, if there was compaction in the queue, and we started working on it.
 */
@VisibleForTesting
protected Boolean findNextCompactionAndExecute(boolean collectGenericStats, boolean collectMrStats) {
    // Make sure nothing escapes this run method and kills the metastore at large,
    // so wrap it in a big catch Throwable statement.
    PerfLogger perfLogger = SessionState.getPerfLogger(false);
    String workerMetric = null;
    CompactionInfo ci = null;
    boolean computeStats = false;
    Table t1 = null;
    // is need to be obtained here.
    if (msc == null) {
        try {
            msc = HiveMetaStoreUtils.getHiveMetastoreClient(conf);
        } catch (Exception e) {
            LOG.error("Failed to connect to HMS", e);
            return false;
        }
    }
    try (CompactionTxn compactionTxn = new CompactionTxn()) {
        FindNextCompactRequest findNextCompactRequest = new FindNextCompactRequest();
        findNextCompactRequest.setWorkerId(workerName);
        findNextCompactRequest.setWorkerVersion(runtimeVersion);
        ci = CompactionInfo.optionalCompactionInfoStructToInfo(msc.findNextCompact(findNextCompactRequest));
        LOG.debug("Processing compaction request " + ci);
        if (ci == null) {
            return false;
        }
        if ((runtimeVersion != null || ci.initiatorVersion != null) && !runtimeVersion.equals(ci.initiatorVersion)) {
            LOG.warn("Worker and Initiator versions do not match. Worker: v{}, Initiator: v{}", runtimeVersion, ci.initiatorVersion);
        }
        checkInterrupt();
        if (MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.METASTORE_ACIDMETRICS_EXT_ON)) {
            workerMetric = MetricsConstants.COMPACTION_WORKER_CYCLE + "_" + (ci.type != null ? ci.type.toString().toLowerCase() : null);
            perfLogger.perfLogBegin(CLASS_NAME, workerMetric);
        }
        // Find the table we will be working with.
        try {
            t1 = resolveTable(ci);
            if (t1 == null) {
                LOG.info("Unable to find table " + ci.getFullTableName() + ", assuming it was dropped and moving on.");
                msc.markCleaned(CompactionInfo.compactionInfoToStruct(ci));
                return false;
            }
        } catch (MetaException e) {
            msc.markCleaned(CompactionInfo.compactionInfoToStruct(ci));
            return false;
        }
        checkInterrupt();
        // This chicanery is to get around the fact that the table needs to be final in order to
        // go into the doAs below.
        final Table t = t1;
        String fullTableName = TxnUtils.getFullTableName(t.getDbName(), t.getTableName());
        // Find the partition we will be working with, if there is one.
        Partition p;
        try {
            p = resolvePartition(ci);
            if (p == null && ci.partName != null) {
                LOG.info("Unable to find partition " + ci.getFullPartitionName() + ", assuming it was dropped and moving on.");
                msc.markCleaned(CompactionInfo.compactionInfoToStruct(ci));
                return false;
            }
        } catch (Exception e) {
            msc.markCleaned(CompactionInfo.compactionInfoToStruct(ci));
            return false;
        }
        checkInterrupt();
        // Find the appropriate storage descriptor
        final StorageDescriptor sd = resolveStorageDescriptor(t, p);
        // Check that the table or partition isn't sorted, as we don't yet support that.
        if (sd.getSortCols() != null && !sd.getSortCols().isEmpty()) {
            LOG.error("Attempt to compact sorted table " + ci.getFullTableName() + ", which is not yet supported!");
            msc.markCleaned(CompactionInfo.compactionInfoToStruct(ci));
            return false;
        }
        if (ci.runAs == null) {
            ci.runAs = TxnUtils.findUserToRunAs(sd.getLocation(), t, conf);
        }
        checkInterrupt();
        /**
         * we cannot have Worker use HiveTxnManager (which is on ThreadLocal) since
         * then the Driver would already have the an open txn but then this txn would have
         * multiple statements in it (for query based compactor) which is not supported (and since
         * this case some of the statements are DDL, even in the future will not be allowed in a
         * multi-stmt txn. {@link Driver#setCompactionWriteIds(ValidWriteIdList, long)}
         */
        compactionTxn.open(ci);
        ValidTxnList validTxnList = msc.getValidTxns(compactionTxn.getTxnId());
        // with this ValidWriteIdList is capped at whatever HWM validTxnList has
        final ValidCompactorWriteIdList tblValidWriteIds = TxnUtils.createValidCompactWriteIdList(msc.getValidWriteIds(Collections.singletonList(fullTableName), validTxnList.writeToString()).get(0));
        LOG.debug("ValidCompactWriteIdList: " + tblValidWriteIds.writeToString());
        conf.set(ValidTxnList.VALID_TXNS_KEY, validTxnList.writeToString());
        ci.highestWriteId = tblValidWriteIds.getHighWatermark();
        // this writes TXN_COMPONENTS to ensure that if compactorTxnId fails, we keep metadata about
        // it until after any data written by it are physically removed
        msc.updateCompactorState(CompactionInfo.compactionInfoToStruct(ci), compactionTxn.getTxnId());
        checkInterrupt();
        final StringBuilder jobName = new StringBuilder(workerName);
        jobName.append("-compactor-");
        jobName.append(ci.getFullPartitionName());
        // Don't start compaction or cleaning if not necessary
        if (isDynPartAbort(t, ci)) {
            msc.markCompacted(CompactionInfo.compactionInfoToStruct(ci));
            compactionTxn.wasSuccessful();
            return false;
        }
        AcidDirectory dir = getAcidStateForWorker(ci, sd, tblValidWriteIds);
        if (!isEnoughToCompact(ci.isMajorCompaction(), dir, sd)) {
            if (needsCleaning(dir, sd)) {
                msc.markCompacted(CompactionInfo.compactionInfoToStruct(ci));
            } else {
                // do nothing
                msc.markCleaned(CompactionInfo.compactionInfoToStruct(ci));
            }
            compactionTxn.wasSuccessful();
            return false;
        }
        if (!ci.isMajorCompaction() && !isMinorCompactionSupported(t.getParameters(), dir)) {
            ci.errorMessage = String.format("Query based Minor compaction is not possible for full acid tables having raw " + "format (non-acid) data in them. Compaction type: %s, Partition: %s, Compaction id: %d", ci.type.toString(), ci.getFullPartitionName(), ci.id);
            LOG.error(ci.errorMessage);
            try {
                msc.markRefused(CompactionInfo.compactionInfoToStruct(ci));
            } catch (Throwable tr) {
                LOG.error("Caught an exception while trying to mark compaction {} as failed: {}", ci, tr);
            }
            return false;
        }
        checkInterrupt();
        try {
            failCompactionIfSetForTest();
            /*
        First try to run compaction via HiveQL queries.
        Compaction for MM tables happens here, or run compaction for Crud tables if query-based compaction is enabled.
        todo Find a more generic approach to collecting files in the same logical bucket to compact within the same
        task (currently we're using Tez split grouping).
        */
            QueryCompactor queryCompactor = QueryCompactorFactory.getQueryCompactor(t, conf, ci);
            computeStats = (queryCompactor == null && collectMrStats) || collectGenericStats;
            LOG.info("Starting " + ci.type.toString() + " compaction for " + ci.getFullPartitionName() + ", id:" + ci.id + " in " + compactionTxn + " with compute stats set to " + computeStats);
            if (queryCompactor != null) {
                LOG.info("Will compact id: " + ci.id + " with query-based compactor class: " + queryCompactor.getClass().getName());
                queryCompactor.runCompaction(conf, t, p, sd, tblValidWriteIds, ci, dir);
            } else {
                LOG.info("Will compact id: " + ci.id + " via MR job");
                runCompactionViaMrJob(ci, t, p, sd, tblValidWriteIds, jobName, dir);
            }
            LOG.info("Completed " + ci.type.toString() + " compaction for " + ci.getFullPartitionName() + " in " + compactionTxn + ", marking as compacted.");
            msc.markCompacted(CompactionInfo.compactionInfoToStruct(ci));
            compactionTxn.wasSuccessful();
            AcidMetricService.updateMetricsFromWorker(ci.dbname, ci.tableName, ci.partName, ci.type, dir.getCurrentDirectories().size(), dir.getDeleteDeltas().size(), conf, msc);
        } catch (Throwable e) {
            LOG.error("Caught exception while trying to compact " + ci + ". Marking failed to avoid repeated failures", e);
            final CompactionType ctype = ci.type;
            markFailed(ci, e.getMessage());
            if (runJobAsSelf(ci.runAs)) {
                cleanupResultDirs(sd, tblValidWriteIds, ctype, dir);
            } else {
                LOG.info("Cleaning as user " + ci.runAs);
                UserGroupInformation ugi = UserGroupInformation.createProxyUser(ci.runAs, UserGroupInformation.getLoginUser());
                ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
                    cleanupResultDirs(sd, tblValidWriteIds, ctype, dir);
                    return null;
                });
                try {
                    FileSystem.closeAllForUGI(ugi);
                } catch (IOException ex) {
                    LOG.error("Could not clean up file-system handles for UGI: " + ugi, e);
                }
            }
        }
    } catch (TException | IOException t) {
        LOG.error("Caught an exception in the main loop of compactor worker " + workerName, t);
        markFailed(ci, t.getMessage());
        if (msc != null) {
            msc.close();
            msc = null;
        }
    } catch (Throwable t) {
        LOG.error("Caught an exception in the main loop of compactor worker " + workerName, t);
    } finally {
        if (workerMetric != null && MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.METASTORE_ACIDMETRICS_EXT_ON)) {
            perfLogger.perfLogEnd(CLASS_NAME, workerMetric);
        }
    }
    if (computeStats) {
        StatsUpdater.gatherStats(ci, conf, runJobAsSelf(ci.runAs) ? ci.runAs : t1.getOwner(), CompactorUtil.getCompactorJobQueueName(conf, ci, t1));
    }
    return true;
}

Example 4 with AcidDirectory

use of org.apache.hadoop.hive.ql.io.AcidDirectory in project hive by apache.

the class VectorizedOrcAcidRowBatchReader method computeOffsetAndBucket.

/**
 * See {@link #next(NullWritable, VectorizedRowBatch)} first and
 * {@link OrcRawRecordMerger.OriginalReaderPair}.
 * When reading a split of an "original" file and we need to decorate data with ROW__ID.
 * This requires treating multiple files that are part of the same bucket (tranche for unbucketed
 * tables) as a single logical file to number rowids consistently.
 */
static OrcSplit.OffsetAndBucketProperty computeOffsetAndBucket(FileStatus file, Path rootDir, boolean isOriginal, boolean hasDeletes, Configuration conf) throws IOException {
    VectorizedRowBatchCtx vrbCtx = Utilities.getVectorizedRowBatchCtx(conf);
    if (!needSyntheticRowIds(isOriginal, hasDeletes, areRowIdsProjected(vrbCtx))) {
        if (isOriginal) {
            /**
             * Even if we don't need to project ROW_IDs, we still need to check the write ID that
             * created the file to see if it's committed.  See more in
             * {@link #next(NullWritable, VectorizedRowBatch)}.  (In practice getAcidState() should
             * filter out base/delta files but this makes fewer dependencies)
             */
            OrcRawRecordMerger.TransactionMetaData syntheticTxnInfo = OrcRawRecordMerger.TransactionMetaData.findWriteIDForSynthetcRowIDs(file.getPath(), rootDir, conf);
            return new OrcSplit.OffsetAndBucketProperty(-1, -1, syntheticTxnInfo.syntheticWriteId);
        }
        return null;
    }
    String txnString = conf.get(ValidWriteIdList.VALID_WRITEIDS_KEY);
    ValidWriteIdList validWriteIdList = (txnString == null) ? new ValidReaderWriteIdList() : new ValidReaderWriteIdList(txnString);
    long rowIdOffset = 0;
    OrcRawRecordMerger.TransactionMetaData syntheticTxnInfo = OrcRawRecordMerger.TransactionMetaData.findWriteIDForSynthetcRowIDs(file.getPath(), rootDir, conf);
    int bucketId = AcidUtils.parseBucketId(file.getPath());
    int bucketProperty = BucketCodec.V1.encode(new AcidOutputFormat.Options(conf).statementId(syntheticTxnInfo.statementId).bucket(bucketId));
    AcidDirectory directoryState = AcidUtils.getAcidState(null, syntheticTxnInfo.folder, conf, validWriteIdList, Ref.from(false), true);
    for (HadoopShims.HdfsFileStatusWithId f : directoryState.getOriginalFiles()) {
        int bucketIdFromPath = AcidUtils.parseBucketId(f.getFileStatus().getPath());
        if (bucketIdFromPath != bucketId) {
            // HIVE-16952
            continue;
        }
        if (f.getFileStatus().getPath().equals(file.getPath())) {
            // 'f' is the file whence this split is
            break;
        }
        Reader reader = OrcFile.createReader(f.getFileStatus().getPath(), OrcFile.readerOptions(conf));
        rowIdOffset += reader.getNumberOfRows();
    }
    return new OrcSplit.OffsetAndBucketProperty(rowIdOffset, bucketProperty, syntheticTxnInfo.syntheticWriteId);
}

Example 5 with AcidDirectory

use of org.apache.hadoop.hive.ql.io.AcidDirectory in project hive by apache.

the class TestStreaming method checkNothingWritten.

private void checkNothingWritten(Path partitionPath) throws Exception {
    AcidDirectory dir = AcidUtils.getAcidState(null, partitionPath, conf, getTransactionContext(conf), null, false);
    Assert.assertEquals(0, dir.getObsolete().size());
    Assert.assertEquals(0, dir.getOriginalFiles().size());
    List<AcidUtils.ParsedDelta> current = dir.getCurrentDirectories();
    Assert.assertEquals(0, current.size());
}