Examples with VisibleForTesting org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting used on opensource projects

Example 16 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class EntityGroupFSTimelineStore method createAndInitYarnClient.

/**
   * Create and initialize the YARN Client. Tests may override/mock this.
   * If they return null, then {@link #getAppState(ApplicationId)} MUST
   * also be overridden
   * @param conf configuration
   * @return the yarn client, or null.
   *
   */
@VisibleForTesting
protected YarnClient createAndInitYarnClient(Configuration conf) {
    YarnClient client = YarnClient.createYarnClient();
    client.init(conf);
    return client;
}

Example 17 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hadoop by apache.

the class EntityGroupFSTimelineStore method scanActiveLogs.

@InterfaceAudience.Private
@VisibleForTesting
int scanActiveLogs() throws IOException {
    long startTime = Time.monotonicNow();
    RemoteIterator<FileStatus> iter = list(activeRootPath);
    int logsToScanCount = 0;
    while (iter.hasNext()) {
        FileStatus stat = iter.next();
        String name = stat.getPath().getName();
        ApplicationId appId = parseApplicationId(name);
        if (appId != null) {
            LOG.debug("scan logs for {} in {}", appId, stat.getPath());
            logsToScanCount++;
            AppLogs logs = getAndSetActiveLog(appId, stat.getPath());
            executor.execute(new ActiveLogParser(logs));
        } else {
            LOG.debug("Unable to parse entry {}", name);
        }
    }
    metrics.addActiveLogDirScanTime(Time.monotonicNow() - startTime);
    return logsToScanCount;
}

Example 18 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hbase by apache.

the class SnapshotFileCache method getSnapshotsInProgress.

@VisibleForTesting
List<String> getSnapshotsInProgress(final SnapshotManager snapshotManager) throws IOException {
    List<String> snapshotInProgress = Lists.newArrayList();
    // only add those files to the cache, but not to the known snapshots
    Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
    // only add those files to the cache, but not to the known snapshots
    FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir);
    if (running != null) {
        for (FileStatus run : running) {
            ReentrantLock lock = null;
            if (snapshotManager != null) {
                lock = snapshotManager.getLocks().acquireLock(run.getPath().getName());
            }
            try {
                snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
            } catch (CorruptedSnapshotException e) {
                // See HBASE-16464
                if (e.getCause() instanceof FileNotFoundException) {
                    // If the snapshot is corrupt, we will delete it
                    fs.delete(run.getPath(), true);
                    LOG.warn("delete the " + run.getPath() + " due to exception:", e.getCause());
                } else {
                    throw e;
                }
            } finally {
                if (lock != null) {
                    lock.unlock();
                }
            }
        }
    }
    return snapshotInProgress;
}

Example 19 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hive by apache.

the class SubstitutionVisitor method splitFilter.

/**
   * Maps a condition onto a target.
   *
   * <p>If condition is stronger than target, returns the residue.
   * If it is equal to target, returns the expression that evaluates to
   * the constant {@code true}. If it is weaker than target, returns
   * {@code null}.</p>
   *
   * <p>The terms satisfy the relation</p>
   *
   * <pre>
   *     {@code condition = target AND residue}
   * </pre>
   *
   * <p>and {@code residue} must be as weak as possible.</p>
   *
   * <p>Example #1: condition stronger than target</p>
   * <ul>
   * <li>condition: x = 1 AND y = 2</li>
   * <li>target: x = 1</li>
   * <li>residue: y = 2</li>
   * </ul>
   *
   * <p>Note that residue {@code x &gt; 0 AND y = 2} would also satisfy the
   * relation {@code condition = target AND residue} but is stronger than
   * necessary, so we prefer {@code y = 2}.</p>
   *
   * <p>Example #2: target weaker than condition (valid, but not currently
   * implemented)</p>
   * <ul>
   * <li>condition: x = 1</li>
   * <li>target: x = 1 OR z = 3</li>
   * <li>residue: NOT (z = 3)</li>
   * </ul>
   *
   * <p>Example #3: condition and target are equivalent</p>
   * <ul>
   * <li>condition: x = 1 AND y = 2</li>
   * <li>target: y = 2 AND x = 1</li>
   * <li>residue: TRUE</li>
   * </ul>
   *
   * <p>Example #4: condition weaker than target</p>
   * <ul>
   * <li>condition: x = 1</li>
   * <li>target: x = 1 AND y = 2</li>
   * <li>residue: null (i.e. no match)</li>
   * </ul>
   *
   * <p>There are many other possible examples. It amounts to solving
   * whether {@code condition AND NOT target} can ever evaluate to
   * true, and therefore is a form of the NP-complete
   * <a href="http://en.wikipedia.org/wiki/Satisfiability">Satisfiability</a>
   * problem.</p>
   */
@VisibleForTesting
public static RexNode splitFilter(final RexBuilder rexBuilder, RexNode condition, RexNode target) {
    // First, try splitting into ORs.
    // Given target    c1 OR c2 OR c3 OR c4
    // and condition   c2 OR c4
    // residue is      NOT c1 AND NOT c3
    // Also deals with case target [x] condition [x] yields residue [true].
    RexNode z = splitOr(rexBuilder, condition, target);
    if (z != null) {
        return z;
    }
    RexNode x = andNot(rexBuilder, target, condition);
    if (mayBeSatisfiable(x)) {
        RexNode x2 = andNot(rexBuilder, condition, target);
        return simplify(rexBuilder, x2);
    }
    return null;
}

Example 20 with VisibleForTesting

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting in project hive by apache.

the class OrcInputFormat method determineSplitStrategies.

@VisibleForTesting
static List<SplitStrategy<?>> determineSplitStrategies(CombinedCtx combinedCtx, Context context, FileSystem fs, Path dir, AcidUtils.Directory dirInfo, List<AcidBaseFileInfo> baseFiles, List<ParsedDelta> parsedDeltas, List<OrcProto.Type> readerTypes, UserGroupInformation ugi, boolean allowSyntheticFileIds) {
    List<SplitStrategy<?>> splitStrategies = new ArrayList<SplitStrategy<?>>();
    SplitStrategy<?> splitStrategy;
    // When no baseFiles, we will just generate a single split strategy and return.
    List<HdfsFileStatusWithId> acidSchemaFiles = new ArrayList<HdfsFileStatusWithId>();
    if (baseFiles.isEmpty()) {
        splitStrategy = determineSplitStrategy(combinedCtx, context, fs, dir, dirInfo, acidSchemaFiles, false, parsedDeltas, readerTypes, ugi, allowSyntheticFileIds);
        if (splitStrategy != null) {
            splitStrategies.add(splitStrategy);
        }
        // return here
        return splitStrategies;
    }
    List<HdfsFileStatusWithId> originalSchemaFiles = new ArrayList<HdfsFileStatusWithId>();
    // Separate the base files into acid schema and non-acid(original) schema files.
    for (AcidBaseFileInfo acidBaseFileInfo : baseFiles) {
        if (acidBaseFileInfo.isOriginal()) {
            originalSchemaFiles.add(acidBaseFileInfo.getHdfsFileStatusWithId());
        } else {
            acidSchemaFiles.add(acidBaseFileInfo.getHdfsFileStatusWithId());
        }
    }
    // Generate split strategy for non-acid schema original files, if any.
    if (!originalSchemaFiles.isEmpty()) {
        splitStrategy = determineSplitStrategy(combinedCtx, context, fs, dir, dirInfo, originalSchemaFiles, true, parsedDeltas, readerTypes, ugi, allowSyntheticFileIds);
        if (splitStrategy != null) {
            splitStrategies.add(splitStrategy);
        }
    }
    // Generate split strategy for acid schema files, if any.
    if (!acidSchemaFiles.isEmpty()) {
        splitStrategy = determineSplitStrategy(combinedCtx, context, fs, dir, dirInfo, acidSchemaFiles, false, parsedDeltas, readerTypes, ugi, allowSyntheticFileIds);
        if (splitStrategy != null) {
            splitStrategies.add(splitStrategy);
        }
    }
    return splitStrategies;
}