Search in sources :

Example 86 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project hive by apache.

the class HiveEventCounter method addToLogger.

@VisibleForTesting
public void addToLogger(String loggerName, Level level) {
    LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
    loggerConfig.addAppender(this, level, null);
    context.updateLoggers();
}
Also used : LoggerConfig(org.apache.logging.log4j.core.config.LoggerConfig) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 87 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project hive by apache.

the class OrcInputFormat method determineSplitStrategies.

@VisibleForTesting
static List<SplitStrategy<?>> determineSplitStrategies(CombinedCtx combinedCtx, Context context, FileSystem fs, Path dir, AcidUtils.Directory dirInfo, List<AcidBaseFileInfo> baseFiles, List<ParsedDelta> parsedDeltas, List<OrcProto.Type> readerTypes, UserGroupInformation ugi, boolean allowSyntheticFileIds) {
    List<SplitStrategy<?>> splitStrategies = new ArrayList<SplitStrategy<?>>();
    SplitStrategy<?> splitStrategy;
    // When no baseFiles, we will just generate a single split strategy and return.
    List<HdfsFileStatusWithId> acidSchemaFiles = new ArrayList<HdfsFileStatusWithId>();
    if (baseFiles.isEmpty()) {
        splitStrategy = determineSplitStrategy(combinedCtx, context, fs, dir, dirInfo, acidSchemaFiles, false, parsedDeltas, readerTypes, ugi, allowSyntheticFileIds);
        if (splitStrategy != null) {
            splitStrategies.add(splitStrategy);
        }
        // return here
        return splitStrategies;
    }
    List<HdfsFileStatusWithId> originalSchemaFiles = new ArrayList<HdfsFileStatusWithId>();
    // Separate the base files into acid schema and non-acid(original) schema files.
    for (AcidBaseFileInfo acidBaseFileInfo : baseFiles) {
        if (acidBaseFileInfo.isOriginal()) {
            originalSchemaFiles.add(acidBaseFileInfo.getHdfsFileStatusWithId());
        } else {
            acidSchemaFiles.add(acidBaseFileInfo.getHdfsFileStatusWithId());
        }
    }
    // Generate split strategy for non-acid schema original files, if any.
    if (!originalSchemaFiles.isEmpty()) {
        splitStrategy = determineSplitStrategy(combinedCtx, context, fs, dir, dirInfo, originalSchemaFiles, true, parsedDeltas, readerTypes, ugi, allowSyntheticFileIds);
        if (splitStrategy != null) {
            splitStrategies.add(splitStrategy);
        }
    }
    // Generate split strategy for acid schema files, if any.
    if (!acidSchemaFiles.isEmpty()) {
        splitStrategy = determineSplitStrategy(combinedCtx, context, fs, dir, dirInfo, acidSchemaFiles, false, parsedDeltas, readerTypes, ugi, allowSyntheticFileIds);
        if (splitStrategy != null) {
            splitStrategies.add(splitStrategy);
        }
    }
    return splitStrategies;
}
Also used : AcidBaseFileInfo(org.apache.hadoop.hive.ql.io.AcidUtils.AcidBaseFileInfo) HdfsFileStatusWithId(org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId) ArrayList(java.util.ArrayList) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 88 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project buck by facebook.

the class CxxErrorTransformerFactory method transformLine.

@VisibleForTesting
String transformLine(String line) {
    for (Pattern pattern : PATH_PATTERNS) {
        Matcher m = pattern.matcher(line);
        if (m.find()) {
            StringBuilder builder = new StringBuilder();
            String prefix = m.group("prefix");
            if (prefix != null) {
                builder.append(prefix);
            }
            builder.append(transformPath(m.group("path")));
            String suffix = m.group("suffix");
            if (suffix != null) {
                builder.append(suffix);
            }
            return m.replaceAll(Matcher.quoteReplacement(builder.toString()));
        }
    }
    return line;
}
Also used : Pattern(java.util.regex.Pattern) Matcher(java.util.regex.Matcher) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 89 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project buck by facebook.

the class AbstractCxxSourceRuleFactory method requireCompileBuildRule.

@VisibleForTesting
CxxPreprocessAndCompile requireCompileBuildRule(String name, CxxSource source) {
    BuildTarget target = createCompileBuildTarget(name);
    Optional<CxxPreprocessAndCompile> existingRule = getResolver().getRuleOptionalWithType(target, CxxPreprocessAndCompile.class);
    if (existingRule.isPresent()) {
        if (!existingRule.get().getInput().equals(source.getPath())) {
            throw new RuntimeException(String.format("Hash collision for %s; a build rule would have been ignored.", name));
        }
        return existingRule.get();
    }
    return createCompileBuildRule(name, source);
}
Also used : UnflavoredBuildTarget(com.facebook.buck.model.UnflavoredBuildTarget) BuildTarget(com.facebook.buck.model.BuildTarget) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 90 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project buck by facebook.

the class AbstractCxxSourceRuleFactory method createCompileBuildRule.

/**
   * @return a {@link CxxPreprocessAndCompile} rule that preprocesses, compiles, and assembles the
   * given {@link CxxSource}.
   */
@VisibleForTesting
public CxxPreprocessAndCompile createCompileBuildRule(String name, CxxSource source) {
    Preconditions.checkArgument(CxxSourceTypes.isCompilableType(source.getType()));
    BuildTarget target = createCompileBuildTarget(name);
    DepsBuilder depsBuilder = new DepsBuilder(getRuleFinder());
    Compiler compiler = CxxSourceTypes.getCompiler(getCxxPlatform(), source.getType()).resolve(getResolver());
    // Build up the list of compiler flags.
    CxxToolFlags flags = CxxToolFlags.explicitBuilder().addAllPlatformFlags(getPicType().getFlags(compiler)).addAllPlatformFlags(getPlatformCompileFlags(source.getType())).addAllRuleFlags(getRuleCompileFlags(source.getType())).addAllRuleFlags(source.getFlags()).build();
    CompilerDelegate compilerDelegate = new CompilerDelegate(getPathResolver(), getCxxPlatform().getCompilerDebugPathSanitizer(), compiler, flags);
    depsBuilder.add(compilerDelegate);
    depsBuilder.add(source);
    // Build the CxxCompile rule and add it to our sorted set of build rules.
    CxxPreprocessAndCompile result = CxxPreprocessAndCompile.compile(getParams().withBuildTarget(target).copyReplacingDeclaredAndExtraDeps(Suppliers.ofInstance(depsBuilder.build()), Suppliers.ofInstance(ImmutableSortedSet.of())), compilerDelegate, getCompileOutputPath(target, name), source.getPath(), source.getType(), getCxxPlatform().getCompilerDebugPathSanitizer(), getCxxPlatform().getAssemblerDebugPathSanitizer(), getSandboxTree());
    getResolver().addToIndex(result);
    return result;
}
Also used : UnflavoredBuildTarget(com.facebook.buck.model.UnflavoredBuildTarget) BuildTarget(com.facebook.buck.model.BuildTarget) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Aggregations

VisibleForTesting (com.google.common.annotations.VisibleForTesting)808 IOException (java.io.IOException)135 ArrayList (java.util.ArrayList)67 Map (java.util.Map)52 Path (java.nio.file.Path)47 File (java.io.File)41 HashMap (java.util.HashMap)37 Path (org.apache.hadoop.fs.Path)33 List (java.util.List)29 ImmutableList (com.google.common.collect.ImmutableList)28 Matcher (java.util.regex.Matcher)26 HashSet (java.util.HashSet)23 ImmutableMap (com.google.common.collect.ImmutableMap)21 FileStatus (org.apache.hadoop.fs.FileStatus)21 SourcePath (com.facebook.buck.rules.SourcePath)20 FileHandle (org.apache.hadoop.nfs.nfs3.FileHandle)19 DFSClient (org.apache.hadoop.hdfs.DFSClient)18 Nfs3FileAttributes (org.apache.hadoop.nfs.nfs3.Nfs3FileAttributes)18 ImmutableSet (com.google.common.collect.ImmutableSet)17 LinkedHashMap (java.util.LinkedHashMap)17