use of com.google.common.collect.ImmutableSortedMap in project druid by druid-io.
the class IndexTask method determineShardSpecs.
/**
* Determines the number of shards for each interval using a hash of queryGranularity timestamp + all dimensions (i.e
* hash-based partitioning). In the future we may want to also support single-dimension partitioning.
*/
private Map<Interval, List<ShardSpec>> determineShardSpecs(final TaskToolbox toolbox, final FirehoseFactory firehoseFactory) throws IOException {
final ObjectMapper jsonMapper = toolbox.getObjectMapper();
final GranularitySpec granularitySpec = ingestionSchema.getDataSchema().getGranularitySpec();
final Granularity queryGranularity = granularitySpec.getQueryGranularity();
final boolean determineNumPartitions = ingestionSchema.getTuningConfig().getNumShards() == null;
final boolean determineIntervals = !ingestionSchema.getDataSchema().getGranularitySpec().bucketIntervals().isPresent();
final Map<Interval, List<ShardSpec>> shardSpecs = Maps.newHashMap();
// if we were given number of shards per interval and the intervals, we don't need to scan the data
if (!determineNumPartitions && !determineIntervals) {
log.info("numShards and intervals provided, skipping determine partition scan");
final SortedSet<Interval> intervals = ingestionSchema.getDataSchema().getGranularitySpec().bucketIntervals().get();
final int numShards = ingestionSchema.getTuningConfig().getNumShards();
for (Interval interval : intervals) {
final List<ShardSpec> intervalShardSpecs = Lists.newArrayListWithCapacity(numShards);
if (numShards > 1) {
for (int i = 0; i < numShards; i++) {
intervalShardSpecs.add(new HashBasedNumberedShardSpec(i, numShards, null, jsonMapper));
}
} else {
intervalShardSpecs.add(NoneShardSpec.instance());
}
shardSpecs.put(interval, intervalShardSpecs);
}
return shardSpecs;
}
// determine intervals containing data and prime HLL collectors
final Map<Interval, Optional<HyperLogLogCollector>> hllCollectors = Maps.newHashMap();
int thrownAway = 0;
log.info("Determining intervals and shardSpecs");
long determineShardSpecsStartMillis = System.currentTimeMillis();
try (final Firehose firehose = firehoseFactory.connect(ingestionSchema.getDataSchema().getParser())) {
while (firehose.hasMore()) {
final InputRow inputRow = firehose.nextRow();
final Interval interval;
if (determineIntervals) {
interval = granularitySpec.getSegmentGranularity().bucket(inputRow.getTimestamp());
} else {
final Optional<Interval> optInterval = granularitySpec.bucketInterval(inputRow.getTimestamp());
if (!optInterval.isPresent()) {
thrownAway++;
continue;
}
interval = optInterval.get();
}
if (!determineNumPartitions) {
// for the interval and don't instantiate a HLL collector
if (!hllCollectors.containsKey(interval)) {
hllCollectors.put(interval, Optional.<HyperLogLogCollector>absent());
}
continue;
}
if (!hllCollectors.containsKey(interval)) {
hllCollectors.put(interval, Optional.of(HyperLogLogCollector.makeLatestCollector()));
}
List<Object> groupKey = Rows.toGroupKey(queryGranularity.bucketStart(inputRow.getTimestamp()).getMillis(), inputRow);
hllCollectors.get(interval).get().add(hashFunction.hashBytes(jsonMapper.writeValueAsBytes(groupKey)).asBytes());
}
}
if (thrownAway > 0) {
log.warn("Unable to to find a matching interval for [%,d] events", thrownAway);
}
final ImmutableSortedMap<Interval, Optional<HyperLogLogCollector>> sortedMap = ImmutableSortedMap.copyOf(hllCollectors, Comparators.intervalsByStartThenEnd());
for (final Map.Entry<Interval, Optional<HyperLogLogCollector>> entry : sortedMap.entrySet()) {
final Interval interval = entry.getKey();
final Optional<HyperLogLogCollector> collector = entry.getValue();
final int numShards;
if (determineNumPartitions) {
final long numRows = new Double(collector.get().estimateCardinality()).longValue();
numShards = (int) Math.ceil((double) numRows / ingestionSchema.getTuningConfig().getTargetPartitionSize());
log.info("Estimated [%,d] rows of data for interval [%s], creating [%,d] shards", numRows, interval, numShards);
} else {
numShards = ingestionSchema.getTuningConfig().getNumShards();
log.info("Creating [%,d] shards for interval [%s]", numShards, interval);
}
final List<ShardSpec> intervalShardSpecs = Lists.newArrayListWithCapacity(numShards);
if (numShards > 1) {
for (int i = 0; i < numShards; i++) {
intervalShardSpecs.add(new HashBasedNumberedShardSpec(i, numShards, null, jsonMapper));
}
} else {
intervalShardSpecs.add(NoneShardSpec.instance());
}
shardSpecs.put(interval, intervalShardSpecs);
}
log.info("Found intervals and shardSpecs in %,dms", System.currentTimeMillis() - determineShardSpecsStartMillis);
return shardSpecs;
}
use of com.google.common.collect.ImmutableSortedMap in project buck by facebook.
the class AndroidResourceDescription method collectInputFiles.
@VisibleForTesting
ImmutableSortedMap<Path, SourcePath> collectInputFiles(final ProjectFilesystem filesystem, Path inputDir) {
final ImmutableSortedMap.Builder<Path, SourcePath> paths = ImmutableSortedMap.naturalOrder();
// We ignore the same files that mini-aapt and aapt ignore.
FileVisitor<Path> fileVisitor = new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attr) throws IOException {
String dirName = dir.getFileName().toString();
// Special case: directory starting with '_' as per aapt.
if (dirName.charAt(0) == '_' || !isPossibleResourceName(dirName)) {
return FileVisitResult.SKIP_SUBTREE;
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attr) throws IOException {
String filename = file.getFileName().toString();
if (isPossibleResourceName(filename)) {
paths.put(MorePaths.relativize(inputDir, file), new PathSourcePath(filesystem, file));
}
return FileVisitResult.CONTINUE;
}
};
try {
filesystem.walkRelativeFileTree(inputDir, fileVisitor);
} catch (IOException e) {
throw new HumanReadableException(e, "Error while searching for android resources in directory %s.", inputDir);
}
return paths.build();
}
use of com.google.common.collect.ImmutableSortedMap in project buck by facebook.
the class AndroidResourceDescription method createSymlinkTree.
private SymlinkTree createSymlinkTree(SourcePathRuleFinder ruleFinder, BuildRuleParams params, Optional<Either<SourcePath, ImmutableSortedMap<String, SourcePath>>> symlinkAttribute, String outputDirName) {
ImmutableMap<Path, SourcePath> links = ImmutableMap.of();
if (symlinkAttribute.isPresent()) {
if (symlinkAttribute.get().isLeft()) {
// If our resources are coming from a `PathSourcePath`, we collect only the inputs we care
// about and pass those in separately, so that that `AndroidResource` rule knows to only
// hash these into it's rule key.
// TODO(k21): This is deprecated and should be disabled or removed.
// Accessing the filesystem during rule creation is problematic because the accesses are
// not cached or tracked in any way.
Preconditions.checkArgument(symlinkAttribute.get().getLeft() instanceof PathSourcePath, "Resource or asset symlink tree can only be built for a PathSourcePath");
PathSourcePath path = (PathSourcePath) symlinkAttribute.get().getLeft();
links = collectInputFiles(path.getFilesystem(), path.getRelativePath());
} else {
links = RichStream.from(symlinkAttribute.get().getRight().entrySet()).map(e -> new AbstractMap.SimpleEntry<>(Paths.get(e.getKey()), e.getValue())).filter(e -> isPossibleResourcePath(e.getKey())).collect(MoreCollectors.toImmutableMap(e -> e.getKey(), e -> e.getValue()));
}
}
Path symlinkTreeRoot = BuildTargets.getGenPath(params.getProjectFilesystem(), params.getBuildTarget(), "%s").resolve(outputDirName);
return new SymlinkTree(params.getBuildTarget(), params.getProjectFilesystem(), symlinkTreeRoot, links, ruleFinder);
}
use of com.google.common.collect.ImmutableSortedMap in project buck by facebook.
the class AndroidBinaryGraphEnhancer method createAdditionalBuildables.
AndroidGraphEnhancementResult createAdditionalBuildables() throws NoSuchBuildTargetException {
ImmutableSortedSet.Builder<BuildRule> enhancedDeps = ImmutableSortedSet.naturalOrder();
enhancedDeps.addAll(originalDeps);
ImmutableList.Builder<BuildRule> additionalJavaLibrariesBuilder = ImmutableList.builder();
AndroidPackageableCollector collector = new AndroidPackageableCollector(originalBuildTarget, buildTargetsToExcludeFromDex, resourcesToExclude, apkModuleGraph);
collector.addPackageables(AndroidPackageableCollector.getPackageableRules(originalDeps));
AndroidPackageableCollection packageableCollection = collector.build();
AndroidPackageableCollection.ResourceDetails resourceDetails = packageableCollection.getResourceDetails();
AndroidNativeLibsGraphEnhancementResult nativeLibsEnhancementResult = nativeLibsEnhancer.enhance(packageableCollection);
Optional<ImmutableMap<APKModule, CopyNativeLibraries>> copyNativeLibraries = nativeLibsEnhancementResult.getCopyNativeLibraries();
if (copyNativeLibraries.isPresent()) {
ruleResolver.addAllToIndex(copyNativeLibraries.get().values());
enhancedDeps.addAll(copyNativeLibraries.get().values());
}
Optional<ImmutableSortedMap<String, String>> sonameMergeMap = nativeLibsEnhancementResult.getSonameMergeMap();
if (sonameMergeMap.isPresent() && nativeLibraryMergeCodeGenerator.isPresent()) {
BuildRule generatorRule = ruleResolver.getRule(nativeLibraryMergeCodeGenerator.get());
GenerateCodeForMergedLibraryMap generateCodeForMergedLibraryMap = new GenerateCodeForMergedLibraryMap(buildRuleParams.withAppendedFlavor(GENERATE_NATIVE_LIB_MERGE_MAP_GENERATED_CODE_FLAVOR).copyReplacingDeclaredAndExtraDeps(Suppliers.ofInstance(ImmutableSortedSet.of(generatorRule)), Suppliers.ofInstance(ImmutableSortedSet.of())), sonameMergeMap.get(), generatorRule);
ruleResolver.addToIndex(generateCodeForMergedLibraryMap);
BuildRuleParams paramsForCompileGenCode = buildRuleParams.withAppendedFlavor(COMPILE_NATIVE_LIB_MERGE_MAP_GENERATED_CODE_FLAVOR).copyReplacingDeclaredAndExtraDeps(Suppliers.ofInstance(ImmutableSortedSet.of(generateCodeForMergedLibraryMap)), Suppliers.ofInstance(ImmutableSortedSet.of()));
DefaultJavaLibrary compileMergedNativeLibMapGenCode = new DefaultJavaLibrary(paramsForCompileGenCode, pathResolver, ruleFinder, ImmutableSet.of(generateCodeForMergedLibraryMap.getSourcePathToOutput()), /* resources */
ImmutableSet.of(), javacOptions.getGeneratedSourceFolderName(), /* proguardConfig */
Optional.empty(), /* postprocessClassesCommands */
ImmutableList.of(), /* exportedDeps */
ImmutableSortedSet.of(), /* providedDeps */
ImmutableSortedSet.of(), JavaLibraryRules.getAbiInputs(ruleResolver, paramsForCompileGenCode.getDeps()), /* trackClassUsage */
false, /* additionalClasspathEntries */
ImmutableSet.of(), new JavacToJarStepFactory(// to 6 in their .buckconfig.
javacOptions.withSourceLevel("7").withTargetLevel("7"), JavacOptionsAmender.IDENTITY), /* resourcesRoot */
Optional.empty(), /* manifest file */
Optional.empty(), /* mavenCoords */
Optional.empty(), ImmutableSortedSet.of(), /* classesToRemoveFromJar */
ImmutableSet.of());
ruleResolver.addToIndex(compileMergedNativeLibMapGenCode);
additionalJavaLibrariesBuilder.add(compileMergedNativeLibMapGenCode);
enhancedDeps.add(compileMergedNativeLibMapGenCode);
}
ImmutableSortedSet<BuildRule> resourceRules = getTargetsAsRules(resourceDetails.getResourcesWithNonEmptyResDir());
ImmutableCollection<BuildRule> rulesWithResourceDirectories = ruleFinder.filterBuildRuleInputs(resourceDetails.getResourceDirectories());
FilteredResourcesProvider filteredResourcesProvider;
boolean needsResourceFiltering = resourceFilter.isEnabled() || resourceCompressionMode.isStoreStringsAsAssets() || !locales.isEmpty();
if (needsResourceFiltering) {
BuildRuleParams paramsForResourcesFilter = buildRuleParams.withAppendedFlavor(RESOURCES_FILTER_FLAVOR).copyReplacingDeclaredAndExtraDeps(Suppliers.ofInstance(ImmutableSortedSet.<BuildRule>naturalOrder().addAll(resourceRules).addAll(rulesWithResourceDirectories).build()), Suppliers.ofInstance(ImmutableSortedSet.of()));
ResourcesFilter resourcesFilter = new ResourcesFilter(paramsForResourcesFilter, resourceDetails.getResourceDirectories(), ImmutableSet.copyOf(resourceDetails.getWhitelistedStringDirectories()), locales, resourceCompressionMode, resourceFilter);
ruleResolver.addToIndex(resourcesFilter);
filteredResourcesProvider = resourcesFilter;
enhancedDeps.add(resourcesFilter);
resourceRules = ImmutableSortedSet.of(resourcesFilter);
} else {
filteredResourcesProvider = new IdentityResourcesProvider(resourceDetails.getResourceDirectories().stream().map(pathResolver::getRelativePath).collect(MoreCollectors.toImmutableList()));
}
// Create the AaptPackageResourcesBuildable.
BuildRuleParams paramsForAaptPackageResources = buildRuleParams.withAppendedFlavor(AAPT_PACKAGE_FLAVOR).copyReplacingDeclaredAndExtraDeps(Suppliers.ofInstance(ImmutableSortedSet.of()), Suppliers.ofInstance(ImmutableSortedSet.of()));
AaptPackageResources aaptPackageResources = new AaptPackageResources(paramsForAaptPackageResources, ruleFinder, ruleResolver, manifest, filteredResourcesProvider, getTargetsAsResourceDeps(resourceDetails.getResourcesWithNonEmptyResDir()), getTargetsAsRules(resourceDetails.getResourcesWithEmptyResButNonEmptyAssetsDir()), packageableCollection.getAssetsDirectories(), resourceUnionPackage, shouldBuildStringSourceMap, skipCrunchPngs, includesVectorDrawables, bannedDuplicateResourceTypes, manifestEntries);
ruleResolver.addToIndex(aaptPackageResources);
enhancedDeps.add(aaptPackageResources);
Optional<PackageStringAssets> packageStringAssets = Optional.empty();
if (resourceCompressionMode.isStoreStringsAsAssets()) {
BuildRuleParams paramsForPackageStringAssets = buildRuleParams.withAppendedFlavor(PACKAGE_STRING_ASSETS_FLAVOR).copyReplacingDeclaredAndExtraDeps(Suppliers.ofInstance(ImmutableSortedSet.<BuildRule>naturalOrder().add(aaptPackageResources).addAll(resourceRules).addAll(rulesWithResourceDirectories).addAll(Iterables.filter(ImmutableList.of(filteredResourcesProvider), BuildRule.class)).build()), Suppliers.ofInstance(ImmutableSortedSet.of()));
packageStringAssets = Optional.of(new PackageStringAssets(paramsForPackageStringAssets, locales, filteredResourcesProvider, aaptPackageResources));
ruleResolver.addToIndex(packageStringAssets.get());
enhancedDeps.add(packageStringAssets.get());
}
// already been added to the APK under test.
if (packageType != PackageType.INSTRUMENTED) {
ImmutableSortedSet<JavaLibrary> buildConfigDepsRules = addBuildConfigDeps(buildRuleParams, packageType, exopackageModes, buildConfigValues, buildConfigValuesFile, ruleResolver, javacOptions, packageableCollection);
enhancedDeps.addAll(buildConfigDepsRules);
additionalJavaLibrariesBuilder.addAll(buildConfigDepsRules);
}
ImmutableList<BuildRule> additionalJavaLibraries = additionalJavaLibrariesBuilder.build();
ImmutableMultimap<APKModule, DexProducedFromJavaLibrary> preDexedLibraries = ImmutableMultimap.of();
if (shouldPreDex) {
preDexedLibraries = createPreDexRulesForLibraries(// TODO(dreiss): Put R.java here.
additionalJavaLibraries, packageableCollection);
}
// Create rule to trim uber R.java sources.
Collection<DexProducedFromJavaLibrary> preDexedLibrariesForResourceIdFiltering = trimResourceIds ? preDexedLibraries.values() : ImmutableList.of();
BuildRuleParams paramsForTrimUberRDotJava = buildRuleParams.withAppendedFlavor(TRIM_UBER_R_DOT_JAVA_FLAVOR).copyReplacingDeclaredAndExtraDeps(Suppliers.ofInstance(ImmutableSortedSet.<BuildRule>naturalOrder().add(aaptPackageResources).addAll(preDexedLibrariesForResourceIdFiltering).build()), Suppliers.ofInstance(ImmutableSortedSet.of()));
TrimUberRDotJava trimUberRDotJava = new TrimUberRDotJava(paramsForTrimUberRDotJava, aaptPackageResources, preDexedLibrariesForResourceIdFiltering, keepResourcePattern);
ruleResolver.addToIndex(trimUberRDotJava);
// Create rule to compile uber R.java sources.
BuildRuleParams paramsForCompileUberRDotJava = buildRuleParams.withAppendedFlavor(COMPILE_UBER_R_DOT_JAVA_FLAVOR).copyReplacingDeclaredAndExtraDeps(Suppliers.ofInstance(ImmutableSortedSet.of(trimUberRDotJava)), Suppliers.ofInstance(ImmutableSortedSet.of()));
JavaLibrary compileUberRDotJava = new DefaultJavaLibrary(paramsForCompileUberRDotJava, pathResolver, ruleFinder, ImmutableSet.of(trimUberRDotJava.getSourcePathToOutput()), /* resources */
ImmutableSet.of(), javacOptions.getGeneratedSourceFolderName(), /* proguardConfig */
Optional.empty(), /* postprocessClassesCommands */
ImmutableList.of(), /* exportedDeps */
ImmutableSortedSet.of(), /* providedDeps */
ImmutableSortedSet.of(), // we can just use its output as the ABI.
JavaLibraryRules.getAbiInputs(ruleResolver, paramsForCompileUberRDotJava.getDeps()), /* trackClassUsage */
false, /* additionalClasspathEntries */
ImmutableSet.of(), new JavacToJarStepFactory(javacOptions.withSourceLevel("7").withTargetLevel("7"), JavacOptionsAmender.IDENTITY), /* resourcesRoot */
Optional.empty(), /* manifest file */
Optional.empty(), /* mavenCoords */
Optional.empty(), ImmutableSortedSet.of(), /* classesToRemoveFromJar */
ImmutableSet.of());
ruleResolver.addToIndex(compileUberRDotJava);
// Create rule to dex uber R.java sources.
BuildRuleParams paramsForDexUberRDotJava = buildRuleParams.withAppendedFlavor(DEX_UBER_R_DOT_JAVA_FLAVOR).copyReplacingDeclaredAndExtraDeps(Suppliers.ofInstance(ImmutableSortedSet.of(compileUberRDotJava)), Suppliers.ofInstance(ImmutableSortedSet.of()));
DexProducedFromJavaLibrary dexUberRDotJava = new DexProducedFromJavaLibrary(paramsForDexUberRDotJava, compileUberRDotJava);
ruleResolver.addToIndex(dexUberRDotJava);
Optional<PreDexMerge> preDexMerge = Optional.empty();
if (shouldPreDex) {
preDexMerge = Optional.of(createPreDexMergeRule(preDexedLibraries, dexUberRDotJava));
enhancedDeps.add(preDexMerge.get());
} else {
enhancedDeps.addAll(getTargetsAsRules(packageableCollection.getJavaLibrariesToDex()));
// If not pre-dexing, AndroidBinary needs to ProGuard and/or dex the compiled R.java.
enhancedDeps.add(compileUberRDotJava);
}
// Add dependencies on all the build rules generating third-party JARs. This is mainly to
// correctly capture deps when a prebuilt_jar forwards the output from another build rule.
enhancedDeps.addAll(ruleFinder.filterBuildRuleInputs(packageableCollection.getPathsToThirdPartyJars()));
Optional<ComputeExopackageDepsAbi> computeExopackageDepsAbi = Optional.empty();
if (!exopackageModes.isEmpty()) {
BuildRuleParams paramsForComputeExopackageAbi = buildRuleParams.withAppendedFlavor(CALCULATE_ABI_FLAVOR).copyReplacingDeclaredAndExtraDeps(Suppliers.ofInstance(enhancedDeps.build()), Suppliers.ofInstance(ImmutableSortedSet.of()));
computeExopackageDepsAbi = Optional.of(new ComputeExopackageDepsAbi(paramsForComputeExopackageAbi, exopackageModes, packageableCollection, copyNativeLibraries, preDexMerge));
ruleResolver.addToIndex(computeExopackageDepsAbi.get());
enhancedDeps.add(computeExopackageDepsAbi.get());
}
return AndroidGraphEnhancementResult.builder().setPackageableCollection(packageableCollection).setAaptPackageResources(aaptPackageResources).setCompiledUberRDotJava(compileUberRDotJava).setCopyNativeLibraries(copyNativeLibraries).setPackageStringAssets(packageStringAssets).setPreDexMerge(preDexMerge).setComputeExopackageDepsAbi(computeExopackageDepsAbi).setClasspathEntriesToDex(ImmutableSet.<SourcePath>builder().addAll(packageableCollection.getClasspathEntriesToDex()).addAll(additionalJavaLibraries.stream().map(BuildRule::getSourcePathToOutput).collect(MoreCollectors.toImmutableList())).build()).setFinalDeps(enhancedDeps.build()).setAPKModuleGraph(apkModuleGraph).build();
}
use of com.google.common.collect.ImmutableSortedMap in project buck by facebook.
the class ProjectGenerator method getHeaderSymlinkTreeHashCode.
private HashCode getHeaderSymlinkTreeHashCode(ImmutableSortedMap<Path, Path> contents, boolean shouldCreateHeadersSymlinks, boolean shouldCreateHeaderMap) {
Hasher hasher = Hashing.sha1().newHasher();
hasher.putBytes(BuckVersion.getVersion().getBytes(Charsets.UTF_8));
String symlinkState = shouldCreateHeadersSymlinks ? "symlinks-enabled" : "symlinks-disabled";
byte[] symlinkStateValue = symlinkState.getBytes(Charsets.UTF_8);
hasher.putInt(symlinkStateValue.length);
hasher.putBytes(symlinkStateValue);
String hmapState = shouldCreateHeaderMap ? "hmap-enabled" : "hmap-disabled";
byte[] hmapStateValue = hmapState.getBytes(Charsets.UTF_8);
hasher.putInt(hmapStateValue.length);
hasher.putBytes(hmapStateValue);
hasher.putInt(0);
for (Map.Entry<Path, Path> entry : contents.entrySet()) {
byte[] key = entry.getKey().toString().getBytes(Charsets.UTF_8);
byte[] value = entry.getValue().toString().getBytes(Charsets.UTF_8);
hasher.putInt(key.length);
hasher.putBytes(key);
hasher.putInt(value.length);
hasher.putBytes(value);
}
return hasher.hash();
}
Aggregations