Examples with Function com.google.common.base.Function used on opensource projects

Example 81 with Function

use of com.google.common.base.Function in project titan by thinkaurelius.

the class FulgoraGraphComputer method submit.

@Override
public Future<ComputerResult> submit() {
    if (executed)
        throw Exceptions.computerHasAlreadyBeenSubmittedAVertexProgram();
    else
        executed = true;
    // it is not possible execute a computer if it has no vertex program nor mapreducers
    if (null == vertexProgram && mapReduces.isEmpty())
        throw GraphComputer.Exceptions.computerHasNoVertexProgramNorMapReducers();
    // it is possible to run mapreducers without a vertex program
    if (null != vertexProgram) {
        GraphComputerHelper.validateProgramOnComputer(this, vertexProgram);
        this.mapReduces.addAll(this.vertexProgram.getMapReducers());
    }
    // if the user didn't set desired persistence/resultgraph, then get from vertex program or else, no persistence
    this.persistMode = GraphComputerHelper.getPersistState(Optional.ofNullable(this.vertexProgram), Optional.ofNullable(this.persistMode));
    this.resultGraphMode = GraphComputerHelper.getResultGraphState(Optional.ofNullable(this.vertexProgram), Optional.ofNullable(this.resultGraphMode));
    // determine the legality persistence and result graph options
    if (!this.features().supportsResultGraphPersistCombination(this.resultGraphMode, this.persistMode))
        throw GraphComputer.Exceptions.resultGraphPersistCombinationNotSupported(this.resultGraphMode, this.persistMode);
    memory = new FulgoraMemory(vertexProgram, mapReduces);
    return CompletableFuture.<ComputerResult>supplyAsync(() -> {
        final long time = System.currentTimeMillis();
        if (null != vertexProgram) {
            // ##### Execute vertex program
            vertexMemory = new FulgoraVertexMemory(expectedNumVertices, graph.getIDManager(), vertexProgram);
            // execute the vertex program
            vertexProgram.setup(memory);
            memory.completeSubRound();
            for (int iteration = 1; ; iteration++) {
                vertexMemory.nextIteration(vertexProgram.getMessageScopes(memory));
                jobId = name + "#" + iteration;
                VertexProgramScanJob.Executor job = VertexProgramScanJob.getVertexProgramScanJob(graph, memory, vertexMemory, vertexProgram);
                StandardScanner.Builder scanBuilder = graph.getBackend().buildEdgeScanJob();
                scanBuilder.setJobId(jobId);
                scanBuilder.setNumProcessingThreads(numThreads);
                scanBuilder.setWorkBlockSize(readBatchSize);
                scanBuilder.setJob(job);
                PartitionedVertexProgramExecutor pvpe = new PartitionedVertexProgramExecutor(graph, memory, vertexMemory, vertexProgram);
                try {
                    //Iterates over all vertices and computes the vertex program on all non-partitioned vertices. For partitioned ones, the data is aggregated
                    ScanMetrics jobResult = scanBuilder.execute().get();
                    long failures = jobResult.get(ScanMetrics.Metric.FAILURE);
                    if (failures > 0) {
                        throw new TitanException("Failed to process [" + failures + "] vertices in vertex program iteration [" + iteration + "]. Computer is aborting.");
                    }
                    //Runs the vertex program on all aggregated, partitioned vertices.
                    pvpe.run(numThreads, jobResult);
                    failures = jobResult.getCustom(PartitionedVertexProgramExecutor.PARTITION_VERTEX_POSTFAIL);
                    if (failures > 0) {
                        throw new TitanException("Failed to process [" + failures + "] partitioned vertices in vertex program iteration [" + iteration + "]. Computer is aborting.");
                    }
                } catch (Exception e) {
                    throw new TitanException(e);
                }
                vertexMemory.completeIteration();
                memory.completeSubRound();
                try {
                    if (this.vertexProgram.terminate(this.memory)) {
                        break;
                    }
                } finally {
                    memory.incrIteration();
                    memory.completeSubRound();
                }
            }
        }
        // ##### Execute mapreduce jobs
        // Collect map jobs
        Map<MapReduce, FulgoraMapEmitter> mapJobs = new HashMap<>(mapReduces.size());
        for (MapReduce mapReduce : mapReduces) {
            if (mapReduce.doStage(MapReduce.Stage.MAP)) {
                FulgoraMapEmitter mapEmitter = new FulgoraMapEmitter<>(mapReduce.doStage(MapReduce.Stage.REDUCE));
                mapJobs.put(mapReduce, mapEmitter);
            }
        }
        // Execute map jobs
        jobId = name + "#map";
        VertexMapJob.Executor job = VertexMapJob.getVertexMapJob(graph, vertexMemory, mapJobs);
        StandardScanner.Builder scanBuilder = graph.getBackend().buildEdgeScanJob();
        scanBuilder.setJobId(jobId);
        scanBuilder.setNumProcessingThreads(numThreads);
        scanBuilder.setWorkBlockSize(readBatchSize);
        scanBuilder.setJob(job);
        try {
            ScanMetrics jobResult = scanBuilder.execute().get();
            long failures = jobResult.get(ScanMetrics.Metric.FAILURE);
            if (failures > 0) {
                throw new TitanException("Failed to process [" + failures + "] vertices in map phase. Computer is aborting.");
            }
            failures = jobResult.getCustom(VertexMapJob.MAP_JOB_FAILURE);
            if (failures > 0) {
                throw new TitanException("Failed to process [" + failures + "] individual map jobs. Computer is aborting.");
            }
        } catch (Exception e) {
            throw new TitanException(e);
        }
        // Execute reduce phase and add to memory
        for (Map.Entry<MapReduce, FulgoraMapEmitter> mapJob : mapJobs.entrySet()) {
            FulgoraMapEmitter<?, ?> mapEmitter = mapJob.getValue();
            MapReduce mapReduce = mapJob.getKey();
            // sort results if a map output sort is defined
            mapEmitter.complete(mapReduce);
            if (mapReduce.doStage(MapReduce.Stage.REDUCE)) {
                final FulgoraReduceEmitter<?, ?> reduceEmitter = new FulgoraReduceEmitter<>();
                try (WorkerPool workers = new WorkerPool(numThreads)) {
                    workers.submit(() -> mapReduce.workerStart(MapReduce.Stage.REDUCE));
                    for (final Map.Entry queueEntry : mapEmitter.reduceMap.entrySet()) {
                        workers.submit(() -> mapReduce.reduce(queueEntry.getKey(), ((Iterable) queueEntry.getValue()).iterator(), reduceEmitter));
                    }
                    workers.submit(() -> mapReduce.workerEnd(MapReduce.Stage.REDUCE));
                } catch (Exception e) {
                    throw new TitanException("Exception while executing reduce phase", e);
                }
                //                    mapEmitter.reduceMap.entrySet().parallelStream().forEach(entry -> mapReduce.reduce(entry.getKey(), entry.getValue().iterator(), reduceEmitter));
                // sort results if a reduce output sort is defined
                reduceEmitter.complete(mapReduce);
                mapReduce.addResultToMemory(this.memory, reduceEmitter.reduceQueue.iterator());
            } else {
                mapReduce.addResultToMemory(this.memory, mapEmitter.mapQueue.iterator());
            }
        }
        // #### Write mutated properties back into graph
        Graph resultgraph = graph;
        if (persistMode == Persist.NOTHING && resultGraphMode == ResultGraph.NEW) {
            resultgraph = EmptyGraph.instance();
        } else if (persistMode != Persist.NOTHING && vertexProgram != null && !vertexProgram.getElementComputeKeys().isEmpty()) {
            //First, create property keys in graph if they don't already exist
            TitanManagement mgmt = graph.openManagement();
            try {
                for (String key : vertexProgram.getElementComputeKeys()) {
                    if (!mgmt.containsPropertyKey(key))
                        log.warn("Property key [{}] is not part of the schema and will be created. It is advised to initialize all keys.", key);
                    mgmt.getOrCreatePropertyKey(key);
                }
                mgmt.commit();
            } finally {
                if (mgmt != null && mgmt.isOpen())
                    mgmt.rollback();
            }
            //TODO: Filter based on VertexProgram
            Map<Long, Map<String, Object>> mutatedProperties = Maps.transformValues(vertexMemory.getMutableVertexProperties(), new Function<Map<String, Object>, Map<String, Object>>() {

                @Nullable
                @Override
                public Map<String, Object> apply(@Nullable Map<String, Object> o) {
                    return Maps.filterKeys(o, s -> !NON_PERSISTING_KEYS.contains(s));
                }
            });
            if (resultGraphMode == ResultGraph.ORIGINAL) {
                AtomicInteger failures = new AtomicInteger(0);
                try (WorkerPool workers = new WorkerPool(numThreads)) {
                    List<Map.Entry<Long, Map<String, Object>>> subset = new ArrayList<>(writeBatchSize / vertexProgram.getElementComputeKeys().size());
                    int currentSize = 0;
                    for (Map.Entry<Long, Map<String, Object>> entry : mutatedProperties.entrySet()) {
                        subset.add(entry);
                        currentSize += entry.getValue().size();
                        if (currentSize >= writeBatchSize) {
                            workers.submit(new VertexPropertyWriter(subset, failures));
                            subset = new ArrayList<>(subset.size());
                            currentSize = 0;
                        }
                    }
                    if (!subset.isEmpty())
                        workers.submit(new VertexPropertyWriter(subset, failures));
                } catch (Exception e) {
                    throw new TitanException("Exception while attempting to persist result into graph", e);
                }
                if (failures.get() > 0)
                    throw new TitanException("Could not persist program results to graph. Check log for details.");
            } else if (resultGraphMode == ResultGraph.NEW) {
                resultgraph = graph.newTransaction();
                for (Map.Entry<Long, Map<String, Object>> vprop : mutatedProperties.entrySet()) {
                    Vertex v = resultgraph.vertices(vprop.getKey()).next();
                    for (Map.Entry<String, Object> prop : vprop.getValue().entrySet()) {
                        v.property(VertexProperty.Cardinality.single, prop.getKey(), prop.getValue());
                    }
                }
            }
        }
        // update runtime and return the newly computed graph
        this.memory.setRuntime(System.currentTimeMillis() - time);
        this.memory.complete();
        return new DefaultComputerResult(resultgraph, this.memory);
    });
}

Example 82 with Function

use of com.google.common.base.Function in project gradle by gradle.

the class AbstractCodeQualityPlugin method configureCheckTaskDependents.

private void configureCheckTaskDependents() {
    final String taskBaseName = getTaskBaseName();
    project.getTasks().getByName("check").dependsOn(new Callable() {

        @Override
        public Object call() {
            return Iterables.transform(extension.getSourceSets(), new Function<SourceSet, String>() {

                @Override
                public String apply(SourceSet sourceSet) {
                    return sourceSet.getTaskName(taskBaseName, null);
                }
            });
        }
    });
}

Example 83 with Function

use of com.google.common.base.Function in project bazel by bazelbuild.

the class SkylarkModuleCycleReporter method maybeReportCycle.

@Override
public boolean maybeReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo, boolean alreadyReported, ExtendedEventHandler eventHandler) {
    ImmutableList<SkyKey> pathToCycle = cycleInfo.getPathToCycle();
    ImmutableList<SkyKey> cycle = cycleInfo.getCycle();
    if (pathToCycle.isEmpty()) {
        return false;
    }
    SkyKey lastPathElement = pathToCycle.get(pathToCycle.size() - 1);
    if (alreadyReported) {
        return true;
    } else if (Iterables.all(cycle, IS_SKYLARK_MODULE_SKY_KEY) && // The last element before the cycle has to be a PackageFunction or SkylarkModule.
    (IS_PACKAGE_SKY_KEY.apply(lastPathElement) || IS_SKYLARK_MODULE_SKY_KEY.apply(lastPathElement))) {
        Function printer = new Function<SkyKey, String>() {

            @Override
            public String apply(SkyKey input) {
                if (input.argument() instanceof SkylarkImportLookupValue.SkylarkImportLookupKey) {
                    return ((SkylarkImportLookupValue.SkylarkImportLookupKey) input.argument()).importLabel.toString();
                } else if (input.argument() instanceof PackageIdentifier) {
                    return ((PackageIdentifier) input.argument()) + "/BUILD";
                } else {
                    throw new UnsupportedOperationException();
                }
            }
        };
        StringBuilder cycleMessage = new StringBuilder().append("cycle detected in extension files: ").append("\n    ").append(printer.apply(lastPathElement));
        AbstractLabelCycleReporter.printCycle(cycleInfo.getCycle(), cycleMessage, printer);
        // TODO(bazel-team): it would be nice to pass the Location of the load Statement in the
        // BUILD file.
        eventHandler.handle(Event.error(null, cycleMessage.toString()));
        return true;
    } else if (Iterables.any(cycle, IS_PACKAGE_LOOKUP) && Iterables.any(cycle, IS_WORKSPACE_FILE) && (IS_REPOSITORY_DIRECTORY.apply(lastPathElement) || IS_PACKAGE_SKY_KEY.apply(lastPathElement) || IS_EXTERNAL_PACKAGE.apply(lastPathElement) || IS_LOCAL_REPOSITORY_LOOKUP.apply(lastPathElement))) {
        // We have a cycle in the workspace file, report as such.
        Label fileLabel = (Label) Iterables.getLast(Iterables.filter(cycle, IS_AST_FILE_LOOKUP)).argument();
        String repositoryName = fileLabel.getPackageIdentifier().getRepository().strippedName();
        eventHandler.handle(Event.error(null, "Failed to load Skylark extension '" + fileLabel.toString() + "'.\n" + "It usually happens when the repository is not defined prior to being used.\n" + "Maybe repository '" + repositoryName + "' was defined later in your WORKSPACE file?"));
        return true;
    }
    return false;
}

Example 84 with Function

use of com.google.common.base.Function in project kylo by Teradata.

the class SparkShellApp method sparkConf.

/**
 * Creates the Spark configuration.
 *
 * @return the Spark configuration
 */
@Bean
public SparkConf sparkConf(final Environment env, @Qualifier("sparkShellPort") final int serverPort) {
    final SparkConf conf = new SparkConf().setAppName("SparkShellServer").set("spark.ui.port", Integer.toString(serverPort + 1));
    final Iterable<Map.Entry<String, Object>> properties = FluentIterable.from(Collections.singleton(env)).filter(AbstractEnvironment.class).transformAndConcat(new Function<AbstractEnvironment, Iterable<?>>() {

        @Nullable
        @Override
        public Iterable<?> apply(@Nullable final AbstractEnvironment input) {
            return (input != null) ? input.getPropertySources() : null;
        }
    }).filter(ResourcePropertySource.class).transform(new Function<ResourcePropertySource, Map<String, Object>>() {

        @Nullable
        @Override
        public Map<String, Object> apply(@Nullable final ResourcePropertySource input) {
            return (input != null) ? input.getSource() : null;
        }
    }).transformAndConcat(new Function<Map<String, Object>, Iterable<Map.Entry<String, Object>>>() {

        @Nullable
        @Override
        public Iterable<Map.Entry<String, Object>> apply(@Nullable final Map<String, Object> input) {
            return (input != null) ? input.entrySet() : null;
        }
    }).filter(new Predicate<Map.Entry<String, Object>>() {

        @Override
        public boolean apply(@Nullable final Map.Entry<String, Object> input) {
            return (input != null && input.getKey().startsWith("spark."));
        }
    });
    for (final Map.Entry<String, Object> entry : properties) {
        conf.set(entry.getKey(), entry.getValue().toString());
    }
    return conf;
}

Example 85 with Function

use of com.google.common.base.Function in project cdap by caskdata.

the class FlowProgramRunner method createFlowlets.

/**
   * Starts all flowlets in the flow program.
   * @param program Program to run
   * @param flowSpec The {@link FlowSpecification}.
   * @return A {@link Table} with row as flowlet id, column as instance id, cell as the {@link ProgramController}
   *         for the flowlet.
   */
private Table<String, Integer, ProgramController> createFlowlets(Program program, ProgramOptions options, FlowSpecification flowSpec) {
    Table<String, Integer, ProgramController> flowlets = HashBasedTable.create();
    try {
        for (Map.Entry<String, FlowletDefinition> entry : flowSpec.getFlowlets().entrySet()) {
            ProgramOptions flowletOptions = resolveFlowletOptions(options, entry.getKey());
            int instanceCount = entry.getValue().getInstances();
            for (int instanceId = 0; instanceId < instanceCount; instanceId++) {
                flowlets.put(entry.getKey(), instanceId, startFlowlet(program, createFlowletOptions(entry.getKey(), instanceId, instanceCount, flowletOptions)));
            }
        }
    } catch (Throwable t) {
        try {
            // Need to stop all started flowlets
            Futures.successfulAsList(Iterables.transform(flowlets.values(), new Function<ProgramController, ListenableFuture<?>>() {

                @Override
                public ListenableFuture<?> apply(ProgramController controller) {
                    return controller.stop();
                }
            })).get();
        } catch (Exception e) {
            LOG.error("Fail to stop all flowlets on failure.");
        }
        throw Throwables.propagate(t);
    }
    return flowlets;
}