Examples with Function com.google.common.base.Function used on opensource projects

Example 31 with Function

use of com.google.common.base.Function in project druid by druid-io.

the class TaskLockbox method findLockPossesForInterval.

/**
   * Return all locks that overlap some search interval.
   */
private List<TaskLockPosse> findLockPossesForInterval(final String dataSource, final Interval interval) {
    giant.lock();
    try {
        final NavigableMap<Interval, TaskLockPosse> dsRunning = running.get(dataSource);
        if (dsRunning == null) {
            // No locks at all
            return Collections.emptyList();
        } else {
            // Tasks are indexed by locked interval, which are sorted by interval start. Intervals are non-overlapping, so:
            final NavigableSet<Interval> dsLockbox = dsRunning.navigableKeySet();
            final Iterable<Interval> searchIntervals = Iterables.concat(// Single interval that starts at or before ours
            Collections.singletonList(dsLockbox.floor(new Interval(interval.getStart(), new DateTime(JodaUtils.MAX_INSTANT)))), // All intervals that start somewhere between our start instant (exclusive) and end instant (exclusive)
            dsLockbox.subSet(new Interval(interval.getStart(), new DateTime(JodaUtils.MAX_INSTANT)), false, new Interval(interval.getEnd(), interval.getEnd()), false));
            return Lists.newArrayList(FunctionalIterable.create(searchIntervals).filter(new Predicate<Interval>() {

                @Override
                public boolean apply(@Nullable Interval searchInterval) {
                    return searchInterval != null && searchInterval.overlaps(interval);
                }
            }).transform(new Function<Interval, TaskLockPosse>() {

                @Override
                public TaskLockPosse apply(Interval interval) {
                    return dsRunning.get(interval);
                }
            }));
        }
    } finally {
        giant.unlock();
    }
}

Example 32 with Function

use of com.google.common.base.Function in project druid by druid-io.

the class PendingTaskBasedWorkerResourceManagementStrategy method doTerminate.

@Override
public boolean doTerminate(WorkerTaskRunner runner) {
    Collection<ImmutableWorkerInfo> zkWorkers = runner.getWorkers();
    synchronized (lock) {
        final WorkerBehaviorConfig workerConfig = workerConfigRef.get();
        if (workerConfig == null) {
            log.warn("No workerConfig available, cannot terminate workers.");
            return false;
        }
        if (!currentlyProvisioning.isEmpty()) {
            log.debug("Already provisioning nodes, Not Terminating any nodes.");
            return false;
        }
        boolean didTerminate = false;
        final Collection<String> workerNodeIds = getWorkerNodeIDs(runner.getLazyWorkers(), workerConfig);
        final Set<String> stillExisting = Sets.newHashSet();
        for (String s : currentlyTerminating) {
            if (workerNodeIds.contains(s)) {
                stillExisting.add(s);
            }
        }
        currentlyTerminating.clear();
        currentlyTerminating.addAll(stillExisting);
        if (currentlyTerminating.isEmpty()) {
            final int maxWorkersToTerminate = maxWorkersToTerminate(zkWorkers, workerConfig);
            final Predicate<ImmutableWorkerInfo> isLazyWorker = ResourceManagementUtil.createLazyWorkerPredicate(config);
            final List<String> laziestWorkerIps = Lists.newArrayList(Collections2.transform(runner.markWorkersLazy(isLazyWorker, maxWorkersToTerminate), new Function<Worker, String>() {

                @Override
                public String apply(Worker zkWorker) {
                    return zkWorker.getIp();
                }
            }));
            if (laziestWorkerIps.isEmpty()) {
                log.debug("Found no lazy workers");
            } else {
                log.info("Terminating %,d lazy workers: %s", laziestWorkerIps.size(), Joiner.on(", ").join(laziestWorkerIps));
                final AutoScalingData terminated = workerConfig.getAutoScaler().terminate(laziestWorkerIps);
                if (terminated != null) {
                    currentlyTerminating.addAll(terminated.getNodeIds());
                    lastTerminateTime = new DateTime();
                    scalingStats.addTerminateEvent(terminated);
                    didTerminate = true;
                }
            }
        } else {
            Duration durSinceLastTerminate = new Duration(lastTerminateTime, new DateTime());
            log.info("%s terminating. Current wait time: %s", currentlyTerminating, durSinceLastTerminate);
            if (durSinceLastTerminate.isLongerThan(config.getMaxScalingDuration().toStandardDuration())) {
                log.makeAlert("Worker node termination taking too long!").addData("millisSinceLastTerminate", durSinceLastTerminate.getMillis()).addData("terminatingCount", currentlyTerminating.size()).emit();
                currentlyTerminating.clear();
            }
        }
        return didTerminate;
    }
}

Example 33 with Function

use of com.google.common.base.Function in project druid by druid-io.

the class EC2AutoScaler method terminate.

@Override
public AutoScalingData terminate(List<String> ips) {
    if (ips.isEmpty()) {
        return new AutoScalingData(Lists.<String>newArrayList());
    }
    DescribeInstancesResult result = amazonEC2Client.describeInstances(new DescribeInstancesRequest().withFilters(new Filter("private-ip-address", ips)));
    List<Instance> instances = Lists.newArrayList();
    for (Reservation reservation : result.getReservations()) {
        instances.addAll(reservation.getInstances());
    }
    try {
        return terminateWithIds(Lists.transform(instances, new Function<Instance, String>() {

            @Override
            public String apply(Instance input) {
                return input.getInstanceId();
            }
        }));
    } catch (Exception e) {
        log.error(e, "Unable to terminate any instances.");
    }
    return null;
}

Example 34 with Function

use of com.google.common.base.Function in project druid by druid-io.

the class ChainedExecutionQueryRunner method run.

@Override
public Sequence<T> run(final Query<T> query, final Map<String, Object> responseContext) {
    final int priority = BaseQuery.getContextPriority(query, 0);
    final Ordering ordering = query.getResultOrdering();
    return new BaseSequence<T, Iterator<T>>(new BaseSequence.IteratorMaker<T, Iterator<T>>() {

        @Override
        public Iterator<T> make() {
            // Make it a List<> to materialize all of the values (so that it will submit everything to the executor)
            ListenableFuture<List<Iterable<T>>> futures = Futures.allAsList(Lists.newArrayList(Iterables.transform(queryables, new Function<QueryRunner<T>, ListenableFuture<Iterable<T>>>() {

                @Override
                public ListenableFuture<Iterable<T>> apply(final QueryRunner<T> input) {
                    if (input == null) {
                        throw new ISE("Null queryRunner! Looks to be some segment unmapping action happening");
                    }
                    return exec.submit(new AbstractPrioritizedCallable<Iterable<T>>(priority) {

                        @Override
                        public Iterable<T> call() throws Exception {
                            try {
                                Sequence<T> result = input.run(query, responseContext);
                                if (result == null) {
                                    throw new ISE("Got a null result! Segments are missing!");
                                }
                                List<T> retVal = Sequences.toList(result, Lists.<T>newArrayList());
                                if (retVal == null) {
                                    throw new ISE("Got a null list of results! WTF?!");
                                }
                                return retVal;
                            } catch (QueryInterruptedException e) {
                                throw Throwables.propagate(e);
                            } catch (Exception e) {
                                log.error(e, "Exception with one of the sequences!");
                                throw Throwables.propagate(e);
                            }
                        }
                    });
                }
            })));
            queryWatcher.registerQuery(query, futures);
            try {
                final Number timeout = query.getContextValue(QueryContextKeys.TIMEOUT, (Number) null);
                return new MergeIterable<>(ordering.nullsFirst(), timeout == null ? futures.get() : futures.get(timeout.longValue(), TimeUnit.MILLISECONDS)).iterator();
            } catch (InterruptedException e) {
                log.warn(e, "Query interrupted, cancelling pending results, query id [%s]", query.getId());
                futures.cancel(true);
                throw new QueryInterruptedException(e);
            } catch (CancellationException e) {
                throw new QueryInterruptedException(e);
            } catch (TimeoutException e) {
                log.info("Query timeout, cancelling pending results for query id [%s]", query.getId());
                futures.cancel(true);
                throw new QueryInterruptedException(e);
            } catch (ExecutionException e) {
                throw Throwables.propagate(e.getCause());
            }
        }

        @Override
        public void cleanup(Iterator<T> tIterator) {
        }
    });
}

Example 35 with Function

use of com.google.common.base.Function in project druid by druid-io.

the class JavaScriptParser method compile.

private static Function<Object, Object> compile(String function) {
    final ContextFactory contextFactory = ContextFactory.getGlobal();
    final Context context = contextFactory.enterContext();
    context.setOptimizationLevel(9);
    final ScriptableObject scope = context.initStandardObjects();
    final org.mozilla.javascript.Function fn = context.compileFunction(scope, function, "fn", 1, null);
    Context.exit();
    return new Function<Object, Object>() {

        public Object apply(Object input) {
            // ideally we need a close() function to discard the context once it is not used anymore
            Context cx = Context.getCurrentContext();
            if (cx == null) {
                cx = contextFactory.enterContext();
            }
            final Object res = fn.call(cx, scope, scope, new Object[] { input });
            return res != null ? Context.toObject(res, scope) : null;
        }
    };
}