Examples with Function com.google.common.base.Function used on opensource projects

Example 61 with Function

use of com.google.common.base.Function in project druid by druid-io.

the class MergeTaskBase method isReady.

/**
   * Checks pre-existing segments in "context" to confirm that this merge query is valid. Specifically, confirm that
   * we are operating on every segment that overlaps the chosen interval.
   */
@Override
public boolean isReady(TaskActionClient taskActionClient) throws Exception {
    // Try to acquire lock
    if (!super.isReady(taskActionClient)) {
        return false;
    } else {
        final Function<DataSegment, String> toIdentifier = new Function<DataSegment, String>() {

            @Override
            public String apply(DataSegment dataSegment) {
                return dataSegment.getIdentifier();
            }
        };
        final Set<String> current = ImmutableSet.copyOf(Iterables.transform(taskActionClient.submit(new SegmentListUsedAction(getDataSource(), getInterval(), null)), toIdentifier));
        final Set<String> requested = ImmutableSet.copyOf(Iterables.transform(segments, toIdentifier));
        final Set<String> missingFromRequested = Sets.difference(current, requested);
        if (!missingFromRequested.isEmpty()) {
            throw new ISE("Merge is invalid: current segment(s) are not in the requested set: %s", Joiner.on(", ").join(missingFromRequested));
        }
        final Set<String> missingFromCurrent = Sets.difference(requested, current);
        if (!missingFromCurrent.isEmpty()) {
            throw new ISE("Merge is invalid: requested segment(s) are not in the current set: %s", Joiner.on(", ").join(missingFromCurrent));
        }
        return true;
    }
}

Example 62 with Function

use of com.google.common.base.Function in project druid by druid-io.

the class MergeTaskBase method run.

@Override
public TaskStatus run(TaskToolbox toolbox) throws Exception {
    final TaskLock myLock = Iterables.getOnlyElement(getTaskLocks(toolbox));
    final ServiceEmitter emitter = toolbox.getEmitter();
    final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder();
    final DataSegment mergedSegment = computeMergedSegment(getDataSource(), myLock.getVersion(), segments);
    final File taskDir = toolbox.getTaskWorkDir();
    try {
        final long startTime = System.currentTimeMillis();
        log.info("Starting merge of id[%s], segments: %s", getId(), Lists.transform(segments, new Function<DataSegment, String>() {

            @Override
            public String apply(DataSegment input) {
                return input.getIdentifier();
            }
        }));
        // download segments to merge
        final Map<DataSegment, File> gettedSegments = toolbox.fetchSegments(segments);
        // merge files together
        final File fileToUpload = merge(toolbox, gettedSegments, new File(taskDir, "merged"));
        emitter.emit(builder.build("merger/numMerged", segments.size()));
        emitter.emit(builder.build("merger/mergeTime", System.currentTimeMillis() - startTime));
        log.info("[%s] : Merged %d segments in %,d millis", mergedSegment.getDataSource(), segments.size(), System.currentTimeMillis() - startTime);
        long uploadStart = System.currentTimeMillis();
        // Upload file
        final DataSegment uploadedSegment = toolbox.getSegmentPusher().push(fileToUpload, mergedSegment);
        emitter.emit(builder.build("merger/uploadTime", System.currentTimeMillis() - uploadStart));
        emitter.emit(builder.build("merger/mergeSize", uploadedSegment.getSize()));
        toolbox.publishSegments(ImmutableList.of(uploadedSegment));
        return TaskStatus.success(getId());
    } catch (Exception e) {
        log.makeAlert(e, "Exception merging[%s]", mergedSegment.getDataSource()).addData("interval", mergedSegment.getInterval()).emit();
        return TaskStatus.failure(getId());
    }
}

Example 63 with Function

use of com.google.common.base.Function in project druid by druid-io.

the class ActionBasedUsedSegmentChecker method findUsedSegments.

@Override
public Set<DataSegment> findUsedSegments(Set<SegmentIdentifier> identifiers) throws IOException {
    // Group by dataSource
    final Map<String, Set<SegmentIdentifier>> identifiersByDataSource = Maps.newTreeMap();
    for (SegmentIdentifier identifier : identifiers) {
        if (!identifiersByDataSource.containsKey(identifier.getDataSource())) {
            identifiersByDataSource.put(identifier.getDataSource(), Sets.<SegmentIdentifier>newHashSet());
        }
        identifiersByDataSource.get(identifier.getDataSource()).add(identifier);
    }
    final Set<DataSegment> retVal = Sets.newHashSet();
    for (Map.Entry<String, Set<SegmentIdentifier>> entry : identifiersByDataSource.entrySet()) {
        final List<Interval> intervals = JodaUtils.condenseIntervals(Iterables.transform(entry.getValue(), new Function<SegmentIdentifier, Interval>() {

            @Override
            public Interval apply(SegmentIdentifier input) {
                return input.getInterval();
            }
        }));
        final List<DataSegment> usedSegmentsForIntervals = taskActionClient.submit(new SegmentListUsedAction(entry.getKey(), null, intervals));
        for (DataSegment segment : usedSegmentsForIntervals) {
            if (identifiers.contains(SegmentIdentifier.fromDataSegment(segment))) {
                retVal.add(segment);
            }
        }
    }
    return retVal;
}

Example 64 with Function

use of com.google.common.base.Function in project core-java by SpineEventEngine.

the class EventBusShould method allow_enrichment_configuration_at_runtime_if_enricher_not_set_previously.

@Test
public void allow_enrichment_configuration_at_runtime_if_enricher_not_set_previously() {
    setUp(null);
    assertNull(eventBus.getEnricher());
    final Class<ProjectId> eventFieldClass = ProjectId.class;
    final Class<String> enrichmentFieldClass = String.class;
    final Function<ProjectId, String> function = new Function<ProjectId, String>() {

        @Override
        public String apply(@Nullable ProjectId input) {
            checkNotNull(input);
            return input.toString();
        }
    };
    eventBus.addFieldEnrichment(eventFieldClass, enrichmentFieldClass, function);
    final EventEnricher enricher = eventBus.getEnricher();
    assertNotNull(enricher);
}

Example 65 with Function

use of com.google.common.base.Function in project intellij-community by JetBrains.

the class PyRemotePackageManagerImpl method getPythonProcessOutput.

@NotNull
@Override
protected ProcessOutput getPythonProcessOutput(@NotNull String helperPath, @NotNull List<String> args, boolean askForSudo, boolean showProgress, @Nullable final String workingDir) throws ExecutionException {
    final Sdk sdk = getSdk();
    final String homePath = sdk.getHomePath();
    if (homePath == null) {
        throw new ExecutionException("Cannot find Python interpreter for SDK " + sdk.getName());
    }
    final SdkAdditionalData sdkData = sdk.getSdkAdditionalData();
    if (sdkData instanceof PyRemoteSdkAdditionalDataBase) {
        //remote interpreter
        final PythonRemoteInterpreterManager manager = PythonRemoteInterpreterManager.getInstance();
        RemoteSdkCredentials remoteSdkCredentials;
        if (CaseCollector.useRemoteCredentials((PyRemoteSdkAdditionalDataBase) sdkData)) {
            try {
                remoteSdkCredentials = ((RemoteSdkAdditionalData) sdkData).getRemoteSdkCredentials(false);
            } catch (InterruptedException e) {
                LOG.error(e);
                remoteSdkCredentials = null;
            } catch (ExecutionException e) {
                throw analyzeException(e, helperPath, args);
            }
            if (manager != null && remoteSdkCredentials != null) {
                if (askForSudo) {
                    askForSudo = !manager.ensureCanWrite(null, remoteSdkCredentials, remoteSdkCredentials.getInterpreterPath());
                }
            } else {
                throw new PyExecutionException(PythonRemoteInterpreterManager.WEB_DEPLOYMENT_PLUGIN_IS_DISABLED, helperPath, args);
            }
        }
        if (manager != null) {
            final List<String> cmdline = new ArrayList<>();
            cmdline.add(homePath);
            cmdline.add(RemoteFile.detectSystemByPath(homePath).createRemoteFile(helperPath).getPath());
            cmdline.addAll(Collections2.transform(args, new Function<String, String>() {

                @Override
                public String apply(@Nullable String input) {
                    return quoteIfNeeded(input);
                }
            }));
            ProcessOutput processOutput;
            do {
                final PyRemoteSdkAdditionalDataBase remoteSdkAdditionalData = (PyRemoteSdkAdditionalDataBase) sdkData;
                final PyRemotePathMapper pathMapper = manager.setupMappings(null, remoteSdkAdditionalData, null);
                try {
                    processOutput = PyRemoteProcessStarterManagerUtil.getManager(remoteSdkAdditionalData).executeRemoteProcess(null, ArrayUtil.toStringArray(cmdline), workingDir, manager, remoteSdkAdditionalData, pathMapper, askForSudo, true);
                } catch (InterruptedException e) {
                    throw new ExecutionException(e);
                }
                if (askForSudo && processOutput.getStderr().contains("sudo: 3 incorrect password attempts")) {
                    continue;
                }
                break;
            } while (true);
            return processOutput;
        } else {
            throw new PyExecutionException(PythonRemoteInterpreterManager.WEB_DEPLOYMENT_PLUGIN_IS_DISABLED, helperPath, args);
        }
    } else {
        throw new PyExecutionException("Invalid remote SDK", helperPath, args);
    }
}