Examples with PullFromAggregate org.opendatakit.briefcase.pull.aggregate.PullFromAggregate used on opensource projects

Example 1 with PullFromAggregate

use of org.opendatakit.briefcase.pull.aggregate.PullFromAggregate in project briefcase by opendatakit.

the class ExportPanel method export.

private void export() {
    Stream<Job<?>> allJobs = forms.getSelectedForms().stream().map(form -> {
        FormKey key = FormKey.from(form);
        FormMetadata formMetadata = formMetadataPort.fetch(key).orElseThrow(BriefcaseException::new);
        form.setStatusString("Starting to export form");
        String formId = form.getFormDefinition().getFormId();
        ExportConfiguration configuration = forms.getConfiguration(formId);
        FormDefinition formDef = FormDefinition.from((BriefcaseFormDefinition) form.getFormDefinition());
        // TODO Abstract away the subtype of RemoteServer. This should say Optional<RemoteServer>
        Optional<AggregateServer> savedPullSource = RemoteServer.readFromPrefs(appPreferences, pullPanelPrefs, form);
        Job<Void> pullJob = configuration.resolvePullBefore() && savedPullSource.isPresent() ? new PullFromAggregate(http, savedPullSource.get(), appPreferences.getBriefcaseDir().orElseThrow(BriefcaseException::new), false, EventBus::publish, formMetadataPort).pull(form, appPreferences.resolveStartFromLast() ? Optional.of(formMetadata.getCursor()) : Optional.empty()) : Job.noOpSupplier();
        Job<Void> exportJob = Job.run(runnerStatus -> ExportToCsv.export(formMetadataPort, formMetadata, form, formDef, appPreferences.getBriefcaseDir().orElseThrow(BriefcaseException::new), configuration, analytics));
        Job<Void> exportGeoJsonJob = configuration.resolveIncludeGeoJsonExport() ? Job.run(runnerStatus -> ExportToGeoJson.export(formMetadata, formDef, configuration, analytics)) : Job.noOp;
        return Job.run(runnerStatus -> form.clearStatusHistory()).thenRun(pullJob).thenRun(exportJob).thenRun(exportGeoJsonJob);
    });
    JobsRunner.launchAsync(allJobs).onComplete(form::unsetExporting).waitForCompletion();
}

Example 2 with PullFromAggregate

use of org.opendatakit.briefcase.pull.aggregate.PullFromAggregate in project briefcase by opendatakit.

the class PullFormFromAggregate method pullFormFromAggregate.

public static void pullFormFromAggregate(Path storageDir, Optional<String> formId, String username, String password, URL server, boolean resumeLastPull, Optional<LocalDate> startFromDate, boolean includeIncomplete, Optional<Integer> maybeMaxHttpConnections) {
    CliEventsCompanion.attach(log);
    Path briefcaseDir = Common.getOrCreateBriefcaseDir(storageDir);
    FormCache formCache = FormCache.from(briefcaseDir);
    formCache.update();
    BriefcasePreferences appPreferences = BriefcasePreferences.appScoped();
    FormMetadataPort formMetadataPort = FileSystemFormMetadataAdapter.at(briefcaseDir);
    int maxHttpConnections = Optionals.race(maybeMaxHttpConnections, appPreferences.getMaxHttpConnections()).orElse(DEFAULT_HTTP_CONNECTIONS);
    Http http = appPreferences.getHttpProxy().map(host -> CommonsHttp.of(maxHttpConnections, host)).orElseGet(() -> CommonsHttp.of(maxHttpConnections));
    AggregateServer aggregateServer = AggregateServer.authenticated(server, new Credentials(username, password));
    Response<List<RemoteFormDefinition>> response = http.execute(aggregateServer.getFormListRequest());
    if (!response.isSuccess()) {
        System.err.println(response.isRedirection() ? "Error connecting to Aggregate: Redirection detected" : response.isUnauthorized() ? "Error connecting to Aggregate: Wrong credentials" : response.isNotFound() ? "Error connecting to Aggregate: Aggregate not found" : "Error connecting to Aggregate");
        return;
    }
    List<FormStatus> filteredForms = response.orElseThrow(BriefcaseException::new).stream().filter(f -> formId.map(id -> f.getFormId().equals(id)).orElse(true)).map(FormStatus::new).collect(toList());
    if (formId.isPresent() && filteredForms.isEmpty())
        throw new BriefcaseException("Form " + formId.get() + " not found");
    TransferForms forms = TransferForms.empty();
    forms.load(filteredForms);
    forms.selectAll();
    PullFromAggregate pullOp = new PullFromAggregate(http, aggregateServer, briefcaseDir, includeIncomplete, PullFormFromAggregate::onEvent, formMetadataPort);
    JobsRunner.launchAsync(forms.map(form -> pullOp.pull(form, resolveCursor(resumeLastPull, startFromDate, form, formMetadataPort))), PullFormFromAggregate::onError).waitForCompletion();
    System.out.println();
    System.out.println("All operations completed");
    System.out.println();
}

Example 3 with PullFromAggregate

use of org.opendatakit.briefcase.pull.aggregate.PullFromAggregate in project briefcase by opendatakit.

the class Export method export.

public static void export(Path storageDir, String formid, Path exportDir, String baseFilename, boolean exportMedia, boolean overwriteFiles, boolean pullBefore, Optional<LocalDate> startDate, Optional<LocalDate> endDate, Optional<Path> maybePemFile, boolean splitSelectMultiples, boolean includeGeoJsonExport, boolean removeGroupNames, boolean smartAppend) {
    CliEventsCompanion.attach(log);
    Path briefcaseDir = Common.getOrCreateBriefcaseDir(storageDir);
    FormCache formCache = FormCache.from(briefcaseDir);
    formCache.update();
    BriefcasePreferences appPreferences = BriefcasePreferences.appScoped();
    BriefcasePreferences exportPrefs = BriefcasePreferences.forClass(ExportPanel.class);
    BriefcasePreferences pullPrefs = BriefcasePreferences.forClass(ExportPanel.class);
    FormMetadataPort formMetadataPort = FileSystemFormMetadataAdapter.at(briefcaseDir);
    int maxHttpConnections = appPreferences.getMaxHttpConnections().orElse(DEFAULT_HTTP_CONNECTIONS);
    Http http = appPreferences.getHttpProxy().map(host -> CommonsHttp.of(maxHttpConnections, host)).orElseGet(() -> CommonsHttp.of(maxHttpConnections));
    Optional<BriefcaseFormDefinition> maybeFormDefinition = formCache.getForms().stream().filter(form -> form.getFormId().equals(formid)).findFirst();
    createDirectories(exportDir);
    BriefcaseFormDefinition formDefinition = maybeFormDefinition.orElseThrow(() -> new BriefcaseException("Form " + formid + " not found"));
    FormDefinition formDef = FormDefinition.from(formDefinition);
    System.out.println("Exporting form " + formDefinition.getFormName() + " (" + formDefinition.getFormId() + ") to: " + exportDir);
    DateRange dateRange = new DateRange(startDate, endDate);
    ExportConfiguration configuration = ExportConfiguration.Builder.empty().setExportFilename(baseFilename).setExportDir(exportDir).setPemFile(maybePemFile).setDateRange(dateRange).setPullBefore(pullBefore).setOverwriteFiles(overwriteFiles).setExportMedia(exportMedia).setSplitSelectMultiples(splitSelectMultiples).setIncludeGeoJsonExport(includeGeoJsonExport).setRemoveGroupNames(removeGroupNames).setSmartAppend(smartAppend).build();
    FormStatus formStatus = new FormStatus(formDefinition);
    FormKey key = FormKey.from(formStatus);
    FormMetadata formMetadata = formMetadataPort.fetch(key).orElseThrow(BriefcaseException::new);
    Job<Void> pullJob = Job.noOpSupplier();
    if (configuration.resolvePullBefore()) {
        Optional<AggregateServer> server = AggregateServer.readFromPrefs(appPreferences, pullPrefs, formStatus);
        if (server.isPresent()) {
            Optional<Cursor> lastCursor = appPreferences.resolveStartFromLast() ? formMetadataPort.query(lastCursorOf(key)) : Optional.empty();
            pullJob = new PullFromAggregate(http, server.get(), briefcaseDir, false, Export::onEvent, formMetadataPort).pull(formStatus, lastCursor);
        }
    }
    Job<Void> exportJob = Job.run(runnerStatus -> ExportToCsv.export(formMetadataPort, formMetadata, formStatus, formDef, briefcaseDir, configuration));
    Job<Void> exportGeoJsonJob = configuration.resolveIncludeGeoJsonExport() ? Job.run(runnerStatus -> ExportToGeoJson.export(formMetadata, formDef, configuration)) : Job.noOp;
    Job<Void> job = pullJob.thenRun(exportJob).thenRun(exportGeoJsonJob).thenRun(__ -> exportPrefs.put(buildExportDateTimePrefix(formDefinition.getFormId()), LocalDateTime.now().format(ISO_DATE_TIME)));
    JobsRunner.launchAsync(job, Export::onError).waitForCompletion();
    System.out.println();
    System.out.println("All operations completed");
    System.out.println();
}

Example 4 with PullFromAggregate

use of org.opendatakit.briefcase.pull.aggregate.PullFromAggregate in project briefcase by opendatakit.

the class Aggregate method pull.

@Override
public JobsRunner pull(TransferForms forms, BriefcasePreferences appPreferences, FormMetadataPort formMetadataPort) {
    boolean resumeLastPull = appPreferences.resolveStartFromLast();
    PullFromAggregate pullOp = new PullFromAggregate(http, server, appPreferences.getBriefcaseDir().orElseThrow(BriefcaseException::new), false, EventBus::publish, formMetadataPort);
    return JobsRunner.launchAsync(forms.map(form -> {
        FormKey key = FormKey.from(form);
        return pullOp.pull(form, resumeLastPull ? formMetadataPort.query(lastCursorOf(key)) : Optional.empty());
    })).onComplete(() -> EventBus.publish(new PullEvent.PullComplete()));
}