use of eu.esdihumboldt.hale.common.instance.tools.InstanceCollectionPartitioner in project hale by halestudio.
the class PartitioningWFSWriter method execute.
@Override
protected IOReport execute(final ProgressIndicator progress, final IOReporter reporter) throws IOProviderConfigurationException, IOException {
progress.begin("Upload to WFS-T", IProgressMonitor.UNKNOWN);
try {
progress.setCurrentTask("Partitioning data");
// create the partitioner
InstanceCollectionPartitioner partitioner = StreamGmlWriter.getPartitioner(this, reporter);
// partition the graph
int threshold = getParameter(PARAM_INSTANCES_THRESHOLD).as(Integer.class, DEFAULT_INSTANCES_THRESHOLD);
try (ResourceIterator<InstanceCollection> parts = partitioner.partition(getInstances(), threshold, reporter)) {
if (partitioner.requiresImmediateConsumption()) {
// handle all parts right here, one after another
int partCount = 0;
boolean failed = false;
if (parts.hasNext()) {
while (parts.hasNext() && !progress.isCanceled()) {
partCount++;
SubtaskProgressIndicator partitionProgress = new SubtaskProgressIndicator(progress);
partitionProgress.begin("Assembling part " + partCount, ProgressIndicator.UNKNOWN);
InstanceCollection part = parts.next();
partitionProgress.end();
progress.setCurrentTask("Upload part " + partCount + ((part.hasSize()) ? (" (" + part.size() + " instances)") : ("")));
IOReport report = uploadInstances(part, reporter, new SubtaskProgressIndicator(progress));
if (!report.isSuccess()) {
failed = true;
reporter.error("Upload of part {0} - {1}", partCount, report.getSummary());
} else {
reporter.info("Upload of part {0} - {1}", partCount, report.getSummary());
}
}
reporter.setSuccess(!failed && reporter.getErrors().isEmpty());
if (!reporter.isSuccess()) {
reporter.setSummary("Errors during upload to WFS-T, please see the report.");
} else {
reporter.setSummary("Successfully uploaded data via WFS-T");
}
} else {
reporter.setSuccess(false);
reporter.setSummary("Partitioning yielded no instances to upload");
}
} else {
// can start requests with separate thread (potentially
// threads, but tests with WFSes show that this usually is
// too much to handle for the service)
int partCount = 0;
final AtomicBoolean failed = new AtomicBoolean();
if (parts.hasNext()) {
ExecutorService requestThread = Executors.newSingleThreadExecutor();
while (parts.hasNext() && !progress.isCanceled()) {
partCount++;
SubtaskProgressIndicator partitionProgress = new SubtaskProgressIndicator(// only used for first
progress);
// partitioning
if (partCount == 1)
partitionProgress.begin("Assembling part " + partCount, ProgressIndicator.UNKNOWN);
// not
final InstanceCollection part = parts.next();
// safe
if (partCount == 1)
partitionProgress.end();
progress.setCurrentTask("Upload part " + partCount + ((part.hasSize()) ? (" (" + part.size() + " instances)") : ("")));
final int currentPart = partCount;
requestThread.submit(new Runnable() {
@Override
public void run() {
try {
IOReport report = uploadInstances(part, reporter, new SubtaskProgressIndicator(progress));
if (!report.isSuccess()) {
failed.set(true);
reporter.error(new IOMessageImpl("Upload of part " + currentPart + " - " + report.getSummary(), null));
} else {
reporter.info(new IOMessageImpl("Upload of part " + currentPart + " - " + report.getSummary(), null));
}
} catch (Exception e) {
failed.set(true);
reporter.error(new IOMessageImpl("Upload of part " + currentPart + " failed", e));
}
}
});
}
// wait for requests completion
requestThread.shutdown();
if (!requestThread.awaitTermination(24, TimeUnit.HOURS)) {
reporter.error(new IOMessageImpl("Timeout reached waiting for completion of WFS requests", null));
}
reporter.setSuccess(!failed.get() && reporter.getErrors().isEmpty());
if (!reporter.isSuccess()) {
reporter.setSummary("Errors during upload to WFS-T, please see the report.");
} else {
reporter.setSummary("Successfully uploaded data via WFS-T");
}
} else {
reporter.setSuccess(false);
reporter.setSummary("Partitioning yielded no instances to upload");
}
}
}
} catch (Exception e) {
reporter.error(new IOMessageImpl("Error during attempt to upload to WFS-T", e));
reporter.setSuccess(false);
} finally {
progress.end();
}
return reporter;
}
use of eu.esdihumboldt.hale.common.instance.tools.InstanceCollectionPartitioner in project hale by halestudio.
the class StreamGmlWriter method execute.
/**
* @see AbstractIOProvider#execute(ProgressIndicator, IOReporter)
*/
@Override
protected IOReport execute(ProgressIndicator progress, IOReporter reporter) throws IOProviderConfigurationException, IOException {
init();
if (isThresholdConfigured()) {
InstanceCollectionPartitioner partitioner = getPartitioner(this, reporter);
int threshold = getParameter(PARAM_INSTANCES_THRESHOLD).as(Integer.class, NO_PARTITIONING);
try (ResourceIterator<InstanceCollection> parts = partition(partitioner, getInstances(), threshold, progress, reporter)) {
writeParts(parts, progress, reporter);
}
} else {
write(getInstances(), getTarget().getOutput(), progress, reporter);
}
return reporter;
}
Aggregations