Examples with IdentifiableInstanceReference eu.esdihumboldt.hale.common.instance.model.IdentifiableInstanceReference used on opensource projects

Example 1 with IdentifiableInstanceReference

use of eu.esdihumboldt.hale.common.instance.model.IdentifiableInstanceReference in project hale by halestudio.

the class StoreInstancesJob method run.

/**
 * @see Job#run(IProgressMonitor)
 */
@Override
public IStatus run(IProgressMonitor monitor) {
    boolean exactProgress = instances.hasSize();
    monitor.beginTask("Store instances in database", (exactProgress) ? (instances.size()) : (IProgressMonitor.UNKNOWN));
    AtomicInteger count = new AtomicInteger(0);
    TObjectIntHashMap<QName> typeCount = new TObjectIntHashMap<>();
    if (report != null) {
        // set the correct start time
        report.setStartTime(new Date());
    }
    // get database connection
    DatabaseReference<ODatabaseDocumentTx> ref = database.openWrite();
    ODatabaseDocumentTx db = ref.getDatabase();
    ATransaction trans = log.begin("Store instances in database");
    try {
        // use intent
        db.declareIntent(new OIntentMassiveInsert());
        // Find all the InstanceProcessors to feed them the stored Instances
        final List<InstanceProcessor> processors;
        if (doProcessing) {
            final InstanceProcessingExtension ext = new InstanceProcessingExtension(serviceProvider);
            processors = ext.getInstanceProcessors();
        } else {
            processors = Collections.emptyList();
        }
        BrowseOrientInstanceCollection browser = new BrowseOrientInstanceCollection(database, null, DataSet.SOURCE);
        final InstanceIndexService indexService;
        if (doProcessing) {
            indexService = serviceProvider.getService(InstanceIndexService.class);
        } else {
            indexService = null;
        }
        // TODO decouple next() and save()?
        SimpleLogContext.withLog(report, () -> {
            if (report != null && instances instanceof LogAware) {
                ((LogAware) instances).setLog(report);
            }
            ResourceIterator<Instance> it = instances.iterator();
            int size = instances.size();
            try {
                while (it.hasNext() && !monitor.isCanceled()) {
                    // last count update
                    long lastUpdate = 0;
                    if (report != null && instances instanceof LogAware) {
                        ((LogAware) instances).setLog(report);
                    }
                    Instance instance = it.next();
                    // further processing before storing
                    processInstance(instance);
                    // get/create OInstance
                    OInstance conv = ((instance instanceof OInstance) ? ((OInstance) instance) : (new OInstance(instance)));
                    conv.setInserted(true);
                    // update the instance to store, e.g. generating
                    // metadata
                    updateInstance(conv);
                    ODatabaseRecordThreadLocal.INSTANCE.set(db);
                    // configure the document
                    ODocument doc = conv.configureDocument(db);
                    // and save it
                    doc.save();
                    // Create an InstanceReference for the saved instance
                    // and
                    // feed it to all known InstanceProcessors. The
                    // decoration
                    // with ResolvableInstanceReference allows the
                    // InstanceProcessors to resolve the instances if
                    // required.
                    OrientInstanceReference oRef = new OrientInstanceReference(doc.getIdentity(), conv.getDataSet(), conv.getDefinition());
                    IdentifiableInstanceReference idRef = new IdentifiableInstanceReference(oRef, doc.getIdentity());
                    ResolvableInstanceReference resolvableRef = new ResolvableInstanceReference(idRef, browser);
                    processors.forEach(p -> p.process(instance, resolvableRef));
                    if (indexService != null) {
                        indexService.add(instance, resolvableRef);
                    }
                    count.incrementAndGet();
                    TypeDefinition type = instance.getDefinition();
                    if (type != null) {
                        typeCount.adjustOrPutValue(type.getName(), 1, 1);
                    }
                    if (exactProgress) {
                        monitor.worked(1);
                    }
                    long now = System.currentTimeMillis();
                    if (now - lastUpdate > 100) {
                        // only update every 100
                        // milliseconds
                        monitor.subTask(MessageFormat.format("{0}{1} instances processed", String.valueOf(count.get()), size != InstanceCollection.UNKNOWN_SIZE ? "/" + String.valueOf(size) : ""));
                        lastUpdate = now;
                    }
                }
            } finally {
                it.close();
                if (report != null && instances instanceof LogAware) {
                    ((LogAware) instances).setLog(null);
                }
            }
        });
        db.declareIntent(null);
    } catch (RuntimeException e) {
        if (report != null) {
            reportTypeCount(report, typeCount);
            report.error(new MessageImpl("Error storing instances in database", e));
            report.setSuccess(false);
            reportHandler.publishReport(report);
        }
        throw e;
    } finally {
        ref.dispose();
        trans.end();
        /*
			 * Reset instances to prevent memory leak. It seems Eclipse
			 * internally holds a reference to the job (in JobInfo and/or
			 * ProgressMonitorFocusJobDialog) and this results in the instance
			 * collection not being garbage collected. This is especially bad,
			 * if an in-memory instance collection is used, e.g. a
			 * DefaultInstanceCollection that is used when loading a Shapefile.
			 */
        instances = null;
    }
    try {
        onComplete();
    } catch (RuntimeException e) {
        String message = "Error while post processing stored instances";
        if (report != null) {
            report.error(new MessageImpl(message, e));
        } else {
            log.error(message, e);
        }
    }
    String message = MessageFormat.format("Stored {0} instances in the database.", count);
    if (monitor.isCanceled()) {
        String warn = "Loading instances was canceled, incomplete data set in the database.";
        if (report != null) {
            report.warn(new MessageImpl(warn, null));
        } else {
            log.warn(warn);
        }
    }
    if (report != null) {
        reportTypeCount(report, typeCount);
        report.setSuccess(true);
        report.setSummary(message);
        reportHandler.publishReport(report);
    } else {
        log.info(message);
    }
    monitor.done();
    return new Status((monitor.isCanceled()) ? (IStatus.CANCEL) : (IStatus.OK), "eu.esdihumboldt.hale.common.instance.orient", message);
}

Example 2 with IdentifiableInstanceReference

use of eu.esdihumboldt.hale.common.instance.model.IdentifiableInstanceReference in project hale by halestudio.

the class StreamGmlWriter method partitionByExtent.

private void partitionByExtent(ProgressIndicator progress, IOReporter reporter) throws IOException {
    int maxNodes = getParameter(PARAM_PARTITION_BY_EXTENT_MAX_NODES).as(Integer.class, 1000);
    String mode = getParameter(PARAM_PARTITION_BY_EXTENT_MODE).as(String.class, PARTITION_BY_EXTENT_MODE_DATASET);
    final SubtaskProgressIndicator qtProgress = new SubtaskProgressIndicator(progress) {

        @Override
        protected String getCombinedTaskName(String taskName, String subtaskName) {
            return taskName + " (" + subtaskName + ")";
        }
    };
    // Map for instances that either contain no or multiple geometries
    Map<String, InstanceReference> unhandledInstances = new HashMap<>();
    QuadtreeBuilder<Point, InstanceReference> builder = new QuadtreeBuilder<>();
    try (ResourceIterator<Instance> it = getInstances().iterator()) {
        qtProgress.begin("Collecting geometries", getInstances().size());
        final XMLInspector gadget = new XMLInspector();
        int i = 0;
        while (it.hasNext()) {
            Instance inst = it.next();
            InstanceReference instRef = getInstances().getReference(inst);
            InstanceTraverser traverser = new DepthFirstInstanceTraverser();
            GeometryFinder finder = new GeometryFinder(getTargetCRS());
            traverser.traverse(inst, finder);
            List<GeometryProperty<?>> geoms = finder.getGeometries();
            if (geoms.isEmpty() || geoms.size() > 1) {
                unhandledInstances.put(gadget.getIdentity(inst), instRef);
            } else {
                GeometryProperty<?> geomProperty = geoms.get(0);
                Geometry geom = geomProperty.getGeometry();
                Point centroid;
                switch(mode) {
                    case PARTITION_BY_EXTENT_MODE_WORLD:
                        CoordinateReferenceSystem sourceCrs = geomProperty.getCRSDefinition().getCRS();
                        CodeDefinition wgs84 = new CodeDefinition("EPSG:4326");
                        try {
                            MathTransform toWgs84 = CRS.findMathTransform(sourceCrs, wgs84.getCRS());
                            Geometry geomWgs84 = JTS.transform(geom, toWgs84);
                            centroid = geomWgs84.getCentroid();
                        } catch (FactoryException | MismatchedDimensionException | TransformException e) {
                            log.error("Unable to transform geometry to WGS 84", e);
                            throw new IllegalStateException(e.getMessage(), e);
                        }
                        break;
                    case PARTITION_BY_EXTENT_MODE_DATASET:
                    // fall through to default
                    default:
                        centroid = geom.getCentroid();
                }
                builder.add(centroid, new IdentifiableInstanceReference(instRef, gadget.getIdentity(inst)));
            }
            qtProgress.advance(1);
            if (++i % 100 == 0) {
                qtProgress.setCurrentTask(MessageFormat.format("{0} instances processed", i));
            }
        }
        qtProgress.setCurrentTask("Building quadtree");
        FixedBoundaryQuadtree<InstanceReference> qt;
        switch(mode) {
            case PARTITION_BY_EXTENT_MODE_DATASET:
                qt = builder.build(maxNodes);
                break;
            case PARTITION_BY_EXTENT_MODE_WORLD:
                Envelope world = new Envelope(-180, 180, -90, 90);
                qt = builder.build(maxNodes, world);
                break;
            default:
                log.error(MessageFormat.format("Unrecognized extent partitioning mode \"{0}\", using dataset boundaries", mode));
                qt = builder.build(maxNodes);
        }
        qtProgress.setCurrentTask("Performing spatial partitioning");
        final Map<String, String> idToKeyMapping = new HashMap<>();
        final Map<String, Collection<InstanceReference>> keyToRefsMapping = new HashMap<>();
        // Instances without geometry or with multiple geometries
        keyToRefsMapping.put(ExtentPartsHandler.KEY_NO_GEOMETRY, unhandledInstances.values());
        unhandledInstances.keySet().stream().forEach(id -> idToKeyMapping.put(id, ExtentPartsHandler.KEY_NO_GEOMETRY));
        buildMappings(qt, idToKeyMapping, keyToRefsMapping);
        // Partition source instances based on quadtree tiles
        Iterator<InstanceCollection> collIt = new Iterator<InstanceCollection>() {

            private final Queue<String> keySet = new LinkedList<>(keyToRefsMapping.keySet());

            @Override
            public boolean hasNext() {
                return !keySet.isEmpty();
            }

            @Override
            public InstanceCollection next() {
                String key = keySet.poll();
                Collection<InstanceReference> refs = keyToRefsMapping.get(key);
                InstanceCollection instColl = new DefaultInstanceCollection(refs.stream().map(ref -> getInstances().getInstance(IdentifiableInstanceReference.getRootReference(ref))).collect(Collectors.toList()));
                return new ExtentPartsHandler.TreeKeyDecorator(instColl, key);
            }
        };
        final Map<String, URI> keyToTargetMapping = new HashMap<>();
        keyToRefsMapping.keySet().stream().forEach(k -> keyToTargetMapping.put(k, new File(ExtentPartsHandler.getTargetFilename(k, getTarget().getLocation())).toURI()));
        final ExtentPartsHandler handler = new ExtentPartsHandler(keyToTargetMapping, idToKeyMapping);
        qtProgress.end();
        try {
            writeParts(collIt, handler, progress, reporter);
        } catch (XMLStreamException e) {
            throw new IOException(e.getMessage(), e);
        }
    }
}

Example 3 with IdentifiableInstanceReference

use of eu.esdihumboldt.hale.common.instance.model.IdentifiableInstanceReference in project hale by halestudio.

the class InstanceIndexUpdateServiceImpl method reindex.

private void reindex() {
    getIndexService().clearIndexedValues();
    InstanceService is = serviceProvider.getService(InstanceService.class);
    InstanceCollection source = is.getInstances(DataSet.SOURCE);
    try (ResourceIterator<Instance> it = source.iterator()) {
        while (it.hasNext()) {
            Instance i = it.next();
            InstanceReference ref = source.getReference(i);
            if (Identifiable.is(ref)) {
                ref = new IdentifiableInstanceReference(ref, Identifiable.getId(ref));
            }
            ResolvableInstanceReference rir = new ResolvableInstanceReference(ref, source);
            getIndexService().add(i, rir);
        }
    }
}

Example 4 with IdentifiableInstanceReference

use of eu.esdihumboldt.hale.common.instance.model.IdentifiableInstanceReference in project hale by halestudio.

the class InstanceIndexServiceImpl method add.

/**
 * @see eu.esdihumboldt.hale.common.instance.index.InstanceIndexService#add(eu.esdihumboldt.hale.common.instance.model.Instance,
 *      eu.esdihumboldt.hale.common.instance.model.InstanceCollection)
 */
@Override
public void add(Instance instance, InstanceCollection instances) {
    InstanceReference ref;
    if (Identifiable.is(instance)) {
        ref = new IdentifiableInstanceReference(instances.getReference(instance), Identifiable.getId(instance));
    } else {
        ref = instances.getReference(instance);
    }
    ResolvableInstanceReference rir = new ResolvableInstanceReference(ref, instances);
    getIndex(instance.getDefinition().getName()).add(rir, instance);
}