Examples with IndexRecord org.fao.geonet.index.model.gn.IndexRecord used on opensource projects

Example 1 with IndexRecord

use of org.fao.geonet.index.model.gn.IndexRecord in project geonetwork-microservices by geonetwork.

the class IndexingService method collectDbProperties.

/**
 * Initialize an {@link IndexRecord} with all properties from the
 * database of an {@link AbstractMetadata}
 * and return its XML representation as string.
 */
protected static String collectDbProperties(AbstractMetadata r) {
    IndexRecord indexRecord = new IndexRecord(r);
    StringWriter sw = new StringWriter();
    try {
        JAXBContext jaxbContext = JAXBContext.newInstance(IndexRecord.class);
        Marshaller marshaller = jaxbContext.createMarshaller();
        marshaller.setProperty(Marshaller.JAXB_FRAGMENT, Boolean.TRUE);
        marshaller.marshal(indexRecord, sw);
    } catch (JAXBException e) {
        e.printStackTrace();
    }
    return sw.toString();
}

Example 2 with IndexRecord

use of org.fao.geonet.index.model.gn.IndexRecord in project geonetwork-microservices by geonetwork.

the class IndexingService method indexRecords.

/**
 * Read record from database, convert it to {@see IndexRecord} and save it index.
 */
public void indexRecords(Exchange e) throws IndexingRecordException {
    Object body = e.getIn().getBody();
    IndexingReport report = new IndexingReport();
    List<Integer> ids = new ArrayList<>();
    if (body instanceof List) {
        ids = (List<Integer>) body;
    } else if (body instanceof String) {
        ids.add(Integer.parseInt((String) body));
    }
    List<Metadata> records = metadataRepository.findAllById(ids);
    log.info(String.format("Indexing %d records in batch", ids.size()));
    if (ids.size() != records.size()) {
        List<Integer> listOfIds = records.stream().map(Metadata::getId).collect(Collectors.toList());
        List<Integer> ghost = new ArrayList<Integer>(ids);
        ghost.removeAll(listOfIds);
        log.warn(String.format("Error while retrieving records from database. " + "%d record(s) missing. Records are %s." + "Records may have been deleted since we started this indexing task.", ghost.size(), ghost.toString()));
        report.setNumberOfGhostRecords(ghost.size());
        e.getIn().setHeader("NUMBER_OF_GHOST", report.getNumberOfGhostRecords());
    }
    Map<String, List<Metadata>> recordsBySchema = records.stream().collect(Collectors.groupingBy(record -> record.getDataInfo().getSchemaId()));
    recordsBySchema.forEach((schema, schemaRecords) -> {
        log.info(String.format("Indexing %d records in schema %s", schemaRecords.size(), schema));
        IndexRecords indexRecords = collectProperties(schema, schemaRecords, report);
        if (indexRecords.getIndexRecord() != null && indexRecords.getIndexRecord().size() > 0) {
            sendToIndex(indexRecords, report);
        }
    });
    e.getIn().setHeader("NUMBER_OF_RECORDS_INDEXED", records.size());
    e.getIn().setHeader("NUMBER_OF_GHOST_RECORDS", report.getNumberOfGhostRecords());
    e.getIn().setHeader("NUMBER_OF_RECORDS_WITH_ERRORS", report.getNumberOfRecordsWithIndexingErrors());
    e.getIn().setHeader("NUMBER_OF_RECORDS_WITH_UNSUPPORTED_SCHEMA", report.getNumberOfRecordsWithUnsupportedSchema());
}

Example 3 with IndexRecord

use of org.fao.geonet.index.model.gn.IndexRecord in project geonetwork-microservices by geonetwork.

the class JsonLdResponseProcessorImpl method processResponse.

@Override
public void processResponse(HttpSession httpSession, InputStream streamFromServer, OutputStream streamToClient, UserInfo userInfo, String bucket, Boolean addPermissions) throws Exception {
    ObjectMapper objectMapper = JsonUtils.getObjectMapper();
    JsonParser parser = parserForStream(streamFromServer);
    JsonGenerator generator = ResponseParser.jsonFactory.createGenerator(streamToClient);
    // TODO: Check to enable it
    // final Set<String> selections = (addPermissions ?
    // SelectionManager.getManager(ApiUtils.getUserSession(httpSession)).getSelection(bucket)
    // : new HashSet<>());
    Set<String> selections = new HashSet<>();
    ResponseParser responseParser = new ResponseParser();
    generator.writeStartObject();
    // https://schema.org/DataFeed
    generator.writeStringField("@context", "https://schema.org/");
    generator.writeStringField("@type", "DataFeed");
    // name
    // url
    // thumbnailUrl
    // description
    // contentReferenceTime
    // position ie. startindex?
    generator.writeArrayFieldStart("dataFeedElement");
    {
        responseParser.matchHits(parser, generator, doc -> {
            if (addPermissions) {
                addUserInfo(doc, userInfo);
                addSelectionInfo(doc, selections);
            }
            // Remove fields with privileges info
            if (doc.has(IndexRecordFieldNames.source)) {
                ObjectNode sourceNode = (ObjectNode) doc.get(IndexRecordFieldNames.source);
                for (ReservedOperation o : ReservedOperation.values()) {
                    sourceNode.remove(IndexRecordFieldNames.opPrefix + o.getId());
                }
                IndexRecord record = objectMapper.readValue(doc.get(IndexRecordFieldNames.source).toPrettyString(), IndexRecord.class);
                try {
                    ObjectNode node = SchemaOrgConverter.convert(record);
                    generator.writeRawValue(node.toString());
                } catch (Exception ex) {
                    log.error(String.format("JSON-LD conversion returned null result for uuid %s. Check http://localhost:9901/collections/main/items/%s?f=schema.org", doc.get("_id").asText(), doc.get("_id").asText()));
                }
            }
        }, false);
    }
    generator.writeEndArray();
    // generator.writeNumberField("took", 0);
    generator.writeNumberField("size", responseParser.total);
    generator.writeEndObject();
    generator.flush();
    generator.close();
}

Example 4 with IndexRecord

use of org.fao.geonet.index.model.gn.IndexRecord in project geonetwork-microservices by geonetwork.

the class RssConverter method convert.

/**
 * Convert JSON index document _source node to RSS Item.
 *
 * <p>GeoNetwork 3 implementation: See https://github.com/geonetwork/core-geonetwork/blob/master/web/src/main/webapp/xslt/services/rss/rss-utils.xsl
 *
 * <p>Differences:
 * * No GeoRSS support * Link only target the landing page of the record
 *
 * <p>Validation: https://validator.w3.org/feed/check.cgi
 */
public Item convert(ObjectNode doc) throws JsonProcessingException {
    try {
        ObjectMapper objectMapper = new ObjectMapper();
        /*
       * Allow single values to be wrapped in list properties where appropriate (for
       * example, groupPublished is a List<String>, but the JSON may come as {...,
       * "groupPublished" : "all"
       */
        objectMapper = objectMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
        IndexRecord record = objectMapper.readValue(doc.get(IndexRecordFieldNames.source).toString(), IndexRecord.class);
        // https://www.rssboard.org/rss-specification#hrelementsOfLtitemgt
        Item item = new Item();
        Guid guid = new Guid();
        guid.setIsPermaLink(false);
        guid.setValue(record.getMetadataIdentifier());
        item.setGuid(guid);
        item.setTitle(record.getResourceTitle().get(defaultText));
        item.setDescription(buildDescription(record));
        item.setLink(formatterConfiguration.buildLandingPageLink(record.getMetadataIdentifier()));
        Optional<Overview> overview = record.getOverview().stream().findFirst();
        if (overview.isPresent()) {
            Enclosure enclosure = new Enclosure();
            String url = overview.get().getUrl();
            enclosure.setUrl(url);
            String extension = url.substring(url.lastIndexOf(".") + 1);
            enclosure.setType("image/" + extension);
            item.setEnclosure(enclosure);
        }
        // Email address of the author of the item.
        record.getContact().forEach(c -> item.setAuthor(c.getEmail()));
        // Includes the item in one or more categories.
        // Category could be tag ? Was hardcoded in GN3
        // Indicates when the item was published.
        // Publication date first, any resource date and fallback to record date
        List<ResourceDate> resourceDates = record.getResourceDate();
        Optional<ResourceDate> publicationDate = resourceDates.stream().filter(d -> "publication".equals(d.getType())).findFirst();
        Optional<ResourceDate> firstDate = resourceDates.stream().findFirst();
        String pubDate;
        if (publicationDate.isPresent()) {
            pubDate = publicationDate.get().getDate();
        } else if (firstDate.isPresent()) {
            pubDate = firstDate.get().getDate();
        } else {
            pubDate = record.getDateStamp();
        }
        if (StringUtils.isNotEmpty(pubDate)) {
            try {
                if (pubDate.length() == 10) {
                    pubDate += "T12:00:00";
                }
                item.setPubDate(OffsetDateTime.parse(pubDate).format(rssDateFormat));
            } catch (DateTimeParseException parseException) {
                log.warn(String.format("Failed to parse date %s in record %s", pubDate, record.getMetadataIdentifier()));
            }
        }
        return item;
    } catch (JsonProcessingException e) {
        throw e;
    }
}

Example 5 with IndexRecord

use of org.fao.geonet.index.model.gn.IndexRecord in project geonetwork-microservices by geonetwork.

the class DcatConverter method convert.

/**
 * Convert an index document into a DCAT object.
 */
public CatalogRecord convert(JsonNode doc) {
    CatalogRecord catalogRecord = null;
    Dataset dcatDataset = null;
    try {
        IndexRecord record = new ObjectMapper().enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY).readValue(doc.get(IndexRecordFieldNames.source).toString(), IndexRecord.class);
        String recordIdentifier = record.getMetadataIdentifier();
        String recordUri = formatterConfiguration.buildLandingPageLink(record.getMetadataIdentifier());
        Optional<ResourceIdentifier> resourceIdentifier = record.getResourceIdentifier().stream().filter(Objects::nonNull).findFirst();
        // TODO: Define strategy to build IRI
        final String resourceIdentifierUri = resourceIdentifier.isPresent() ? "local:" + resourceIdentifier.get().getCode() : null;
        String language = record.getMainLanguage() == null ? defaultLanguage : record.getMainLanguage();
        String languageUpperCase = language.toUpperCase();
        // TODO: Need language mapper
        String iso2letterLanguage = language.substring(0, 2);
        List<String> resourceLanguage = record.getResourceLanguage();
        List<String> resourceType = record.getResourceType();
        boolean isInspireResource = resourceType.contains("dataset") || resourceType.contains("series") || resourceType.contains("service");
        // TODO: Add multilingual support
        // TODO .resource("https://creativecommons.org/publicdomain/zero/1.0/deed")
        DatasetBuilder datasetBuilder = Dataset.builder().identifier(record.getResourceIdentifier().stream().map(c -> c.getCode()).collect(Collectors.toList())).title(listOfNullable(record.getResourceTitle().get(defaultText))).description(listOfNullable(record.getResourceAbstract().get(defaultText))).landingPage(listOfNullable(DcatDocument.builder().foafDocument(FoafDocument.builder().about(formatterConfiguration.buildLandingPageLink(record.getMetadataIdentifier())).title(record.getResourceTitle().get(defaultText)).build()).build())).provenance(record.getResourceLineage().stream().map(l -> Provenance.builder().provenanceStatement(ProvenanceStatement.builder().label(l.get(defaultText)).build()).build()).collect(Collectors.toList())).type(record.getResourceType().stream().map(t -> new RdfResource(null, "dcat:" + RESSOURCE_TYPE_MAPPING.get(t), null)).collect(Collectors.toList())).modified(toDate(record.getChangeDate())).theme(record.getCodelists().get(topic).stream().map(t -> Subject.builder().skosConcept(SkosConcept.builder().prefLabel(t.getProperties().get(defaultText)).build()).build()).collect(Collectors.toList())).theme(record.getTag().stream().map(t -> Subject.builder().skosConcept(SkosConcept.builder().prefLabel(t.get(defaultText)).build()).build()).collect(Collectors.toList()));
        record.getResourceDate().stream().filter(d -> "creation".equals(d.getType())).forEach(d -> datasetBuilder.created(toDate(d.getDate())));
        record.getResourceDate().stream().filter(d -> "publication".equals(d.getType())).forEach(d -> datasetBuilder.issued(toDate(d.getDate())));
        record.getResourceDate().stream().filter(d -> "revision".equals(d.getType())).forEach(d -> datasetBuilder.modified(toDate(d.getDate())));
        // TODO: Convert to meter ?
        datasetBuilder.spatialResolutionInMeters(record.getResolutionScaleDenominator().stream().map(BigDecimal::new).collect(Collectors.toList()));
        // INSPIRE
        if (record.getSpecificationConformance().size() > 0) {
            datasetBuilder.wasUsedBy(record.getSpecificationConformance().stream().map(c -> DcatActivity.builder().activity(// https://github.com/SEMICeu/iso-19139-to-dcat-ap/blob/master/iso-19139-to-dcat-ap.xsl#L837-L840
            ProvActivity.builder().used(new RdfResource(null, resourceIdentifierUri, null)).qualifiedAssociation(ProvQualifiedAssociation.builder().hadPlan(ProvHadPlan.builder().wasDerivedFrom(new RdfResource("Resource", null, null, c.getTitle(), null)).build()).build()).generated(ProvGenerated.builder().type(new RdfResource("http://inspire.ec.europa.eu/metadata-codelist/DegreeOfConformity/" + INSPIRE_DEGREE_OF_CONFORMITY.get(c.getPass()), null)).build()).build()).build()).collect(Collectors.toList()));
        }
        if (record.getResourceLanguage() != null) {
            // TODO: Where to put resource language ?
            datasetBuilder.language(record.getResourceLanguage().stream().map(l -> new RdfResource(null, "http://publications.europa.eu/resource/authority/language/" + l.toUpperCase(), null)).collect(Collectors.toList()));
        }
        ArrayList<Codelist> updateFrequencyList = record.getCodelists().get(Codelists.maintenanceAndUpdateFrequency);
        if (updateFrequencyList != null && updateFrequencyList.size() > 0) {
            datasetBuilder.accrualPeriodicity(new RdfResource(null, ACCRUAL_PERIODICITY_URI_PREFIX + ACCRUAL_PERIODICITY_TO_ISO.get(updateFrequencyList.get(0).getProperties().get(CommonField.key)), null));
        }
        // <dct:spatial rdf:parseType="Resource">
        datasetBuilder.spatial(record.getGeometries().stream().map(g -> DctSpatial.builder().location(DctLocation.builder().geometry(g).build()).build()).collect(Collectors.toList()));
        datasetBuilder.temporal(record.getResourceTemporalExtentDateRange().stream().map(range -> {
            DctPeriodOfTimeBuilder periodOfTime = DctPeriodOfTime.builder();
            if (StringUtils.isNotEmpty(range.getGte())) {
                periodOfTime.startDate(toDate(range.getGte()));
            }
            if (StringUtils.isNotEmpty(range.getLte())) {
                periodOfTime.endDate(toDate(range.getLte()));
            }
            return DctTemporal.builder().periodOfTime(periodOfTime.build()).build();
        }).collect(Collectors.toList()));
        record.getLinks().stream().forEach(link -> {
            DcatDistributionBuilder dcatDistributionBuilder = DcatDistribution.builder().title(listOfNullable(link.getName())).description(listOfNullable(link.getDescription())).representationTechnique(Subject.builder().skosConcept(SkosConcept.builder().prefLabel(link.getProtocol()).build()).build());
            // TODO: depending on function/protocol build page/accessUrl/downloadUrl
            dcatDistributionBuilder.accessUrl(link.getUrl());
            datasetBuilder.distribution(listOfNullable(DcatDistributionContainer.builder().distribution(dcatDistributionBuilder.build()).build()));
        });
        datasetBuilder.contactPoint(record.getContactForResource().stream().map(contact -> DcatContactPoint.builder().contact(VcardContact.builder().title(contact.getOrganisation()).role(contact.getRole()).hasEmail(contact.getEmail()).build()).build()).collect(Collectors.toList()));
        dcatDataset = datasetBuilder.build();
        catalogRecord = CatalogRecord.builder().identifier(listOfNullable(record.getMetadataIdentifier())).created(toDate(record.getCreateDate())).modified(toDate(record.getChangeDate())).language(listOfNullable(new RdfResource(null, "http://publications.europa.eu/resource/authority/language/" + record.getMainLanguage().toUpperCase()))).primaryTopic(listOfNullable(new ResourceContainer(dcatDataset, null))).build();
    } catch (JsonMappingException e) {
        e.printStackTrace();
    } catch (JsonProcessingException e) {
        e.printStackTrace();
    }
    return catalogRecord;
}