Examples with EnrichItem io.openk9.datasource.model.EnrichItem used on opensource projects

Example 1 with EnrichItem

use of io.openk9.datasource.model.EnrichItem in project openk9 by smclab.

the class JsEnrichProcessor method process.

@Override
public Mono<ObjectNode> process(ObjectNode objectNode, DatasourceContext context, EnrichItem enrichItem, PluginDriverDTO pluginDriverName) {
    return Mono.defer(() -> {
        JsonNode datasourceConfiguration = _jsonFactory.fromJsonToJsonNode(enrichItem.getJsonConfig());
        if (!datasourceConfiguration.isObject()) {
            return Mono.error(new RuntimeException("jsonConfig must be an instance of ObjectNode " + datasourceConfiguration.toString()));
        }
        ObjectNode request = prepareRequestRawContent(objectNode, datasourceConfiguration.toObjectNode(), context, pluginDriverName);
        return Mono.from(_httpClient.request(getMethod(), getPath(), request.toString(), getHeaders())).map(_jsonFactory::fromJsonToJsonNode).map(JsonNode::toObjectNode).map(objectNode::merge);
    });
}

Example 2 with EnrichItem

use of io.openk9.datasource.model.EnrichItem in project openk9 by smclab.

the class DatasourceProcessor method process.

@Incoming("ingestion")
@Outgoing("ingestion-datasource")
public Uni<IngestionDatasourcePayload> process(Object obj) {
    JsonObject jsonObject = obj instanceof JsonObject ? (JsonObject) obj : new JsonObject(new String((byte[]) obj));
    long datasourceId = jsonObject.getLong("datasourceId");
    Uni<Datasource> datasourceUni = Datasource.findById(datasourceId);
    return Panache.withTransaction(() -> datasourceUni.flatMap(datasource -> EnrichPipeline.findByDatasourceId(datasource.getDatasourceId()).onItem().ifNull().continueWith(EnrichPipeline::new).flatMap(enrichPipeline -> {
        Uni<List<EnrichItem>> enrichItemUni;
        if (enrichPipeline.getEnrichPipelineId() != null) {
            enrichItemUni = EnrichItem.findByEnrichPipelineId(enrichPipeline.getEnrichPipelineId()).onItem().ifNull().continueWith(List::of);
        } else {
            enrichItemUni = Uni.createFrom().item(List.of());
        }
        return Uni.combine().all().unis(Tenant.findById(datasource.getTenantId()), enrichItemUni).combinedWith((tenantObj, enrichItemList) -> {
            Tenant tenant = (Tenant) tenantObj;
            IngestionPayload ingestionPayload = jsonObject.mapTo(IngestionPayload.class);
            ingestionPayload.setTenantId(tenant.getTenantId());
            DatasourceContext datasourceContext = DatasourceContext.of(datasource, tenant, enrichPipeline, enrichItemList);
            return IngestionDatasourcePayload.of(ingestionPayload, datasourceContext);
        });
    })).eventually(() -> Datasource.<Datasource>findById(datasourceId).flatMap(datasource -> {
        datasource.setLastIngestionDate(Instant.ofEpochMilli(jsonObject.getLong("parsingDate")));
        return datasource.persist();
    })));
}

Example 3 with EnrichItem

use of io.openk9.datasource.model.EnrichItem in project openk9 by smclab.

the class EnrichPipelineProcessor method _mapToEnrichProcessorContext.

private ObjectNode _mapToEnrichProcessorContext(Tuple3<DatasourceContext, ObjectNode, PluginDriverDTO> t3) {
    List<String> dependencies = t3.getT1().getEnrichItems().stream().map(EnrichItem::getServiceName).collect(Collectors.toList());
    ObjectNode objectNode = _jsonFactory.createObjectNode();
    objectNode.put("pluginDriver", _jsonFactory.fromObjectToJsonNode(t3.getT3()));
    objectNode.put("dependencies", _jsonFactory.fromObjectToJsonNode(dependencies));
    objectNode.put("datasourceContext", _jsonFactory.fromObjectToJsonNode(t3.getT1()));
    objectNode.put("payload", t3.getT2());
    return objectNode;
}

Example 4 with EnrichItem

use of io.openk9.datasource.model.EnrichItem in project openk9 by smclab.

the class InitDataActivator method activate.

@Activate
void activate(BundleContext bundleContext) throws Exception {
    Bundle bundle = bundleContext.getBundle();
    Tenant tenant = _jsonFileToObj(bundle, "tenant.json", Tenant.class);
    Datasource datasource = _jsonFileToObj(bundle, "datasource.json", Datasource.class);
    EnrichItem enrichItem = _jsonFileToObj(bundle, "enrich-item.json", EnrichItem.class);
    EnrichPipeline enrichPipeline = _jsonFileToObj(bundle, "enrich-pipeline.json", EnrichPipeline.class);
    _disposable = _tenantRepository.findByVirtualHost(tenant.getVirtualHost()).thenEmpty(_tenantRepository.insert(tenant).then(_datasourceRepository.insert(datasource)).then(_enrichItemRepository.insert(enrichItem)).then(_enrichPipelineRepository.insert(enrichPipeline)).then(Mono.empty())).subscribe();
}

Example 5 with EnrichItem

use of io.openk9.datasource.model.EnrichItem in project openk9 by smclab.

the class BaseNerEnrichProcessor method process.

@Override
public Mono<ObjectNode> process(ObjectNode objectNode, DatasourceContext context, EnrichItem enrichItem, PluginDriverDTO pluginDriverName) {
    return Mono.defer(() -> {
        JsonNode datasourceConfiguration = _jsonFactory.fromJsonToJsonNode(enrichItem.getJsonConfig());
        if (!datasourceConfiguration.isObject()) {
            return Mono.error(new RuntimeException("jsonConfig must be an instance of ObjectNode " + datasourceConfiguration.toString()));
        }
        ObjectNode request = prepareRequestRawContent(objectNode, datasourceConfiguration.toObjectNode(), context, pluginDriverName);
        return Mono.from(_httpClient.request(getMethod(), getPath(), request.toString(), getHeaders())).retry(5).map(_jsonFactory::fromJsonToJsonNode).map(JsonNode::toObjectNode).map(jsonNodes -> {
            jsonNodes.put(Constants.TENANT_ID, context.getTenant().getTenantId());
            jsonNodes.put(Constants.DATASOURCE_ID, context.getDatasource().getDatasourceId());
            jsonNodes.put(Constants.CONTENT_ID, objectNode.get(Constants.CONTENT_ID));
            jsonNodes.put(Constants.RAW_CONTENT, objectNode.get(Constants.RAW_CONTENT));
            jsonNodes.put(Constants.INGESTION_ID, objectNode.get(Constants.INGESTION_ID));
            return jsonNodes;
        }).flatMap(this::_getEntityOrCreate).map(entities -> objectNode.set(entitiesField(), entities));
    });
}