use of io.openk9.datasource.processor.payload.IngestionPayload in project openk9 by smclab.
the class DatasourceProcessor method process.
@Incoming("ingestion")
@Outgoing("ingestion-datasource")
public Uni<IngestionDatasourcePayload> process(Object obj) {
JsonObject jsonObject = obj instanceof JsonObject ? (JsonObject) obj : new JsonObject(new String((byte[]) obj));
long datasourceId = jsonObject.getLong("datasourceId");
Uni<Datasource> datasourceUni = Datasource.findById(datasourceId);
return Panache.withTransaction(() -> datasourceUni.flatMap(datasource -> EnrichPipeline.findByDatasourceId(datasource.getDatasourceId()).onItem().ifNull().continueWith(EnrichPipeline::new).flatMap(enrichPipeline -> {
Uni<List<EnrichItem>> enrichItemUni;
if (enrichPipeline.getEnrichPipelineId() != null) {
enrichItemUni = EnrichItem.findByEnrichPipelineId(enrichPipeline.getEnrichPipelineId()).onItem().ifNull().continueWith(List::of);
} else {
enrichItemUni = Uni.createFrom().item(List.of());
}
return Uni.combine().all().unis(Tenant.findById(datasource.getTenantId()), enrichItemUni).combinedWith((tenantObj, enrichItemList) -> {
Tenant tenant = (Tenant) tenantObj;
IngestionPayload ingestionPayload = jsonObject.mapTo(IngestionPayload.class);
ingestionPayload.setTenantId(tenant.getTenantId());
DatasourceContext datasourceContext = DatasourceContext.of(datasource, tenant, enrichPipeline, enrichItemList);
return IngestionDatasourcePayload.of(ingestionPayload, datasourceContext);
});
})).eventually(() -> Datasource.<Datasource>findById(datasourceId).flatMap(datasource -> {
datasource.setLastIngestionDate(Instant.ofEpochMilli(jsonObject.getLong("parsingDate")));
return datasource.persist();
})));
}
use of io.openk9.datasource.processor.payload.IngestionPayload in project openk9 by smclab.
the class EnrichPipelineProcessor method _adaptIngestionPayload.
private Tuple3<DatasourceContext, ObjectNode, PluginDriverDTO> _adaptIngestionPayload(DatasourceContext datasourceContext, ObjectNode ingestionPayload, PluginDriverDTO pluginDriverDTO) {
ObjectNode newIngestionPayload = ingestionPayload;
if (newIngestionPayload.hasNonNull(io.openk9.core.api.constant.Constants.DATASOURCE_PAYLOAD)) {
JsonNode datasourcePayload = newIngestionPayload.remove(io.openk9.core.api.constant.Constants.DATASOURCE_PAYLOAD);
if (datasourcePayload.isObject()) {
ObjectNode jsonNodes = datasourcePayload.toObjectNode();
for (Map.Entry<String, JsonNode> field : jsonNodes.fields()) {
newIngestionPayload.set(field.getKey(), field.getValue());
}
}
}
if (!ingestionPayload.hasNonNull(Constants.DATASOURCE_NAME)) {
newIngestionPayload = newIngestionPayload.deepCopy().put(Constants.DATASOURCE_NAME, pluginDriverDTO.getName());
}
if (ingestionPayload.hasNonNull(io.openk9.core.api.constant.Constants.TYPE) && !ingestionPayload.get(io.openk9.core.api.constant.Constants.TYPE).toArrayNode().isEmpty()) {
return Tuples.of(datasourceContext, ingestionPayload, pluginDriverDTO);
}
DocumentTypeDTO documentType = pluginDriverDTO.getDefaultDocumentType();
return Tuples.of(datasourceContext, newIngestionPayload.set(io.openk9.core.api.constant.Constants.TYPE, _jsonFactory.createArrayNode().add(documentType.getName())), pluginDriverDTO);
}
Aggregations