Examples with RDFChunkDeserializer org.wikidata.query.rdf.updater.RDFChunkDeserializer used on opensource projects

Example 1 with RDFChunkDeserializer

use of org.wikidata.query.rdf.updater.RDFChunkDeserializer in project wikidata-query-rdf by wikimedia.

the class StreamingUpdate method build.

static StreamingUpdaterConsumer build(StreamingUpdateOptions options, MetricRegistry metrics) {
    RDFChunkDeserializer deser = new RDFChunkDeserializer(new RDFParserSuppliers(RDFParserRegistry.getInstance()));
    KafkaStreamConsumer consumer = KafkaStreamConsumer.build(options.brokers(), options.topic(), options.partition(), options.consumerGroup(), options.batchSize(), deser, parseInitialOffset(options), KafkaStreamConsumerMetricsListener.forRegistry(metrics), options.bufferedInputMessages(), buildFilter(StreamingUpdateOptions.entityFilterPattern(options)));
    HttpClient httpClient = buildHttpClient(getHttpProxyHost(), getHttpProxyPort());
    Retryer<ContentResponse> retryer = buildHttpClientRetryer();
    Duration rdfClientTimeout = RdfRepositoryUpdater.getRdfClientTimeout();
    RdfClient rdfClient = new RdfClient(httpClient, StreamingUpdateOptions.sparqlUri(options), retryer, rdfClientTimeout);
    UrisScheme uris = UrisSchemeFactory.getURISystem();
    return new StreamingUpdaterConsumer(consumer, new RdfRepositoryUpdater(rdfClient, uris), metrics, options.inconsistenciesWarningThreshold());
}