Examples with BatchSource com.hazelcast.jet.pipeline.BatchSource used on opensource projects

Example 1 with BatchSource

use of com.hazelcast.jet.pipeline.BatchSource in project hazelcast by hazelcast.

the class CommonElasticSourcesTest method given_documents_when_readFromElasticSourceWithQuery_then_resultHasMatchingDocuments.

@Test
public void given_documents_when_readFromElasticSourceWithQuery_then_resultHasMatchingDocuments() {
    indexDocument("my-index", of("name", "Frantisek"));
    indexDocument("my-index", of("name", "Vladimir"));
    Pipeline p = Pipeline.create();
    BatchSource<String> source = new ElasticSourceBuilder<>().clientFn(elasticClientSupplier()).searchRequestFn(() -> new SearchRequest("my-index").source(new SearchSourceBuilder().query(QueryBuilders.matchQuery("name", "Frantisek")))).mapToItemFn(hit -> (String) hit.getSourceAsMap().get("name")).build();
    p.readFrom(source).writeTo(Sinks.list(results));
    submitJob(p);
    assertThat(results).containsOnlyOnce("Frantisek");
}

Example 2 with BatchSource

use of com.hazelcast.jet.pipeline.BatchSource in project hazelcast by hazelcast.

the class CommonElasticSourcesTest method given_multipleIndexes_when_readFromElasticSourceWithIndexWildcard_then_resultDocumentsFromAllIndexes.

@Test
public void given_multipleIndexes_when_readFromElasticSourceWithIndexWildcard_then_resultDocumentsFromAllIndexes() {
    indexDocument("my-index-1", of("name", "Frantisek"));
    indexDocument("my-index-2", of("name", "Vladimir"));
    Pipeline p = Pipeline.create();
    BatchSource<String> source = new ElasticSourceBuilder<>().clientFn(elasticClientSupplier()).searchRequestFn(() -> new SearchRequest("my-index-*")).mapToItemFn(hit -> (String) hit.getSourceAsMap().get("name")).build();
    p.readFrom(source).writeTo(Sinks.list(results));
    submitJob(p);
    assertThat(results).containsOnlyOnce("Frantisek", "Vladimir");
}

Example 3 with BatchSource

use of com.hazelcast.jet.pipeline.BatchSource in project hazelcast by hazelcast.

the class CommonElasticSourcesTest method given_indexWithOneDocument_whenReadFromElasticSource_thenFinishWithOneResult.

@Test
public void given_indexWithOneDocument_whenReadFromElasticSource_thenFinishWithOneResult() {
    indexDocument("my-index", of("name", "Frantisek"));
    Pipeline p = Pipeline.create();
    BatchSource<String> source = new ElasticSourceBuilder<>().clientFn(elasticClientSupplier()).searchRequestFn(() -> new SearchRequest("my-index")).mapToItemFn(hit -> (String) hit.getSourceAsMap().get("name")).build();
    p.readFrom(source).writeTo(Sinks.list(results));
    submitJob(p);
    assertThat(results).containsExactly("Frantisek");
}

Example 4 with BatchSource

use of com.hazelcast.jet.pipeline.BatchSource in project hazelcast by hazelcast.

the class TestAllTypesSqlConnector method fullScanReader.

@Nonnull
@Override
public Vertex fullScanReader(@Nonnull DAG dag, @Nonnull Table table, @Nullable Expression<Boolean> predicate, @Nonnull List<Expression<?>> projection, @Nullable FunctionEx<ExpressionEvalContext, EventTimePolicy<JetSqlRow>> eventTimePolicyProvider) {
    if (eventTimePolicyProvider != null) {
        throw QueryException.error("Ordering function are not supported for " + TYPE_NAME + " mappings");
    }
    BatchSource<JetSqlRow> source = SourceBuilder.batch("batch", ExpressionEvalContext::from).<JetSqlRow>fillBufferFn((ctx, buf) -> {
        JetSqlRow row = ExpressionUtil.evaluate(predicate, projection, VALUES, ctx);
        if (row != null) {
            buf.add(row);
        }
        buf.close();
    }).build();
    ProcessorMetaSupplier pms = ((BatchSourceTransform<JetSqlRow>) source).metaSupplier;
    return dag.newUniqueVertex(table.toString(), pms);
}

Example 5 with BatchSource

use of com.hazelcast.jet.pipeline.BatchSource in project hazelcast by hazelcast.

the class ManagedContextTest method testSources.

private void testSources(SupplierEx<? extends AnotherSourceContext> sourceSupplier) {
    BatchSource<String> src = SourceBuilder.batch("source", c -> sourceSupplier.get()).<String>fillBufferFn((c, b) -> {
        b.add(c.injectedValue);
        b.close();
    }).build();
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(src).writeTo(assertAnyOrder(singletonList(INJECTED_VALUE)));
    hz.getJet().newJob(pipeline).join();
}