Examples with Sink com.hazelcast.jet.pipeline.Sink used on opensource projects

Example 1 with Sink

use of com.hazelcast.jet.pipeline.Sink in project hazelcast-jet by hazelcast.

the class PipelineImpl method drainTo.

@Override
public <T> SinkStage drainTo(@Nonnull Sink<T> sink, GeneralStage<?>... stagesToDrain) {
    if (stagesToDrain == null || stagesToDrain.length == 0) {
        throw new IllegalArgumentException("No stages supplied to Pipeline.drainTo()");
    }
    List<Transform> upstream = Arrays.stream(stagesToDrain).map(s -> (AbstractStage) s).map(s -> s.transform).collect(toList());
    int[] ordinalsToAdapt = IntStream.range(0, stagesToDrain.length).filter(i -> ((ComputeStageImplBase) stagesToDrain[i]).fnAdapter == ADAPT_TO_JET_EVENT).toArray();
    SinkImpl sinkImpl = (SinkImpl) sink;
    SinkTransform sinkTransform = new SinkTransform(sinkImpl, upstream, ordinalsToAdapt);
    SinkStageImpl sinkStage = new SinkStageImpl(sinkTransform, this);
    sinkImpl.onAssignToStage();
    connect(upstream, sinkTransform);
    return sinkStage;
}

Example 2 with Sink

use of com.hazelcast.jet.pipeline.Sink in project hazelcast by hazelcast.

the class PipelineImpl method writeTo.

@Nonnull
@Override
@SuppressWarnings({ "rawtypes", "unchecked" })
public <T> SinkStage writeTo(@Nonnull Sink<? super T> sink, @Nonnull GeneralStage<? extends T> stage0, @Nonnull GeneralStage<? extends T> stage1, @Nonnull GeneralStage<? extends T>... moreStages) {
    List<GeneralStage> stages = new ArrayList<>(asList(moreStages));
    stages.add(0, stage0);
    stages.add(1, stage1);
    List<Transform> upstream = stages.stream().map(s -> (AbstractStage) s).map(s -> s.transform).collect(toList());
    int[] ordinalsToAdapt = IntStream.range(0, stages.size()).filter(i -> ((ComputeStageImplBase) stages.get(i)).fnAdapter == ADAPT_TO_JET_EVENT).toArray();
    SinkImpl sinkImpl = (SinkImpl) sink;
    SinkTransform sinkTransform = new SinkTransform(sinkImpl, upstream, ordinalsToAdapt);
    SinkStageImpl sinkStage = new SinkStageImpl(sinkTransform, this);
    sinkImpl.onAssignToStage();
    connectGeneralStages(stages, sinkTransform);
    return sinkStage;
}

Example 3 with Sink

use of com.hazelcast.jet.pipeline.Sink in project hazelcast by hazelcast.

the class SinkStressTestUtil method test_withRestarts.

public static void test_withRestarts(@Nonnull HazelcastInstance instance, @Nonnull ILogger logger, @Nonnull Sink<Integer> sink, boolean graceful, boolean exactlyOnce, @Nonnull SupplierEx<List<Integer>> actualItemsSupplier) {
    int numItems = 1000;
    Pipeline p = Pipeline.create();
    p.readFrom(SourceBuilder.stream("src", procCtx -> new int[] { procCtx.globalProcessorIndex() == 0 ? 0 : Integer.MAX_VALUE }).<Integer>fillBufferFn((ctx, buf) -> {
        if (ctx[0] < numItems) {
            buf.add(ctx[0]++);
            sleepMillis(5);
        }
    }).distributed(1).createSnapshotFn(ctx -> ctx[0] < Integer.MAX_VALUE ? ctx[0] : null).restoreSnapshotFn((ctx, state) -> ctx[0] = ctx[0] != Integer.MAX_VALUE ? state.get(0) : Integer.MAX_VALUE).build()).withoutTimestamps().peek().writeTo(sink);
    JobConfig config = new JobConfig().setProcessingGuarantee(exactlyOnce ? EXACTLY_ONCE : AT_LEAST_ONCE).setSnapshotIntervalMillis(50);
    JobProxy job = (JobProxy) instance.getJet().newJob(p, config);
    long endTime = System.nanoTime() + SECONDS.toNanos(TEST_TIMEOUT_SECONDS);
    int lastCount = 0;
    String expectedRows = IntStream.range(0, numItems).mapToObj(i -> i + (exactlyOnce ? "=1" : "")).collect(joining("\n"));
    // We'll restart once, then restart again after a short sleep (possibly during initialization),
    // and then assert some output so that the test isn't constantly restarting without any progress
    Long lastExecutionId = null;
    for (; ; ) {
        lastExecutionId = assertJobRunningEventually(instance, job, lastExecutionId);
        job.restart(graceful);
        lastExecutionId = assertJobRunningEventually(instance, job, lastExecutionId);
        sleepMillis(ThreadLocalRandom.current().nextInt(400));
        job.restart(graceful);
        try {
            List<Integer> actualItems;
            Set<Integer> distinctActualItems;
            do {
                actualItems = actualItemsSupplier.get();
                distinctActualItems = new HashSet<>(actualItems);
            } while (distinctActualItems.size() < Math.min(numItems, 100 + lastCount) && System.nanoTime() < endTime);
            lastCount = distinctActualItems.size();
            logger.info("number of committed items in the sink so far: " + lastCount);
            if (exactlyOnce) {
                String actualItemsStr = actualItems.stream().collect(groupingBy(identity(), TreeMap::new, counting())).entrySet().stream().map(Object::toString).collect(joining("\n"));
                assertEquals(expectedRows, actualItemsStr);
            } else {
                assertEquals(expectedRows, distinctActualItems.stream().map(Objects::toString).collect(joining("\n")));
            }
            // if content matches, break the loop. Otherwise restart and try again
            break;
        } catch (AssertionError e) {
            if (System.nanoTime() >= endTime) {
                throw e;
            }
        }
    }
}

Example 4 with Sink

use of com.hazelcast.jet.pipeline.Sink in project hazelcast by hazelcast.

the class ElasticSinkBuilder method build.

/**
 * Create a sink that writes data into Elasticsearch based on this builder configuration
 */
@Nonnull
public Sink<T> build() {
    requireNonNull(clientFn, "clientFn is not set");
    requireNonNull(mapToRequestFn, "mapToRequestFn is not set");
    return SinkBuilder.sinkBuilder(DEFAULT_NAME, ctx -> new BulkContext(new RestHighLevelClient(clientFn.get()), bulkRequestFn, optionsFn, retries, ctx.logger())).<T>receiveFn((bulkContext, item) -> bulkContext.add(mapToRequestFn.apply(item))).flushFn(BulkContext::flush).destroyFn(BulkContext::close).preferredLocalParallelism(DEFAULT_LOCAL_PARALLELISM).build();
}

Example 5 with Sink

use of com.hazelcast.jet.pipeline.Sink in project hazelcast by hazelcast.

the class LocalElasticSinkTest method when_writeToSink_then_shouldCloseClient.

@Test
public void when_writeToSink_then_shouldCloseClient() throws IOException {
    ClientHolder.elasticClients.clear();
    Sink<String> elasticSink = new ElasticSinkBuilder<>().clientFn(() -> {
        RestClientBuilder builder = spy(RestClient.builder(HttpHost.create(ElasticSupport.elastic.get().getHttpHostAddress())));
        when(builder.build()).thenAnswer(invocation -> {
            Object result = invocation.callRealMethod();
            RestClient client = (RestClient) spy(result);
            ClientHolder.elasticClients.add(client);
            return client;
        });
        return builder;
    }).bulkRequestFn(() -> new BulkRequest().setRefreshPolicy(RefreshPolicy.IMMEDIATE)).mapToRequestFn((String item) -> new IndexRequest("my-index").source(Collections.emptyMap())).build();
    Pipeline p = Pipeline.create();
    p.readFrom(TestSources.items("a", "b", "c")).writeTo(elasticSink);
    hz.getJet().newJob(p).join();
    for (RestClient client : ClientHolder.elasticClients) {
        verify(client).close();
    }
}