Examples with SupplierEx com.hazelcast.function.SupplierEx used on opensource projects

Example 11 with SupplierEx

use of com.hazelcast.function.SupplierEx in project hazelcast by hazelcast.

the class S3Sources method s3.

/**
 * Creates an AWS S3 {@link BatchSource} which lists all the objects in the
 * bucket-list using given {@code prefix}, reads them line by line,
 * transforms each line to the desired output object using given {@code
 * mapFn} and emits them to downstream.
 * <p>
 * The source does not save any state to snapshot. If the job is restarted,
 * it will re-emit all entries.
 * <p>
 * The default local parallelism for this processor is 2.
 * <p>
 * Here is an example which reads the objects from a single bucket with
 * applying the given prefix.
 *
 * <pre>{@code
 * Pipeline p = Pipeline.create();
 * BatchStage<String> srcStage = p.readFrom(S3Sources.s3(
 *      Arrays.asList("bucket1", "bucket2"),
 *      "prefix",
 *      StandardCharsets.UTF_8,
 *      () -> S3Client.create(),
 *      (filename, line) -> line
 * ));
 * }</pre>
 *
 * @param bucketNames    list of bucket-names
 * @param prefix         the prefix to filter the objects. Optional, passing
 *                       {@code null} will list all objects.
 * @param clientSupplier function which returns the s3 client to use
 *                       one client per processor instance is used
 * @param mapFn          the function which creates output object from each
 *                       line. Gets the object name and line as parameters
 * @param <T>            the type of the items the source emits
 */
@Nonnull
public static <T> BatchSource<T> s3(@Nonnull List<String> bucketNames, @Nullable String prefix, @Nonnull Charset charset, @Nonnull SupplierEx<? extends S3Client> clientSupplier, @Nonnull BiFunctionEx<String, String, ? extends T> mapFn) {
    String charsetName = charset.name();
    FunctionEx<InputStream, Stream<String>> readFileFn = responseInputStream -> {
        BufferedReader reader = new BufferedReader(new InputStreamReader(responseInputStream, Charset.forName(charsetName)));
        return reader.lines();
    };
    return s3(bucketNames, prefix, clientSupplier, readFileFn, mapFn);
}

Example 12 with SupplierEx

use of com.hazelcast.function.SupplierEx in project hazelcast by hazelcast.

the class ManagedContextTest method testSinks.

private void testSinks(SupplierEx<? extends AnotherSinkContext> sinkSupplier) {
    Sink<Object> sink = SinkBuilder.sinkBuilder("sink", c -> sinkSupplier.get()).receiveFn((c, i) -> assertEquals(INJECTED_VALUE, c.injectedValue)).build();
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(TestSources.items(1)).writeTo(sink);
    hz.getJet().newJob(pipeline).join();
}

Example 13 with SupplierEx

use of com.hazelcast.function.SupplierEx in project hazelcast by hazelcast.

the class JobMetrics_NonSharedClusterTest method when_noMetricCollectionYet_then_emptyMetrics.

@Test
public void when_noMetricCollectionYet_then_emptyMetrics() {
    Config config = smallInstanceConfig();
    config.getMetricsConfig().setCollectionFrequencySeconds(10_000);
    HazelcastInstance inst = createHazelcastInstance(config);
    DAG dag = new DAG();
    dag.newVertex("v1", (SupplierEx<Processor>) NoOutputSourceP::new).localParallelism(1);
    // Initial collection interval is 1 second. So let's run a job and wait until it has metrics.
    Job job1 = inst.getJet().newJob(dag, JOB_CONFIG_WITH_METRICS);
    try {
        JetTestSupport.assertTrueEventually(() -> assertFalse(job1.getMetrics().metrics().isEmpty()), 10);
    } catch (AssertionError e) {
        // If we don't get metrics in 10 seconds, ignore it, we probably missed the first collection
        // with this job. We might have caught a different error, let's log it at least.
        logger.warning("Ignoring this error: " + e, e);
    }
    // Let's do a second job for which we know there will be no metrics collection. It should
    // return empty metrics because the next collection will be in 10_000 seconds.
    Job job2 = inst.getJet().newJob(dag, JOB_CONFIG_WITH_METRICS);
    assertJobStatusEventually(job2, RUNNING);
    assertTrue(job2.getMetrics().metrics().isEmpty());
}

Example 14 with SupplierEx

use of com.hazelcast.function.SupplierEx in project hazelcast by hazelcast.

the class ExecutionLifecycleTest method when_clientJoinBeforeAndAfterComplete_then_exceptionEquals.

@Test
public void when_clientJoinBeforeAndAfterComplete_then_exceptionEquals() {
    // not applicable to light jobs - we can't connect to light jobs after they complete
    assumeFalse(useLightJob);
    DAG dag = new DAG();
    Vertex noop = dag.newVertex("noop", (SupplierEx<Processor>) NoOutputSourceP::new).localParallelism(1);
    Vertex faulty = dag.newVertex("faulty", () -> new MockP().setCompleteError(MOCK_ERROR)).localParallelism(1);
    dag.edge(between(noop, faulty));
    Job job = newJob(client(), dag, null);
    assertJobStatusEventually(job, RUNNING);
    NoOutputSourceP.proceedLatch.countDown();
    Throwable excBeforeComplete;
    Throwable excAfterComplete;
    try {
        job.join();
        throw new AssertionError("should have failed");
    } catch (Exception e) {
        excBeforeComplete = e;
    }
    // create a new client that will join the job after completion
    HazelcastInstance client2 = factory().newHazelcastClient();
    Job job2 = client2.getJet().getJob(job.getId());
    try {
        job2.join();
        throw new AssertionError("should have failed");
    } catch (Exception e) {
        excAfterComplete = e;
    }
    logger.info("exception before completion", excBeforeComplete);
    logger.info("exception after completion", excAfterComplete);
    // Then
    assertInstanceOf(CompletionException.class, excBeforeComplete);
    assertInstanceOf(CompletionException.class, excAfterComplete);
    Throwable causeBefore = excBeforeComplete.getCause();
    Throwable causeAfter = excAfterComplete.getCause();
    assertEquals(causeBefore.getClass(), causeAfter.getClass());
    assertContains(causeAfter.getMessage(), causeBefore.getMessage());
}

Example 15 with SupplierEx

use of com.hazelcast.function.SupplierEx in project hazelcast by hazelcast.

the class WriteJdbcPTest method failTwiceDataSourceSupplier.

private static SupplierEx<DataSource> failTwiceDataSourceSupplier() {
    return new SupplierEx<DataSource>() {

        int remainingFailures = 2;

        @Override
        public DataSource getEx() throws SQLException {
            DataSource realDs = (DataSource) createDataSource(false);
            DataSource mockDs = mock(DataSource.class);
            doAnswer(invocation -> {
                if (remainingFailures-- > 0) {
                    throw new SQLException("connection failure");
                }
                return realDs.getConnection();
            }).when(mockDs).getConnection();
            return mockDs;
        }
    };
}