Search in sources :

Example 41 with FunctionEx

use of com.hazelcast.function.FunctionEx in project hazelcast by hazelcast.

the class OrderedStreamParallelismTest method applyTransformAndGetDag.

private DAG applyTransformAndGetDag(FunctionEx<StreamStage<Integer>, StreamStage<Integer>> transform) {
    PipelineImpl p = (PipelineImpl) Pipeline.create().setPreserveOrder(true);
    StreamStage<Integer> source = p.readFrom(TestSources.items(1)).setLocalParallelism(UPSTREAM_PARALLELISM).addTimestamps(t -> 0, Long.MAX_VALUE);
    StreamStage<Integer> applied = source.apply(transform);
    applied.mapStateful(LongAccumulator::new, (s, x) -> x).writeTo(Sinks.noop());
    return p.toDag(PIPELINE_CTX);
}
Also used : PipelineImpl(com.hazelcast.jet.impl.pipeline.PipelineImpl) FunctionEx(com.hazelcast.function.FunctionEx) ParallelJVMTest(com.hazelcast.test.annotation.ParallelJVMTest) Arrays(java.util.Arrays) HazelcastParametrizedRunner(com.hazelcast.test.HazelcastParametrizedRunner) QuickTest(com.hazelcast.test.annotation.QuickTest) Parameter(org.junit.runners.Parameterized.Parameter) Collection(java.util.Collection) RunWith(org.junit.runner.RunWith) Parameters(org.junit.runners.Parameterized.Parameters) Test(org.junit.Test) HazelcastSerialParametersRunnerFactory(com.hazelcast.test.HazelcastSerialParametersRunnerFactory) Traversers(com.hazelcast.jet.Traversers) Category(org.junit.experimental.categories.Category) Vertex(com.hazelcast.jet.core.Vertex) TestSources(com.hazelcast.jet.pipeline.test.TestSources) List(java.util.List) LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) DAG(com.hazelcast.jet.core.DAG) Context(com.hazelcast.jet.impl.pipeline.PipelineImpl.Context) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) UseParametersRunnerFactory(org.junit.runners.Parameterized.UseParametersRunnerFactory) PipelineImpl(com.hazelcast.jet.impl.pipeline.PipelineImpl)

Example 42 with FunctionEx

use of com.hazelcast.function.FunctionEx in project hazelcast by hazelcast.

the class StreamJmsPTest method initializeProcessor.

private void initializeProcessor(String destinationName, boolean isQueue, FunctionEx<Message, String> projectionFn) throws Exception {
    processorConnection = getConnectionFactory().createConnection();
    processorConnection.start();
    FunctionEx<Session, MessageConsumer> consumerFn = s -> s.createConsumer(isQueue ? s.createQueue(destinationName) : s.createTopic(destinationName));
    if (projectionFn == null) {
        projectionFn = m -> ((TextMessage) m).getText();
    }
    processor = new StreamJmsP<>(processorConnection, consumerFn, Message::getJMSMessageID, projectionFn, noEventTime(), NONE);
    outbox = new TestOutbox(1);
    processor.init(outbox, new TestProcessorContext());
}
Also used : Address(com.hazelcast.cluster.Address) ParallelJVMTest(com.hazelcast.test.annotation.ParallelJVMTest) QuickTest(com.hazelcast.test.annotation.QuickTest) RunWith(org.junit.runner.RunWith) TestOutbox(com.hazelcast.jet.core.test.TestOutbox) StreamSource(com.hazelcast.jet.pipeline.StreamSource) ActiveMQConnectionFactory(org.apache.activemq.artemis.jms.client.ActiveMQConnectionFactory) Function(java.util.function.Function) TestProcessorContext(com.hazelcast.jet.core.test.TestProcessorContext) ArrayList(java.util.ArrayList) Collections.singletonList(java.util.Collections.singletonList) Session(javax.jms.Session) Arrays.asList(java.util.Arrays.asList) After(org.junit.After) MessageProducer(javax.jms.MessageProducer) ClassRule(org.junit.ClassRule) Message(javax.jms.Message) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) EventTimePolicy.noEventTime(com.hazelcast.jet.core.EventTimePolicy.noEventTime) FunctionEx(com.hazelcast.function.FunctionEx) Connection(javax.jms.Connection) JetTestSupport(com.hazelcast.jet.core.JetTestSupport) TextMessage(javax.jms.TextMessage) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) Test(org.junit.Test) Category(org.junit.experimental.categories.Category) EmbeddedActiveMQResource(org.apache.activemq.artemis.junit.EmbeddedActiveMQResource) NONE(com.hazelcast.jet.config.ProcessingGuarantee.NONE) Sources(com.hazelcast.jet.pipeline.Sources) List(java.util.List) MessageConsumer(javax.jms.MessageConsumer) Destination(javax.jms.Destination) HazelcastParallelClassRunner(com.hazelcast.test.HazelcastParallelClassRunner) StreamSourceTransform(com.hazelcast.jet.impl.pipeline.transform.StreamSourceTransform) Queue(java.util.Queue) ConnectionFactory(javax.jms.ConnectionFactory) Assert.assertEquals(org.junit.Assert.assertEquals) MessageConsumer(javax.jms.MessageConsumer) TestOutbox(com.hazelcast.jet.core.test.TestOutbox) TestProcessorContext(com.hazelcast.jet.core.test.TestProcessorContext) Session(javax.jms.Session)

Example 43 with FunctionEx

use of com.hazelcast.function.FunctionEx in project hazelcast by hazelcast.

the class RebalanceStreamStageTest method when_peekAndRebalanceAndMap_then_dagEdgeDistributed.

@Test
public void when_peekAndRebalanceAndMap_then_dagEdgeDistributed() {
    // Given
    List<Integer> input = sequence(itemCount);
    StreamStage<Integer> srcStage = streamStageFromList(input);
    FunctionEx<Integer, String> formatFn = i -> String.format("%04d-string", i);
    // When
    StreamStage<String> mapped = srcStage.peek().rebalance().map(formatFn);
    // Then
    mapped.writeTo(sink);
    DAG dag = p.toDag();
    Edge srcToMap = dag.getInboundEdges("map").get(0);
    assertTrue("Rebalancing should make the edge distributed", srcToMap.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", srcToMap.getPartitioner());
    execute();
    assertEquals(streamToString(input.stream(), formatFn), streamToString(sinkStreamOf(String.class), identity()));
}
Also used : FunctionEx(com.hazelcast.function.FunctionEx) KeyedWindowResult(com.hazelcast.jet.datamodel.KeyedWindowResult) Assert.assertNotNull(org.junit.Assert.assertNotNull) AggregateOperations(com.hazelcast.jet.aggregate.AggregateOperations) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) AggregateOperation1(com.hazelcast.jet.aggregate.AggregateOperation1) JetException(com.hazelcast.jet.JetException) List(java.util.List) Assert.assertNull(org.junit.Assert.assertNull) WindowDefinition.tumbling(com.hazelcast.jet.pipeline.WindowDefinition.tumbling) LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) Function.identity(java.util.function.Function.identity) DAG(com.hazelcast.jet.core.DAG) Edge(com.hazelcast.jet.core.Edge) Assert.assertEquals(org.junit.Assert.assertEquals) WindowResult(com.hazelcast.jet.datamodel.WindowResult) DAG(com.hazelcast.jet.core.DAG) Edge(com.hazelcast.jet.core.Edge) Test(org.junit.Test)

Example 44 with FunctionEx

use of com.hazelcast.function.FunctionEx in project hazelcast by hazelcast.

the class TestHazelcastInstanceFactory method registerTestMetricsPublisher.

private void registerTestMetricsPublisher(HazelcastInstance hazelcastInstance) {
    if (metricsRule != null && metricsRule.isEnabled()) {
        MetricsService metricService = getNodeEngineImpl(hazelcastInstance).getService(MetricsService.SERVICE_NAME);
        metricService.registerPublisher((FunctionEx<NodeEngine, MetricsPublisher>) nodeEngine -> metricsRule.getMetricsPublisher(hazelcastInstance));
    }
}
Also used : NodeEngine(com.hazelcast.spi.impl.NodeEngine) IntStream(java.util.stream.IntStream) Address(com.hazelcast.cluster.Address) HazelcastInstanceImpl(com.hazelcast.instance.impl.HazelcastInstanceImpl) Arrays(java.util.Arrays) MetricsRule(com.hazelcast.test.metrics.MetricsRule) HazelcastTestSupport.spawn(com.hazelcast.test.HazelcastTestSupport.spawn) Accessors.getAddress(com.hazelcast.test.Accessors.getAddress) HazelcastInstanceFactory(com.hazelcast.instance.impl.HazelcastInstanceFactory) TestNodeRegistry(com.hazelcast.test.mocknetwork.TestNodeRegistry) Function(java.util.function.Function) HazelcastTestSupport.assertClusterSizeEventually(com.hazelcast.test.HazelcastTestSupport.assertClusterSizeEventually) ConcurrentMap(java.util.concurrent.ConcurrentMap) HashSet(java.util.HashSet) Collections.unmodifiableCollection(java.util.Collections.unmodifiableCollection) MetricsPublisher(com.hazelcast.internal.metrics.MetricsPublisher) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Arrays.asList(java.util.Arrays.asList) YamlConfigBuilder(com.hazelcast.config.YamlConfigBuilder) Accessors.getNode(com.hazelcast.test.Accessors.getNode) Config(com.hazelcast.config.Config) HazelcastInstance(com.hazelcast.core.HazelcastInstance) FunctionEx(com.hazelcast.function.FunctionEx) Accessors.getNodeEngineImpl(com.hazelcast.test.Accessors.getNodeEngineImpl) NodeEngine(com.hazelcast.spi.impl.NodeEngine) TestUtil.terminateInstance(com.hazelcast.instance.impl.TestUtil.terminateInstance) DefaultNodeContext(com.hazelcast.instance.impl.DefaultNodeContext) Collection(java.util.Collection) XmlConfigBuilder(com.hazelcast.config.XmlConfigBuilder) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) HazelcastInstanceProxy(com.hazelcast.instance.impl.HazelcastInstanceProxy) Preconditions.checkNotNull(com.hazelcast.internal.util.Preconditions.checkNotNull) UnknownHostException(java.net.UnknownHostException) ClusterProperty(com.hazelcast.spi.properties.ClusterProperty) NetworkConfig(com.hazelcast.config.NetworkConfig) Collectors.toList(java.util.stream.Collectors.toList) NodeContext(com.hazelcast.instance.impl.NodeContext) MetricsService(com.hazelcast.internal.metrics.impl.MetricsService) Hazelcast(com.hazelcast.core.Hazelcast) SYSPROP_MEMBER_CONFIG(com.hazelcast.internal.config.DeclarativeConfigUtil.SYSPROP_MEMBER_CONFIG) YAML_ACCEPTED_SUFFIXES(com.hazelcast.internal.config.DeclarativeConfigUtil.YAML_ACCEPTED_SUFFIXES) Util.uncheckCall(com.hazelcast.jet.impl.util.Util.uncheckCall) Comparator(java.util.Comparator) DeclarativeConfigUtil.isAcceptedSuffixConfigured(com.hazelcast.internal.config.DeclarativeConfigUtil.isAcceptedSuffixConfigured) Collections(java.util.Collections) MetricsService(com.hazelcast.internal.metrics.impl.MetricsService) MetricsPublisher(com.hazelcast.internal.metrics.MetricsPublisher)

Aggregations

FunctionEx (com.hazelcast.function.FunctionEx)44 List (java.util.List)30 Nonnull (javax.annotation.Nonnull)20 DAG (com.hazelcast.jet.core.DAG)18 Test (org.junit.Test)18 Collections.singletonList (java.util.Collections.singletonList)15 Assert.assertEquals (org.junit.Assert.assertEquals)15 Edge (com.hazelcast.jet.core.Edge)14 BiFunctionEx (com.hazelcast.function.BiFunctionEx)13 Function (java.util.function.Function)13 LongAccumulator (com.hazelcast.jet.accumulator.LongAccumulator)12 ArrayList (java.util.ArrayList)12 ToLongFunctionEx (com.hazelcast.function.ToLongFunctionEx)11 Vertex (com.hazelcast.jet.core.Vertex)11 ParallelJVMTest (com.hazelcast.test.annotation.ParallelJVMTest)11 Entry (java.util.Map.Entry)11 Assert.assertTrue (org.junit.Assert.assertTrue)11 JetException (com.hazelcast.jet.JetException)10 AggregateOperation1 (com.hazelcast.jet.aggregate.AggregateOperation1)10 QuickTest (com.hazelcast.test.annotation.QuickTest)10