Search in sources :

Example 1 with ToLongFunctionEx

use of com.hazelcast.function.ToLongFunctionEx in project hazelcast by hazelcast.

the class CreateDagVisitor method onSlidingWindowAggregate.

public Vertex onSlidingWindowAggregate(SlidingWindowAggregatePhysicalRel rel) {
    FunctionEx<JetSqlRow, ?> groupKeyFn = rel.groupKeyFn();
    AggregateOperation<?, JetSqlRow> aggregateOperation = rel.aggrOp();
    Expression<?> timestampExpression = rel.timestampExpression();
    ToLongFunctionEx<JetSqlRow> timestampFn = row -> WindowUtils.extractMillis(timestampExpression.eval(row.getRow(), MOCK_EEC));
    SlidingWindowPolicy windowPolicy = rel.windowPolicyProvider().apply(MOCK_EEC);
    KeyedWindowResultFunction<? super Object, ? super JetSqlRow, ?> resultMapping = rel.outputValueMapping();
    if (rel.numStages() == 1) {
        Vertex vertex = dag.newUniqueVertex("Sliding-Window-AggregateByKey", Processors.aggregateToSlidingWindowP(singletonList(groupKeyFn), singletonList(timestampFn), TimestampKind.EVENT, windowPolicy, 0, aggregateOperation, resultMapping));
        connectInput(rel.getInput(), vertex, edge -> edge.distributeTo(localMemberAddress).allToOne(""));
        return vertex;
    } else {
        assert rel.numStages() == 2;
        Vertex vertex1 = dag.newUniqueVertex("Sliding-Window-AccumulateByKey", Processors.accumulateByFrameP(singletonList(groupKeyFn), singletonList(timestampFn), TimestampKind.EVENT, windowPolicy, aggregateOperation));
        Vertex vertex2 = dag.newUniqueVertex("Sliding-Window-CombineByKey", Processors.combineToSlidingWindowP(windowPolicy, aggregateOperation, resultMapping));
        connectInput(rel.getInput(), vertex1, edge -> edge.partitioned(groupKeyFn));
        dag.edge(between(vertex1, vertex2).distributed().partitioned(entryKey()));
        return vertex2;
    }
}
Also used : Address(com.hazelcast.cluster.Address) Traverser(com.hazelcast.jet.Traverser) ExpressionValues(com.hazelcast.jet.sql.impl.opt.ExpressionValues) PlanObjectKey(com.hazelcast.sql.impl.optimizer.PlanObjectKey) Processors.mapP(com.hazelcast.jet.core.processor.Processors.mapP) SourceProcessors.convenientSourceP(com.hazelcast.jet.core.processor.SourceProcessors.convenientSourceP) QueryParameterMetadata(com.hazelcast.sql.impl.QueryParameterMetadata) Collections.singletonList(java.util.Collections.singletonList) BiFunctionEx(com.hazelcast.function.BiFunctionEx) HazelcastTable(com.hazelcast.jet.sql.impl.schema.HazelcastTable) ObjectArrayKey(com.hazelcast.jet.sql.impl.ObjectArrayKey) SlidingWindowPolicy(com.hazelcast.jet.core.SlidingWindowPolicy) SqlConnectorUtil(com.hazelcast.jet.sql.impl.connector.SqlConnectorUtil) AggregateOperation(com.hazelcast.jet.aggregate.AggregateOperation) Functions.entryKey(com.hazelcast.function.Functions.entryKey) SqlConnectorUtil.getJetSqlConnector(com.hazelcast.jet.sql.impl.connector.SqlConnectorUtil.getJetSqlConnector) DAG(com.hazelcast.jet.core.DAG) RootResultConsumerSink.rootResultConsumerSink(com.hazelcast.jet.sql.impl.processors.RootResultConsumerSink.rootResultConsumerSink) ServiceFactories(com.hazelcast.jet.pipeline.ServiceFactories) FunctionEx(com.hazelcast.function.FunctionEx) KeyedWindowResultFunction(com.hazelcast.jet.core.function.KeyedWindowResultFunction) Predicate(java.util.function.Predicate) Collections.emptyList(java.util.Collections.emptyList) Set(java.util.Set) ConsumerEx(com.hazelcast.function.ConsumerEx) IMapSqlConnector(com.hazelcast.jet.sql.impl.connector.map.IMapSqlConnector) ExpressionUtil(com.hazelcast.jet.sql.impl.ExpressionUtil) Processors.filterUsingServiceP(com.hazelcast.jet.core.processor.Processors.filterUsingServiceP) List(java.util.List) ExpressionEvalContext(com.hazelcast.sql.impl.expression.ExpressionEvalContext) Processors.mapUsingServiceP(com.hazelcast.jet.core.processor.Processors.mapUsingServiceP) Table(com.hazelcast.sql.impl.schema.Table) ComparatorEx(com.hazelcast.function.ComparatorEx) QueryDataType(com.hazelcast.sql.impl.type.QueryDataType) Processors(com.hazelcast.jet.core.processor.Processors) Processors.flatMapUsingServiceP(com.hazelcast.jet.core.processor.Processors.flatMapUsingServiceP) TimestampKind(com.hazelcast.jet.core.TimestampKind) Function(java.util.function.Function) JetSqlRow(com.hazelcast.sql.impl.row.JetSqlRow) HashSet(java.util.HashSet) Edge.from(com.hazelcast.jet.core.Edge.from) Edge(com.hazelcast.jet.core.Edge) Expression(com.hazelcast.sql.impl.expression.Expression) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) VertexWithInputConfig(com.hazelcast.jet.sql.impl.connector.SqlConnector.VertexWithInputConfig) SingleRel(org.apache.calcite.rel.SingleRel) Nullable(javax.annotation.Nullable) JetJoinInfo(com.hazelcast.jet.sql.impl.JetJoinInfo) NodeEngine(com.hazelcast.spi.impl.NodeEngine) ProcessorMetaSupplier(com.hazelcast.jet.core.ProcessorMetaSupplier) RelNode(org.apache.calcite.rel.RelNode) Consumer(java.util.function.Consumer) Vertex(com.hazelcast.jet.core.Vertex) ToLongFunctionEx(com.hazelcast.function.ToLongFunctionEx) SqlHashJoinP(com.hazelcast.jet.sql.impl.processors.SqlHashJoinP) ConstantExpression(com.hazelcast.sql.impl.expression.ConstantExpression) Processors.sortP(com.hazelcast.jet.core.processor.Processors.sortP) DefaultSerializationServiceBuilder(com.hazelcast.internal.serialization.impl.DefaultSerializationServiceBuilder) WindowUtils(com.hazelcast.jet.sql.impl.aggregate.WindowUtils) Edge.between(com.hazelcast.jet.core.Edge.between) Vertex(com.hazelcast.jet.core.Vertex) SlidingWindowPolicy(com.hazelcast.jet.core.SlidingWindowPolicy) JetSqlRow(com.hazelcast.sql.impl.row.JetSqlRow)

Example 2 with ToLongFunctionEx

use of com.hazelcast.function.ToLongFunctionEx in project hazelcast by hazelcast.

the class SourceBuilderTest method stream_distributed_socketSource_withTimestamps.

@Test
public void stream_distributed_socketSource_withTimestamps() throws IOException {
    // Given
    try (ServerSocket serverSocket = new ServerSocket(0)) {
        startServer(serverSocket);
        // When
        int localPort = serverSocket.getLocalPort();
        ToLongFunctionEx<String> timestampFn = line -> Long.valueOf(line.substring(LINE_PREFIX.length()));
        BatchSource<String> socketSource = SourceBuilder.batch("socket-source-with-timestamps", ctx -> socketReader(localPort)).<String>fillBufferFn((in, buf) -> {
            String line = in.readLine();
            if (line != null) {
                buf.add(line);
            } else {
                buf.close();
            }
        }).destroyFn(BufferedReader::close).distributed(PREFERRED_LOCAL_PARALLELISM).build();
        // Then
        Pipeline p = Pipeline.create();
        p.readFrom(socketSource).addTimestamps(timestampFn, 1000).window(tumbling(1)).aggregate(AggregateOperations.counting()).writeTo(sinkList());
        hz().getJet().newJob(p).join();
        List<WindowResult<Long>> expected = LongStream.range(1, itemCount + 1).mapToObj(i -> new WindowResult<>(i - 1, i, (long) PREFERRED_LOCAL_PARALLELISM * MEMBER_COUNT)).collect(toList());
        assertEquals(expected, new ArrayList<>(sinkList));
    }
}
Also used : IntStream(java.util.stream.IntStream) Socket(java.net.Socket) NANOSECONDS(java.util.concurrent.TimeUnit.NANOSECONDS) ArrayList(java.util.ArrayList) ServerSocket(java.net.ServerSocket) WindowDefinition.tumbling(com.hazelcast.jet.pipeline.WindowDefinition.tumbling) Map(java.util.Map) UuidUtil(com.hazelcast.internal.util.UuidUtil) OutputStreamWriter(java.io.OutputStreamWriter) JobStatus(com.hazelcast.jet.core.JobStatus) Job(com.hazelcast.jet.Job) IList(com.hazelcast.collection.IList) JobRepository(com.hazelcast.jet.impl.JobRepository) PrintWriter(java.io.PrintWriter) FunctionEx(com.hazelcast.function.FunctionEx) LongStream(java.util.stream.LongStream) Iterator(java.util.Iterator) EXACTLY_ONCE(com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE) JobConfig(com.hazelcast.jet.config.JobConfig) FileWriter(java.io.FileWriter) AggregateOperations(com.hazelcast.jet.aggregate.AggregateOperations) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) InputStreamReader(java.io.InputStreamReader) Collectors(java.util.stream.Collectors) File(java.io.File) FileNotFoundException(java.io.FileNotFoundException) Serializable(java.io.Serializable) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) ToLongFunctionEx(com.hazelcast.function.ToLongFunctionEx) Assert.assertFalse(org.junit.Assert.assertFalse) BufferedReader(java.io.BufferedReader) FileReader(java.io.FileReader) Assert.assertEquals(org.junit.Assert.assertEquals) WindowResult(com.hazelcast.jet.datamodel.WindowResult) BufferedReader(java.io.BufferedReader) ServerSocket(java.net.ServerSocket) WindowResult(com.hazelcast.jet.datamodel.WindowResult) Test(org.junit.Test)

Example 3 with ToLongFunctionEx

use of com.hazelcast.function.ToLongFunctionEx in project hazelcast by hazelcast.

the class SourceBuilderTest method stream_socketSource_withTimestamps.

@Test
public void stream_socketSource_withTimestamps() throws IOException {
    // Given
    try (ServerSocket serverSocket = new ServerSocket(0)) {
        startServer(serverSocket);
        // When
        int localPort = serverSocket.getLocalPort();
        ToLongFunctionEx<String> timestampFn = line -> Long.valueOf(line.substring(LINE_PREFIX.length()));
        BatchSource<String> socketSource = SourceBuilder.batch("socket-source-with-timestamps", ctx -> socketReader(localPort)).<String>fillBufferFn((in, buf) -> {
            String line = in.readLine();
            if (line != null) {
                buf.add(line);
            } else {
                buf.close();
            }
        }).destroyFn(BufferedReader::close).build();
        // Then
        Pipeline p = Pipeline.create();
        p.readFrom(socketSource).addTimestamps(timestampFn, 0).window(tumbling(1)).aggregate(AggregateOperations.counting()).writeTo(sinkList());
        hz().getJet().newJob(p).join();
        List<WindowResult<Long>> expected = LongStream.range(1, itemCount + 1).mapToObj(i -> new WindowResult<>(i - 1, i, 1L)).collect(toList());
        assertEquals(expected, new ArrayList<>(sinkList));
    }
}
Also used : IntStream(java.util.stream.IntStream) Socket(java.net.Socket) NANOSECONDS(java.util.concurrent.TimeUnit.NANOSECONDS) ArrayList(java.util.ArrayList) ServerSocket(java.net.ServerSocket) WindowDefinition.tumbling(com.hazelcast.jet.pipeline.WindowDefinition.tumbling) Map(java.util.Map) UuidUtil(com.hazelcast.internal.util.UuidUtil) OutputStreamWriter(java.io.OutputStreamWriter) JobStatus(com.hazelcast.jet.core.JobStatus) Job(com.hazelcast.jet.Job) IList(com.hazelcast.collection.IList) JobRepository(com.hazelcast.jet.impl.JobRepository) PrintWriter(java.io.PrintWriter) FunctionEx(com.hazelcast.function.FunctionEx) LongStream(java.util.stream.LongStream) Iterator(java.util.Iterator) EXACTLY_ONCE(com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE) JobConfig(com.hazelcast.jet.config.JobConfig) FileWriter(java.io.FileWriter) AggregateOperations(com.hazelcast.jet.aggregate.AggregateOperations) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) InputStreamReader(java.io.InputStreamReader) Collectors(java.util.stream.Collectors) File(java.io.File) FileNotFoundException(java.io.FileNotFoundException) Serializable(java.io.Serializable) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) ToLongFunctionEx(com.hazelcast.function.ToLongFunctionEx) Assert.assertFalse(org.junit.Assert.assertFalse) BufferedReader(java.io.BufferedReader) FileReader(java.io.FileReader) Assert.assertEquals(org.junit.Assert.assertEquals) WindowResult(com.hazelcast.jet.datamodel.WindowResult) ServerSocket(java.net.ServerSocket) WindowResult(com.hazelcast.jet.datamodel.WindowResult) Test(org.junit.Test)

Example 4 with ToLongFunctionEx

use of com.hazelcast.function.ToLongFunctionEx in project hazelcast by hazelcast.

the class SlidingWindowPTest method before.

@Before
public void before() {
    SlidingWindowPolicy winPolicy = slidingWinPolicy(4, 1);
    AggregateOperation1<Entry<?, Long>, LongAccumulator, Long> operation = AggregateOperation.withCreate(LongAccumulator::new).andAccumulate((LongAccumulator acc, Entry<?, Long> item) -> acc.add(item.getValue())).andCombine(LongAccumulator::add).andDeduct(hasDeduct ? LongAccumulator::subtract : null).andExportFinish(LongAccumulator::get);
    FunctionEx<?, Long> keyFn = t -> KEY;
    ToLongFunctionEx<Entry<Long, Long>> timestampFn = Entry::getKey;
    SupplierEx<Processor> procSupplier = singleStageProcessor ? aggregateToSlidingWindowP(singletonList(keyFn), singletonList(timestampFn), TimestampKind.EVENT, winPolicy, 0L, operation, KeyedWindowResult::new) : combineToSlidingWindowP(winPolicy, operation, KeyedWindowResult::new);
    // new supplier to save the last supplied instance
    supplier = () -> lastSuppliedProcessor = (SlidingWindowP) procSupplier.get();
}
Also used : LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) Collections.shuffle(java.util.Collections.shuffle) ParallelJVMTest(com.hazelcast.test.annotation.ParallelJVMTest) Arrays(java.util.Arrays) KeyedWindowResult(com.hazelcast.jet.datamodel.KeyedWindowResult) QuickTest(com.hazelcast.test.annotation.QuickTest) RunWith(org.junit.runner.RunWith) Parameters(org.junit.runners.Parameterized.Parameters) Processor(com.hazelcast.jet.core.Processor) TimestampKind(com.hazelcast.jet.core.TimestampKind) TestSupport.verifyProcessor(com.hazelcast.jet.core.test.TestSupport.verifyProcessor) SlidingWindowPolicy.slidingWinPolicy(com.hazelcast.jet.core.SlidingWindowPolicy.slidingWinPolicy) ArrayList(java.util.ArrayList) Collections.singletonList(java.util.Collections.singletonList) Watermark(com.hazelcast.jet.core.Watermark) SlidingWindowPolicy(com.hazelcast.jet.core.SlidingWindowPolicy) Processors.aggregateToSlidingWindowP(com.hazelcast.jet.core.processor.Processors.aggregateToSlidingWindowP) AggregateOperation(com.hazelcast.jet.aggregate.AggregateOperation) Util.entry(com.hazelcast.jet.Util.entry) Arrays.asList(java.util.Arrays.asList) After(org.junit.After) JetTestSupport.wm(com.hazelcast.jet.core.JetTestSupport.wm) Processors.combineToSlidingWindowP(com.hazelcast.jet.core.processor.Processors.combineToSlidingWindowP) ExpectedException(org.junit.rules.ExpectedException) Before(org.junit.Before) UseParametersRunnerFactory(org.junit.runners.Parameterized.UseParametersRunnerFactory) FunctionEx(com.hazelcast.function.FunctionEx) LongStream(java.util.stream.LongStream) HazelcastParametrizedRunner(com.hazelcast.test.HazelcastParametrizedRunner) Parameter(org.junit.runners.Parameterized.Parameter) Collection(java.util.Collection) Assert.assertTrue(org.junit.Assert.assertTrue) HazelcastParallelParametersRunnerFactory(com.hazelcast.test.HazelcastParallelParametersRunnerFactory) Test(org.junit.Test) AggregateOperation1(com.hazelcast.jet.aggregate.AggregateOperation1) Category(org.junit.experimental.categories.Category) SupplierEx(com.hazelcast.function.SupplierEx) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) ToLongFunctionEx(com.hazelcast.function.ToLongFunctionEx) Rule(org.junit.Rule) LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) Entry(java.util.Map.Entry) Entry(java.util.Map.Entry) SlidingWindowPolicy(com.hazelcast.jet.core.SlidingWindowPolicy) Processor(com.hazelcast.jet.core.Processor) TestSupport.verifyProcessor(com.hazelcast.jet.core.test.TestSupport.verifyProcessor) Processors.aggregateToSlidingWindowP(com.hazelcast.jet.core.processor.Processors.aggregateToSlidingWindowP) Processors.combineToSlidingWindowP(com.hazelcast.jet.core.processor.Processors.combineToSlidingWindowP) Before(org.junit.Before)

Example 5 with ToLongFunctionEx

use of com.hazelcast.function.ToLongFunctionEx in project hazelcast by hazelcast.

the class SlidingWindowP_twoStageSnapshotTest method before.

@Before
public void before() {
    SlidingWindowPolicy windowDef = slidingWinPolicy(4, 1);
    AggregateOperation1<Entry<?, Long>, LongAccumulator, Long> aggrOp = AggregateOperation.withCreate(LongAccumulator::new).andAccumulate((LongAccumulator acc, Entry<?, Long> item) -> acc.add(item.getValue())).andCombine(LongAccumulator::add).andDeduct(LongAccumulator::subtract).andExportFinish(LongAccumulator::get);
    SupplierEx<Processor> procSupplier1 = Processors.accumulateByFrameP(singletonList((FunctionEx<? super Entry<Long, Long>, ?>) t -> KEY), singletonList((ToLongFunctionEx<? super Entry<Long, Long>>) Entry::getKey), TimestampKind.EVENT, windowDef, aggrOp.withIdentityFinish());
    SupplierEx<Processor> procSupplier2 = combineToSlidingWindowP(windowDef, aggrOp, KeyedWindowResult::new);
    // new supplier to save the last supplied instance
    stage1Supplier = () -> lastSuppliedStage1Processor = (SlidingWindowP<?, ?, ?, ?>) procSupplier1.get();
    stage2Supplier = () -> lastSuppliedStage2Processor = (SlidingWindowP<?, ?, ?, ?>) procSupplier2.get();
}
Also used : LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) Entry(java.util.Map.Entry) SlidingWindowPolicy(com.hazelcast.jet.core.SlidingWindowPolicy) Processor(com.hazelcast.jet.core.Processor) ToLongFunctionEx(com.hazelcast.function.ToLongFunctionEx) FunctionEx(com.hazelcast.function.FunctionEx) ToLongFunctionEx(com.hazelcast.function.ToLongFunctionEx) Processors.combineToSlidingWindowP(com.hazelcast.jet.core.processor.Processors.combineToSlidingWindowP) KeyedWindowResult(com.hazelcast.jet.datamodel.KeyedWindowResult) Before(org.junit.Before)

Aggregations

FunctionEx (com.hazelcast.function.FunctionEx)5 ToLongFunctionEx (com.hazelcast.function.ToLongFunctionEx)5 List (java.util.List)4 SlidingWindowPolicy (com.hazelcast.jet.core.SlidingWindowPolicy)3 ArrayList (java.util.ArrayList)3 Collectors.toList (java.util.stream.Collectors.toList)3 LongStream (java.util.stream.LongStream)3 Assert.assertTrue (org.junit.Assert.assertTrue)3 Test (org.junit.Test)3 IList (com.hazelcast.collection.IList)2 UuidUtil (com.hazelcast.internal.util.UuidUtil)2 Job (com.hazelcast.jet.Job)2 LongAccumulator (com.hazelcast.jet.accumulator.LongAccumulator)2 AggregateOperation (com.hazelcast.jet.aggregate.AggregateOperation)2 AggregateOperations (com.hazelcast.jet.aggregate.AggregateOperations)2 JobConfig (com.hazelcast.jet.config.JobConfig)2 EXACTLY_ONCE (com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE)2 JobStatus (com.hazelcast.jet.core.JobStatus)2 Processor (com.hazelcast.jet.core.Processor)2 TimestampKind (com.hazelcast.jet.core.TimestampKind)2