Examples with Pipe com.ociweb.pronghorn.pipe.Pipe used on opensource projects

Example 26 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project FogLight-Examples by oci-pronghorn.

the class IoTApp method matrixTest.

public <M extends MatrixSchema<M>> void matrixTest() {
    // speed
    // slow     Doubles  Longs    6.15 5.8      7.024  7.18
    // Decimals          5.9           9.40 - 13
    // Floats            6.06           6.26
    // fast     Integers          5.80           5.95
    // Decimals;//Integers; //2, 3328335 longs/ints/doubles   [0,332833152] floats
    MatrixTypes type = MatrixTypes.Integers;
    // TypeMask.Decimal;
    // when matrix size is larger than CPU cache we run into issues.
    int leftRows = 1200;
    int leftColumns = 1000;
    int rightColumns = 1000;
    int rightRows = leftColumns;
    // walk leftRows , by rightCol for output
    // 5x2
    // 2x3
    MatrixSchema leftSchema = BuildMatrixCompute.buildSchema(leftRows, leftColumns, type);
    RowSchema<M> leftRowSchema = new RowSchema<M>(leftSchema);
    MatrixSchema rightSchema = BuildMatrixCompute.buildSchema(rightRows, rightColumns, type);
    RowSchema<M> rightRowSchema = new RowSchema<M>(rightSchema);
    MatrixSchema resultSchema = BuildMatrixCompute.buildResultSchema(leftSchema, rightSchema);
    RowSchema<M> rowResultSchema = new RowSchema<M>(resultSchema);
    DecimalSchema result2Schema = new DecimalSchema<M>(resultSchema);
    GraphManager gm = new GraphManager();
    // GraphManager.addDefaultNota(gm, GraphManager.SCHEDULE_RATE, 500);
    // TODO: not sure why but the splitter that consumes left needs a minimum ring size or it gets stuck,
    Pipe<RowSchema<M>> left = new Pipe<RowSchema<M>>(new PipeConfig<RowSchema<M>>(leftRowSchema, leftRows));
    Pipe<RowSchema<M>> right = new Pipe<RowSchema<M>>(new PipeConfig<RowSchema<M>>(rightRowSchema, rightRows));
    // NOTE: reqires 2 or JSON will not write out !!
    Pipe<RowSchema<M>> result = new Pipe<RowSchema<M>>(new PipeConfig<RowSchema<M>>(rowResultSchema, /*Math.min(16,*/
    resultSchema.getRows()));
    // Pipe<DecimalSchema<MatrixSchema>> result2 = new Pipe<DecimalSchema<MatrixSchema>>(new PipeConfig<DecimalSchema<MatrixSchema>>(result2Schema, resultSchema.getRows())); //NOTE: reqires 2 or JSON will not write out !!
    // 60; //105967ms
    int targetThreadCount = 10;
    Pipe<ColumnSchema<M>>[] colResults = BuildMatrixCompute.buildProductGraphRC(gm, left, right, targetThreadCount - 2);
    // int x = colResults.length;
    // PipeCleanerStage[] watches = new PipeCleanerStage[colResults.length];
    // while (--x>=0) {
    // watches[x] =  new PipeCleanerStage<>(gm, colResults[x]);
    // }
    ColumnsToRowsStage<M> ctr = new ColumnsToRowsStage(gm, colResults, result);
    // ByteArrayOutputStream baos = new ByteArrayOutputStream();
    // ConvertToDecimalStage<MatrixSchema> convert = new ConvertToDecimalStage<MatrixSchema>(gm, resultSchema, result, result2);
    // ConsoleJSONDumpStage<?> watch = new ConsoleJSONDumpStage<>(gm, result , new PrintStream(baos));
    // ConsoleSummaryStage<RowSchema<MatrixSchema>> watch = new ConsoleSummaryStage<>(gm, result);
    PipeCleanerStage watch = new PipeCleanerStage<>(gm, result);
    GraphManager.enableBatching(gm);
    // gm.exportGraphDotFile();//TODO: this may not work and cause issues.
    // MonitorConsoleStage.attach(gm);
    StageScheduler scheduler = new ThreadPerStageScheduler(gm);
    // new FixedThreadsScheduler(gm, targetThreadCount);
    scheduler.startup();
    // 5000;
    int testSize = 100;
    int k = testSize;
    long timeout = 0;
    while (--k >= 0) {
        timeout = System.currentTimeMillis() + 5000;
        for (int c = 0; c < rightRows; c++) {
            while (!Pipe.hasRoomForWrite(right)) {
                Thread.yield();
                if (System.currentTimeMillis() > timeout) {
                    scheduler.shutdown();
                    scheduler.awaitTermination(20, TimeUnit.SECONDS);
                    System.err.println("A, unable to write all of test data!");
                    return;
                }
            }
            Pipe.addMsgIdx(right, resultSchema.rowId);
            for (int r = 0; r < rightColumns; r++) {
                type.addValue(c, right);
            }
            Pipe.confirmLowLevelWrite(right, Pipe.sizeOf(right, resultSchema.rowId));
            Pipe.publishWrites(right);
        }
        for (int c = 0; c < leftRows; c++) {
            while (!Pipe.hasRoomForWrite(left)) {
                Thread.yield();
                if (System.currentTimeMillis() > timeout) {
                    scheduler.shutdown();
                    scheduler.awaitTermination(20, TimeUnit.SECONDS);
                    System.err.println("B, unable to write all of test data!");
                    return;
                }
            }
            Pipe.addMsgIdx(left, resultSchema.rowId);
            for (int r = 0; r < leftColumns; r++) {
                type.addValue(r, left);
            }
            Pipe.confirmLowLevelWrite(left, Pipe.sizeOf(left, resultSchema.rowId));
            Pipe.publishWrites(left);
        }
    }
    if (k < 0) {
        // only works because we have multiple threads in play
        while (!Pipe.hasRoomForWrite(left, Pipe.EOF_SIZE)) {
            Pipe.spinWork(left);
        }
        while (!Pipe.hasRoomForWrite(right, Pipe.EOF_SIZE)) {
            Pipe.spinWork(right);
        }
        Pipe.publishEOF(left);
        Pipe.publishEOF(right);
    }
    if (GraphManager.blockUntilStageBeginsShutdown(gm, watch, 6000)) {
    // timeout in ms
    } else {
        System.err.println("time out!");
    }
    scheduler.shutdown();
    // all the requests have now been sent.
    // int w = watches.length;
    // while (--w>=0) {
    // if (!GraphManager.blockUntilStageBeginsShutdown(gm, watches[w], 5000)) {//timeout in ms
    // System.err.println("ERROR SHUTDOWN");
    // System.exit(-1);
    // }
    // }
    scheduler.awaitTermination(10, TimeUnit.SECONDS);
// System.out.println("len "+baos.toByteArray().length+"  "+new String(baos.toByteArray()));
}

Example 27 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project GreenLightning by oci-pronghorn.

the class BuilderImpl method buildStages.

public void buildStages(MsgRuntime runtime) {
    IntHashTable subscriptionPipeLookup2 = MsgRuntime.getSubPipeLookup(runtime);
    GraphManager gm = MsgRuntime.getGraphManager(runtime);
    Pipe<NetResponseSchema>[] httpClientResponsePipes = GraphManager.allPipesOfTypeWithNoProducer(gm, NetResponseSchema.instance);
    Pipe<MessageSubscription>[] subscriptionPipes = GraphManager.allPipesOfTypeWithNoProducer(gm, MessageSubscription.instance);
    Pipe<TrafficOrderSchema>[] orderPipes = GraphManager.allPipesOfTypeWithNoConsumer(gm, TrafficOrderSchema.instance);
    Pipe<ClientHTTPRequestSchema>[] httpClientRequestPipes = GraphManager.allPipesOfTypeWithNoConsumer(gm, ClientHTTPRequestSchema.instance);
    Pipe<MessagePubSub>[] messagePubSub = GraphManager.allPipesOfTypeWithNoConsumer(gm, MessagePubSub.instance);
    Pipe<IngressMessages>[] ingressMessagePipes = GraphManager.allPipesOfTypeWithNoConsumer(gm, IngressMessages.instance);
    // TODO: no longer right now that we have no cops..
    int commandChannelCount = orderPipes.length;
    int eventSchemas = 0;
    IDX_MSG = (IntHashTable.isEmpty(subscriptionPipeLookup2) && subscriptionPipes.length == 0 && messagePubSub.length == 0) ? -1 : eventSchemas++;
    IDX_NET = useNetClient(httpClientRequestPipes) ? eventSchemas++ : -1;
    // 20 seconds
    long timeout = 20_000;
    int maxGoPipeId = 0;
    Pipe<TrafficReleaseSchema>[][] masterGoOut = new Pipe[eventSchemas][0];
    Pipe<TrafficAckSchema>[][] masterAckIn = new Pipe[eventSchemas][0];
    if (IDX_MSG >= 0) {
        masterGoOut[IDX_MSG] = new Pipe[messagePubSub.length];
        masterAckIn[IDX_MSG] = new Pipe[messagePubSub.length];
    }
    if (IDX_NET >= 0) {
        masterGoOut[IDX_NET] = new Pipe[httpClientResponsePipes.length];
        masterAckIn[IDX_NET] = new Pipe[httpClientResponsePipes.length];
    }
    int copGoAck = commandChannelCount;
    // logger.info("command channel count to be checked {}",copGoAck);
    while (--copGoAck >= 0) {
        Pipe<TrafficReleaseSchema>[] goOut = new Pipe[eventSchemas];
        Pipe<TrafficAckSchema>[] ackIn = new Pipe[eventSchemas];
        // only setup the go and in pipes if the cop is used.
        if (null != orderPipes[copGoAck]) {
            int features = getFeatures(gm, orderPipes[copGoAck]);
            boolean hasConnections = false;
            if ((features & Behavior.DYNAMIC_MESSAGING) != 0) {
                hasConnections = true;
                maxGoPipeId = populateGoAckPipes(maxGoPipeId, masterGoOut, masterAckIn, goOut, ackIn, IDX_MSG);
            }
            if ((features & Behavior.NET_REQUESTER) != 0) {
                hasConnections = true;
                maxGoPipeId = populateGoAckPipes(maxGoPipeId, masterGoOut, masterAckIn, goOut, ackIn, IDX_NET);
            }
            TrafficCopStage.newInstance(gm, timeout, orderPipes[copGoAck], ackIn, goOut, runtime, this);
        } else {
            logger.info("oops get features skipped since no cops but needed for private topics");
        }
    // if (true | hasConnections) {
    // TrafficCopStage trafficCopStage = new TrafficCopStage(gm,
    // timeout, orderPipes[t],
    // ackIn, goOut,
    // runtime, this);
    // } else {
    // //this optimization can no longer be done due to the use of shutdown on command channel.
    // //    revisit this later...
    // //TODO: we can reintroduce this as long has we have a stage here which does shutdown on -1;
    // PipeCleanerStage.newInstance(gm, orderPipes[t]);
    // }
    }
    initChannelBlocker(maxGoPipeId);
    buildHTTPClientGraph(runtime, httpClientResponsePipes, httpClientRequestPipes, masterGoOut, masterAckIn);
    // TODO: only create when subscriptionPipeLookup is not empty and subscriptionPipes has zero length.
    if (IDX_MSG < 0) {
        logger.trace("saved some resources by not starting up the unused pub sub service.");
    } else {
        if (!isAllPrivateTopics) {
            // logger.info("builder created pub sub");
            createMessagePubSubStage(runtime, subscriptionPipeLookup2, ingressMessagePipes, messagePubSub, masterGoOut[IDX_MSG], masterAckIn[IDX_MSG], subscriptionPipes);
        }
    }
}

Example 28 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project GreenLightning by oci-pronghorn.

the class BuilderImpl method buildHTTPClientGraph.

public void buildHTTPClientGraph(MsgRuntime<?, ?> runtime, Pipe<NetResponseSchema>[] netResponsePipes, Pipe<ClientHTTPRequestSchema>[] netRequestPipes, Pipe<TrafficReleaseSchema>[][] masterGoOut, Pipe<TrafficAckSchema>[][] masterAckIn) {
    // //////
    if (useNetClient(netRequestPipes)) {
        int maxPartialResponses = Math.max(2, ClientHostPortInstance.getSessionCount());
        int connectionsInBits = (int) Math.ceil(Math.log(maxPartialResponses) / Math.log(2));
        int netResponseCount = 8;
        int responseQueue = 10;
        // must be adjusted together
        // Multipler per session for total connections ,count of pipes to channel writer
        int outputsCount = 8;
        // count of channel writer stages
        int clientWriters = 1;
        PipeConfig<NetPayloadSchema> clientNetRequestConfig = pcm.getConfig(NetPayloadSchema.class);
        // BUILD GRAPH
        ccm = new ClientCoordinator(connectionsInBits, maxPartialResponses, this.client.getCertificates(), gm.recordTypeData);
        Pipe<NetPayloadSchema>[] clientRequests = new Pipe[outputsCount];
        int r = outputsCount;
        while (--r >= 0) {
            clientRequests[r] = new Pipe<NetPayloadSchema>(clientNetRequestConfig);
        }
        if (isAllNull(masterGoOut[IDX_NET])) {
            // this one has much lower latency and should be used if possible
            new HTTPClientRequestStage(gm, ccm, netRequestPipes, clientRequests);
        } else {
            logger.info("Warning, the slower HTTP Client Request code was called. 2ms latency may be introduced.");
            // this may stay for as long as 2ms before returning due to timeout of
            // traffic logic, this is undesirable in some low latency cases.
            new HTTPClientRequestTrafficStage(gm, runtime, this, ccm, netRequestPipes, masterGoOut[IDX_NET], masterAckIn[IDX_NET], clientRequests);
        }
        int releaseCount = 1024;
        // takes more memory but limits writes, each creating a little GC
        int writeBufferMultiplier = 100;
        int responseUnwrapCount = 2;
        int clientWrapperCount = 2;
        NetGraphBuilder.buildHTTPClientGraph(gm, ccm, responseQueue, clientRequests, netResponsePipes, netResponseCount, releaseCount, writeBufferMultiplier, responseUnwrapCount, clientWrapperCount, clientWriters);
    }
}

Example 29 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project GreenLightning by oci-pronghorn.

the class BuilderImpl method lazyCreatePipeLookupMatrix.

private void lazyCreatePipeLookupMatrix() {
    if (null == collectedHTTPRequstPipes) {
        int parallelism = parallelTracks();
        int routesCount = routerConfig().totalPathsCount();
        assert (parallelism >= 1);
        assert (routesCount > -1);
        // for catch all route since we have no specific routes.
        if (routesCount == 0) {
            routesCount = 1;
        }
        collectedHTTPRequstPipes = (ArrayList<Pipe<HTTPRequestSchema>>[][]) new ArrayList[parallelism][routesCount];
        int p = parallelism;
        while (--p >= 0) {
            int r = routesCount;
            while (--r >= 0) {
                collectedHTTPRequstPipes[p][r] = new ArrayList();
            }
        }
    }
}

Example 30 with Pipe

use of com.ociweb.pronghorn.pipe.Pipe in project GreenLightning by oci-pronghorn.

the class MsgRuntime method addFileServer.

public RouteFilter addFileServer(String path) {
    // adds server to all routes
    final int parallelIndex = (-1 == parallelInstanceUnderActiveConstruction) ? 0 : parallelInstanceUnderActiveConstruction;
    // due to internal implementation we must keep the same number of outputs as inputs.
    Pipe<HTTPRequestSchema>[] inputs = new Pipe[1];
    Pipe<ServerResponseSchema>[] outputs = new Pipe[1];
    populateHTTPInOut(inputs, outputs, 0, parallelIndex);
    File rootPath = buildFilePath(path);
    FileReadModuleStage.newInstance(gm, inputs, outputs, builder.httpSpec, rootPath);
    return new StageRouteFilter(inputs[0], builder, parallelIndex);
}