Examples with GraphManager com.ociweb.pronghorn.stage.scheduling.GraphManager used on opensource projects

Example 1 with GraphManager

use of com.ociweb.pronghorn.stage.scheduling.GraphManager in project GreenLightning by oci-pronghorn.

the class GreenLightning method main.

public static void main(String[] args) {
    String path = HTTPServer.getOptArg("-site", "--s", args, null);
    String resourceRoot = HTTPServer.getOptArg("-resourcesRoot", "--rr", args, null == path ? "/site/index.html" : null);
    String rootFolder = null;
    if (null == path) {
        if (null == resourceRoot) {
            System.out.println("Path to site must be defined with -site or --s");
            return;
        } else {
            // use internal resources
            int endOfRoot = resourceRoot.lastIndexOf('/');
            if (-1 == endOfRoot) {
                System.out.println("resourceRoot must contain at least one / to define the subfolder inside the resources folder");
                return;
            }
            rootFolder = resourceRoot.substring(0, endOfRoot);
            System.out.println("reading site data from internal resources: " + rootFolder);
        }
    } else {
        if (null == resourceRoot) {
            // normal file path site
            System.out.println("reading site data from: " + path);
        } else {
            System.out.println("use -size for file paths or -resourcesRoot for packaged resources. Only one can be used at a time.");
            return;
        }
    }
    String isTLS = HTTPServer.getOptArg("-tls", "--t", args, "True");
    String isLarge = HTTPServer.getOptArg("-large", "--l", args, "False");
    String strPort = HTTPServer.getOptArg("-port", "--p", args, "8080");
    int port = Integer.parseInt(strPort);
    String bindHost = HTTPServer.getOptArg("-host", "--h", args, null);
    boolean large = Boolean.parseBoolean(isLarge);
    if (null == bindHost) {
        bindHost = bindHost();
    }
    // makes big performance difference.
    final int fileOutgoing = large ? 2048 : 1024;
    final int fileChunkSize = large ? 1 << 14 : 1 << 10;
    int processors = large ? 8 : -1;
    GraphManager gm = new GraphManager();
    TLSCertificates certs = Boolean.parseBoolean(isTLS) ? TLSCertificates.defaultCerts : null;
    HTTPServer.startupHTTPServer(gm, processors, GreenLightning.simpleModuleConfig(path, resourceRoot, rootFolder, fileOutgoing, fileChunkSize), bindHost, port, certs);
    System.out.println("Press \"ENTER\" to exit...");
    int value = -1;
    do {
        try {
            value = System.in.read();
        } catch (IOException e) {
            e.printStackTrace();
        }
    } while (value != 10);
    System.exit(0);
}

Example 2 with GraphManager

use of com.ociweb.pronghorn.stage.scheduling.GraphManager in project FogLight-Examples by oci-pronghorn.

the class IoTApp method matrixTest.

public <M extends MatrixSchema<M>> void matrixTest() {
    // speed
    // slow     Doubles  Longs    6.15 5.8      7.024  7.18
    // Decimals          5.9           9.40 - 13
    // Floats            6.06           6.26
    // fast     Integers          5.80           5.95
    // Decimals;//Integers; //2, 3328335 longs/ints/doubles   [0,332833152] floats
    MatrixTypes type = MatrixTypes.Integers;
    // TypeMask.Decimal;
    // when matrix size is larger than CPU cache we run into issues.
    int leftRows = 1200;
    int leftColumns = 1000;
    int rightColumns = 1000;
    int rightRows = leftColumns;
    // walk leftRows , by rightCol for output
    // 5x2
    // 2x3
    MatrixSchema leftSchema = BuildMatrixCompute.buildSchema(leftRows, leftColumns, type);
    RowSchema<M> leftRowSchema = new RowSchema<M>(leftSchema);
    MatrixSchema rightSchema = BuildMatrixCompute.buildSchema(rightRows, rightColumns, type);
    RowSchema<M> rightRowSchema = new RowSchema<M>(rightSchema);
    MatrixSchema resultSchema = BuildMatrixCompute.buildResultSchema(leftSchema, rightSchema);
    RowSchema<M> rowResultSchema = new RowSchema<M>(resultSchema);
    DecimalSchema result2Schema = new DecimalSchema<M>(resultSchema);
    GraphManager gm = new GraphManager();
    // GraphManager.addDefaultNota(gm, GraphManager.SCHEDULE_RATE, 500);
    // TODO: not sure why but the splitter that consumes left needs a minimum ring size or it gets stuck,
    Pipe<RowSchema<M>> left = new Pipe<RowSchema<M>>(new PipeConfig<RowSchema<M>>(leftRowSchema, leftRows));
    Pipe<RowSchema<M>> right = new Pipe<RowSchema<M>>(new PipeConfig<RowSchema<M>>(rightRowSchema, rightRows));
    // NOTE: reqires 2 or JSON will not write out !!
    Pipe<RowSchema<M>> result = new Pipe<RowSchema<M>>(new PipeConfig<RowSchema<M>>(rowResultSchema, /*Math.min(16,*/
    resultSchema.getRows()));
    // Pipe<DecimalSchema<MatrixSchema>> result2 = new Pipe<DecimalSchema<MatrixSchema>>(new PipeConfig<DecimalSchema<MatrixSchema>>(result2Schema, resultSchema.getRows())); //NOTE: reqires 2 or JSON will not write out !!
    // 60; //105967ms
    int targetThreadCount = 10;
    Pipe<ColumnSchema<M>>[] colResults = BuildMatrixCompute.buildProductGraphRC(gm, left, right, targetThreadCount - 2);
    // int x = colResults.length;
    // PipeCleanerStage[] watches = new PipeCleanerStage[colResults.length];
    // while (--x>=0) {
    // watches[x] =  new PipeCleanerStage<>(gm, colResults[x]);
    // }
    ColumnsToRowsStage<M> ctr = new ColumnsToRowsStage(gm, colResults, result);
    // ByteArrayOutputStream baos = new ByteArrayOutputStream();
    // ConvertToDecimalStage<MatrixSchema> convert = new ConvertToDecimalStage<MatrixSchema>(gm, resultSchema, result, result2);
    // ConsoleJSONDumpStage<?> watch = new ConsoleJSONDumpStage<>(gm, result , new PrintStream(baos));
    // ConsoleSummaryStage<RowSchema<MatrixSchema>> watch = new ConsoleSummaryStage<>(gm, result);
    PipeCleanerStage watch = new PipeCleanerStage<>(gm, result);
    GraphManager.enableBatching(gm);
    // gm.exportGraphDotFile();//TODO: this may not work and cause issues.
    // MonitorConsoleStage.attach(gm);
    StageScheduler scheduler = new ThreadPerStageScheduler(gm);
    // new FixedThreadsScheduler(gm, targetThreadCount);
    scheduler.startup();
    // 5000;
    int testSize = 100;
    int k = testSize;
    long timeout = 0;
    while (--k >= 0) {
        timeout = System.currentTimeMillis() + 5000;
        for (int c = 0; c < rightRows; c++) {
            while (!Pipe.hasRoomForWrite(right)) {
                Thread.yield();
                if (System.currentTimeMillis() > timeout) {
                    scheduler.shutdown();
                    scheduler.awaitTermination(20, TimeUnit.SECONDS);
                    System.err.println("A, unable to write all of test data!");
                    return;
                }
            }
            Pipe.addMsgIdx(right, resultSchema.rowId);
            for (int r = 0; r < rightColumns; r++) {
                type.addValue(c, right);
            }
            Pipe.confirmLowLevelWrite(right, Pipe.sizeOf(right, resultSchema.rowId));
            Pipe.publishWrites(right);
        }
        for (int c = 0; c < leftRows; c++) {
            while (!Pipe.hasRoomForWrite(left)) {
                Thread.yield();
                if (System.currentTimeMillis() > timeout) {
                    scheduler.shutdown();
                    scheduler.awaitTermination(20, TimeUnit.SECONDS);
                    System.err.println("B, unable to write all of test data!");
                    return;
                }
            }
            Pipe.addMsgIdx(left, resultSchema.rowId);
            for (int r = 0; r < leftColumns; r++) {
                type.addValue(r, left);
            }
            Pipe.confirmLowLevelWrite(left, Pipe.sizeOf(left, resultSchema.rowId));
            Pipe.publishWrites(left);
        }
    }
    if (k < 0) {
        // only works because we have multiple threads in play
        while (!Pipe.hasRoomForWrite(left, Pipe.EOF_SIZE)) {
            Pipe.spinWork(left);
        }
        while (!Pipe.hasRoomForWrite(right, Pipe.EOF_SIZE)) {
            Pipe.spinWork(right);
        }
        Pipe.publishEOF(left);
        Pipe.publishEOF(right);
    }
    if (GraphManager.blockUntilStageBeginsShutdown(gm, watch, 6000)) {
    // timeout in ms
    } else {
        System.err.println("time out!");
    }
    scheduler.shutdown();
    // all the requests have now been sent.
    // int w = watches.length;
    // while (--w>=0) {
    // if (!GraphManager.blockUntilStageBeginsShutdown(gm, watches[w], 5000)) {//timeout in ms
    // System.err.println("ERROR SHUTDOWN");
    // System.exit(-1);
    // }
    // }
    scheduler.awaitTermination(10, TimeUnit.SECONDS);
// System.out.println("len "+baos.toByteArray().length+"  "+new String(baos.toByteArray()));
}

Example 3 with GraphManager

use of com.ociweb.pronghorn.stage.scheduling.GraphManager in project FogLight-Examples by oci-pronghorn.

the class UARTDataParserTest method ParserStageTest.

@Test
public void ParserStageTest() {
    GraphManager gm = new GraphManager();
    Pipe<RawDataSchema> input = RawDataSchema.instance.newPipe(4, 512);
    Pipe<ValveSchema> filter = ValveSchema.instance.newPipe(64, 128);
    Pipe<ValveSchema> output = ValveSchema.instance.newPipe(64, 128);
    ByteArrayOutputStream results = new ByteArrayOutputStream();
    ValveDataParserStage.newInstance(gm, input, filter);
    FilterStage.newInstance(gm, filter, output);
    ConsoleJSONDumpStage.newInstance(gm, output, new PrintStream(results));
    // MonitorConsoleStage.attach(gm);
    NonThreadScheduler scheduler = new NonThreadScheduler(gm);
    scheduler.startup();
    // ////////////////
    // //setup the test data
    // ////////////////
    Pipe.addMsgIdx(input, RawDataSchema.MSG_CHUNKEDSTREAM_1);
    String message = "587lf5pf\"L\"vf0sp55] [" + "st1" + "sn100100" + "pn\"NX-DCV-SM-BLU-1-1-VO-L1-SO-OO\"" + "lr-100" + "cc184587" + "lf0" + "pf\"L\"" + "vf0" + "sp80" + "vf0" + "sp42" + "]";
    Pipe.addUTF8(message, input);
    Pipe.confirmLowLevelWrite(input, Pipe.sizeOf(input, RawDataSchema.MSG_CHUNKEDSTREAM_1));
    Pipe.publishWrites(input);
    // ////////////////
    int i = 1000;
    while (--i >= 0) {
        scheduler.run();
    }
    scheduler.shutdown();
    // //////////////
    // /confirm the results
    // //////////////
    String stringResults = new String(results.toByteArray());
    // for debug to inspect the values
    System.err.println(stringResults);
    assertFalse(stringResults.contains("55"));
    assertTrue(stringResults.contains("{\"PartNumber\":\"NX-DCV-SM-BLU-1-1-VO-L1-SO-OO\"}"));
    assertTrue(stringResults.contains("{\"ResidualOfDynamicAnalysis\":4294967196}"));
    int firstSP = stringResults.indexOf("{\"SupplyPressure\":80}");
    assertTrue(firstSP != -1);
    int secondSP = stringResults.indexOf("{\"SupplyPressure\":42}", firstSP + 1);
    assertTrue(secondSP != -1);
    int firstVP = stringResults.indexOf("{\"ValveFault\":0}");
    assertTrue(firstVP != -1);
    int secondVP = stringResults.indexOf("{\"ValveFault\":0}", firstVP + 1);
    assertTrue(secondVP == -1);
}

Example 4 with GraphManager

use of com.ociweb.pronghorn.stage.scheduling.GraphManager in project GreenLightning by oci-pronghorn.

the class BuilderImpl method buildStages.

public void buildStages(MsgRuntime runtime) {
    IntHashTable subscriptionPipeLookup2 = MsgRuntime.getSubPipeLookup(runtime);
    GraphManager gm = MsgRuntime.getGraphManager(runtime);
    Pipe<NetResponseSchema>[] httpClientResponsePipes = GraphManager.allPipesOfTypeWithNoProducer(gm, NetResponseSchema.instance);
    Pipe<MessageSubscription>[] subscriptionPipes = GraphManager.allPipesOfTypeWithNoProducer(gm, MessageSubscription.instance);
    Pipe<TrafficOrderSchema>[] orderPipes = GraphManager.allPipesOfTypeWithNoConsumer(gm, TrafficOrderSchema.instance);
    Pipe<ClientHTTPRequestSchema>[] httpClientRequestPipes = GraphManager.allPipesOfTypeWithNoConsumer(gm, ClientHTTPRequestSchema.instance);
    Pipe<MessagePubSub>[] messagePubSub = GraphManager.allPipesOfTypeWithNoConsumer(gm, MessagePubSub.instance);
    Pipe<IngressMessages>[] ingressMessagePipes = GraphManager.allPipesOfTypeWithNoConsumer(gm, IngressMessages.instance);
    // TODO: no longer right now that we have no cops..
    int commandChannelCount = orderPipes.length;
    int eventSchemas = 0;
    IDX_MSG = (IntHashTable.isEmpty(subscriptionPipeLookup2) && subscriptionPipes.length == 0 && messagePubSub.length == 0) ? -1 : eventSchemas++;
    IDX_NET = useNetClient(httpClientRequestPipes) ? eventSchemas++ : -1;
    // 20 seconds
    long timeout = 20_000;
    int maxGoPipeId = 0;
    Pipe<TrafficReleaseSchema>[][] masterGoOut = new Pipe[eventSchemas][0];
    Pipe<TrafficAckSchema>[][] masterAckIn = new Pipe[eventSchemas][0];
    if (IDX_MSG >= 0) {
        masterGoOut[IDX_MSG] = new Pipe[messagePubSub.length];
        masterAckIn[IDX_MSG] = new Pipe[messagePubSub.length];
    }
    if (IDX_NET >= 0) {
        masterGoOut[IDX_NET] = new Pipe[httpClientResponsePipes.length];
        masterAckIn[IDX_NET] = new Pipe[httpClientResponsePipes.length];
    }
    int copGoAck = commandChannelCount;
    // logger.info("command channel count to be checked {}",copGoAck);
    while (--copGoAck >= 0) {
        Pipe<TrafficReleaseSchema>[] goOut = new Pipe[eventSchemas];
        Pipe<TrafficAckSchema>[] ackIn = new Pipe[eventSchemas];
        // only setup the go and in pipes if the cop is used.
        if (null != orderPipes[copGoAck]) {
            int features = getFeatures(gm, orderPipes[copGoAck]);
            boolean hasConnections = false;
            if ((features & Behavior.DYNAMIC_MESSAGING) != 0) {
                hasConnections = true;
                maxGoPipeId = populateGoAckPipes(maxGoPipeId, masterGoOut, masterAckIn, goOut, ackIn, IDX_MSG);
            }
            if ((features & Behavior.NET_REQUESTER) != 0) {
                hasConnections = true;
                maxGoPipeId = populateGoAckPipes(maxGoPipeId, masterGoOut, masterAckIn, goOut, ackIn, IDX_NET);
            }
            TrafficCopStage.newInstance(gm, timeout, orderPipes[copGoAck], ackIn, goOut, runtime, this);
        } else {
            logger.info("oops get features skipped since no cops but needed for private topics");
        }
    // if (true | hasConnections) {
    // TrafficCopStage trafficCopStage = new TrafficCopStage(gm,
    // timeout, orderPipes[t],
    // ackIn, goOut,
    // runtime, this);
    // } else {
    // //this optimization can no longer be done due to the use of shutdown on command channel.
    // //    revisit this later...
    // //TODO: we can reintroduce this as long has we have a stage here which does shutdown on -1;
    // PipeCleanerStage.newInstance(gm, orderPipes[t]);
    // }
    }
    initChannelBlocker(maxGoPipeId);
    buildHTTPClientGraph(runtime, httpClientResponsePipes, httpClientRequestPipes, masterGoOut, masterAckIn);
    // TODO: only create when subscriptionPipeLookup is not empty and subscriptionPipes has zero length.
    if (IDX_MSG < 0) {
        logger.trace("saved some resources by not starting up the unused pub sub service.");
    } else {
        if (!isAllPrivateTopics) {
            // logger.info("builder created pub sub");
            createMessagePubSubStage(runtime, subscriptionPipeLookup2, ingressMessagePipes, messagePubSub, masterGoOut[IDX_MSG], masterAckIn[IDX_MSG], subscriptionPipes);
        }
    }
}