Examples with ScanNormInputDataMessage ml.shifu.shifu.message.ScanNormInputDataMessage used on opensource projects

Example 1 with ScanNormInputDataMessage

use of ml.shifu.shifu.message.ScanNormInputDataMessage in project shifu by ShifuML.

the class NormalizeDataActor method onReceive.

/*
     * (non-Javadoc)
     * 
     * @see akka.actor.UntypedActor#onReceive(java.lang.Object)
     */
@Override
public void onReceive(Object message) throws Exception {
    if (message instanceof AkkaActorInputMessage) {
        resultCnt = 0;
        AkkaActorInputMessage msg = (AkkaActorInputMessage) message;
        List<Scanner> scanners = msg.getScanners();
        for (Scanner scanner : scanners) {
            dataLoadRef.tell(new ScanNormInputDataMessage(scanners.size(), scanner), getSelf());
        }
    } else if (message instanceof NormResultDataMessage) {
        NormResultDataMessage msg = (NormResultDataMessage) message;
        int targetMsgCnt = msg.getTargetMsgCnt();
        writeDataIntoFile(msg.getNormalizedDataList());
        writeSelectDataIntoFile(msg.getSelectDataList());
        resultCnt++;
        if (resultCnt == targetMsgCnt) {
            log.info("Received " + resultCnt + " messages. Finished normalizing train data.");
            normDataWriter.close();
            selectDataWriter.close();
            getContext().system().shutdown();
        }
    } else if (message instanceof ExceptionMessage) {
        // since some children actors meet some exception, shutdown the system
        ExceptionMessage msg = (ExceptionMessage) message;
        getContext().system().shutdown();
        // and wrapper the exception into Return status
        addExceptionIntoCondition(msg.getException());
    } else {
        unhandled(message);
    }
}

Example 2 with ScanNormInputDataMessage

use of ml.shifu.shifu.message.ScanNormInputDataMessage in project shifu by ShifuML.

the class DataLoadWorker method handleMsg.

/*
     * (non-Javadoc)
     * 
     * @see akka.actor.UntypedActor#onReceive(java.lang.Object)
     */
@Override
public void handleMsg(Object message) {
    if (message instanceof ScanStatsRawDataMessage) {
        log.info("DataLoaderActor Starting ...");
        ScanStatsRawDataMessage msg = (ScanStatsRawDataMessage) message;
        Scanner scanner = msg.getScanner();
        int totalMsgCnt = msg.getTotalMsgCnt();
        List<String> rawDataList = readDataIntoList(scanner);
        log.info("DataLoaderActor Finished: Loaded " + rawDataList.size() + " Records.");
        nextActorRef.tell(new StatsPartRawDataMessage(totalMsgCnt, rawDataList), getSelf());
    } else if (message instanceof ScanNormInputDataMessage) {
        log.info("DataLoaderActor Starting ...");
        ScanNormInputDataMessage msg = (ScanNormInputDataMessage) message;
        Scanner scanner = msg.getScanner();
        int totalMsgCnt = msg.getTotalMsgCnt();
        List<String> rawDataList = readDataIntoList(scanner);
        log.info("DataLoaderActor Finished: Loaded " + rawDataList.size() + " Records.");
        nextActorRef.tell(new NormPartRawDataMessage(totalMsgCnt, rawDataList), getSelf());
    } else if (message instanceof ScanTrainDataMessage) {
        ScanTrainDataMessage msg = (ScanTrainDataMessage) message;
        Scanner scanner = msg.getScanner();
        int totalMsgCnt = msg.getTotalMsgCnt();
        List<MLDataPair> mlDataPairList = readTrainingData(scanner, msg.isDryRun());
        log.info("DataLoaderActor Finished: Loaded " + mlDataPairList.size() + " Records for Training.");
        nextActorRef.tell(new TrainPartDataMessage(totalMsgCnt, msg.isDryRun(), mlDataPairList), getSelf());
    } else if (message instanceof ScanEvalDataMessage) {
        log.info("DataLoaderActor Starting ...");
        ScanEvalDataMessage msg = (ScanEvalDataMessage) message;
        Scanner scanner = msg.getScanner();
        int streamId = msg.getStreamId();
        int totalStreamCnt = msg.getTotalStreamCnt();
        splitDataIntoMultiMessages(streamId, totalStreamCnt, scanner, Environment.getInt(Environment.RECORD_CNT_PER_MESSAGE, 100000));
    /*
             * List<String> evalDataList = readDataIntoList(scanner);
             * 
             * log.info("DataLoaderActor Finished: Loaded " + evalDataList.size() + " Records.");
             * nextActorRef.tell( new RunModelDataMessage(totalMsgCnt, evalDataList), getSelf());
             */
    } else {
        unhandled(message);
    }
}