Search in sources :

Example 1 with ScanStatsRawDataMessage

use of ml.shifu.shifu.message.ScanStatsRawDataMessage in project shifu by ShifuML.

the class DataLoadWorker method handleMsg.

/*
     * (non-Javadoc)
     * 
     * @see akka.actor.UntypedActor#onReceive(java.lang.Object)
     */
@Override
public void handleMsg(Object message) {
    if (message instanceof ScanStatsRawDataMessage) {
        log.info("DataLoaderActor Starting ...");
        ScanStatsRawDataMessage msg = (ScanStatsRawDataMessage) message;
        Scanner scanner = msg.getScanner();
        int totalMsgCnt = msg.getTotalMsgCnt();
        List<String> rawDataList = readDataIntoList(scanner);
        log.info("DataLoaderActor Finished: Loaded " + rawDataList.size() + " Records.");
        nextActorRef.tell(new StatsPartRawDataMessage(totalMsgCnt, rawDataList), getSelf());
    } else if (message instanceof ScanNormInputDataMessage) {
        log.info("DataLoaderActor Starting ...");
        ScanNormInputDataMessage msg = (ScanNormInputDataMessage) message;
        Scanner scanner = msg.getScanner();
        int totalMsgCnt = msg.getTotalMsgCnt();
        List<String> rawDataList = readDataIntoList(scanner);
        log.info("DataLoaderActor Finished: Loaded " + rawDataList.size() + " Records.");
        nextActorRef.tell(new NormPartRawDataMessage(totalMsgCnt, rawDataList), getSelf());
    } else if (message instanceof ScanTrainDataMessage) {
        ScanTrainDataMessage msg = (ScanTrainDataMessage) message;
        Scanner scanner = msg.getScanner();
        int totalMsgCnt = msg.getTotalMsgCnt();
        List<MLDataPair> mlDataPairList = readTrainingData(scanner, msg.isDryRun());
        log.info("DataLoaderActor Finished: Loaded " + mlDataPairList.size() + " Records for Training.");
        nextActorRef.tell(new TrainPartDataMessage(totalMsgCnt, msg.isDryRun(), mlDataPairList), getSelf());
    } else if (message instanceof ScanEvalDataMessage) {
        log.info("DataLoaderActor Starting ...");
        ScanEvalDataMessage msg = (ScanEvalDataMessage) message;
        Scanner scanner = msg.getScanner();
        int streamId = msg.getStreamId();
        int totalStreamCnt = msg.getTotalStreamCnt();
        splitDataIntoMultiMessages(streamId, totalStreamCnt, scanner, Environment.getInt(Environment.RECORD_CNT_PER_MESSAGE, 100000));
    /*
             * List<String> evalDataList = readDataIntoList(scanner);
             * 
             * log.info("DataLoaderActor Finished: Loaded " + evalDataList.size() + " Records.");
             * nextActorRef.tell( new RunModelDataMessage(totalMsgCnt, evalDataList), getSelf());
             */
    } else {
        unhandled(message);
    }
}
Also used : BasicMLDataPair(org.encog.ml.data.basic.BasicMLDataPair) MLDataPair(org.encog.ml.data.MLDataPair) Scanner(java.util.Scanner) StatsPartRawDataMessage(ml.shifu.shifu.message.StatsPartRawDataMessage) ScanTrainDataMessage(ml.shifu.shifu.message.ScanTrainDataMessage) ScanNormInputDataMessage(ml.shifu.shifu.message.ScanNormInputDataMessage) ScanStatsRawDataMessage(ml.shifu.shifu.message.ScanStatsRawDataMessage) TrainPartDataMessage(ml.shifu.shifu.message.TrainPartDataMessage) NormPartRawDataMessage(ml.shifu.shifu.message.NormPartRawDataMessage) ScanEvalDataMessage(ml.shifu.shifu.message.ScanEvalDataMessage) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) List(java.util.List)

Example 2 with ScanStatsRawDataMessage

use of ml.shifu.shifu.message.ScanStatsRawDataMessage in project shifu by ShifuML.

the class TrainDtModelActor method onReceive.

/* (non-Javadoc)
     * @see akka.actor.UntypedActor#onReceive(java.lang.Object)
     */
@Override
public void onReceive(Object message) throws Exception {
    if (message instanceof AkkaActorInputMessage) {
        resultCnt = 0;
        AkkaActorInputMessage msg = (AkkaActorInputMessage) message;
        List<Scanner> scanners = msg.getScanners();
        log.debug("Num of Scanners: " + scanners.size());
        for (Scanner scanner : scanners) {
            dataLoadRef.tell(new ScanStatsRawDataMessage(scanners.size(), scanner), getSelf());
        }
    } else if (message instanceof TrainResultMessage) {
        resultCnt++;
        if (resultCnt == trainerCnt) {
            log.info("Received " + resultCnt + " finish message. Close System.");
            Encog.getInstance().shutdown();
            getContext().system().shutdown();
        }
    } else if (message instanceof ExceptionMessage) {
        // since some children actors meet some exception, shutdown the system
        ExceptionMessage msg = (ExceptionMessage) message;
        getContext().system().shutdown();
        // and wrapper the exception into Return status
        addExceptionIntoCondition(msg.getException());
    } else {
        unhandled(message);
    }
}
Also used : AkkaActorInputMessage(ml.shifu.shifu.message.AkkaActorInputMessage) Scanner(java.util.Scanner) ExceptionMessage(ml.shifu.shifu.message.ExceptionMessage) TrainResultMessage(ml.shifu.shifu.message.TrainResultMessage) ScanStatsRawDataMessage(ml.shifu.shifu.message.ScanStatsRawDataMessage)

Example 3 with ScanStatsRawDataMessage

use of ml.shifu.shifu.message.ScanStatsRawDataMessage in project shifu by ShifuML.

the class CalculateStatsActor method onReceive.

/* (non-Javadoc)
     * @see akka.actor.UntypedActor#onReceive(java.lang.Object)
     */
@Override
public void onReceive(Object message) throws Exception {
    if (message instanceof AkkaActorInputMessage) {
        resultCnt = 0;
        AkkaActorInputMessage msg = (AkkaActorInputMessage) message;
        List<Scanner> scanners = msg.getScanners();
        log.debug("Num of Scanners: " + scanners.size());
        for (Scanner scanner : scanners) {
            dataLoadRef.tell(new ScanStatsRawDataMessage(scanners.size(), scanner), getSelf());
        }
    } else if (message instanceof StatsResultMessage) {
        StatsResultMessage statsRstMsg = (StatsResultMessage) message;
        ColumnConfig columnConfig = statsRstMsg.getColumnConfig();
        columnConfigList.set(columnConfig.getColumnNum(), columnConfig);
        resultCnt++;
        if (resultCnt == columnNumToActorMap.size()) {
            log.info("Received " + resultCnt + " messages. Finished Calculating Stats.");
            PathFinder pathFinder = new PathFinder(modelConfig);
            JSONUtils.writeValue(new File(pathFinder.getColumnConfigPath()), columnConfigList);
            getContext().system().shutdown();
        }
    } else if (message instanceof ExceptionMessage) {
        // since some children actors meet some exception, shutdown the system
        ExceptionMessage msg = (ExceptionMessage) message;
        getContext().system().shutdown();
        // and wrapper the exception into Return status
        addExceptionIntoCondition(msg.getException());
    } else {
        unhandled(message);
    }
}
Also used : AkkaActorInputMessage(ml.shifu.shifu.message.AkkaActorInputMessage) Scanner(java.util.Scanner) ExceptionMessage(ml.shifu.shifu.message.ExceptionMessage) ColumnConfig(ml.shifu.shifu.container.obj.ColumnConfig) StatsResultMessage(ml.shifu.shifu.message.StatsResultMessage) PathFinder(ml.shifu.shifu.fs.PathFinder) File(java.io.File) ScanStatsRawDataMessage(ml.shifu.shifu.message.ScanStatsRawDataMessage)

Aggregations

Scanner (java.util.Scanner)3 ScanStatsRawDataMessage (ml.shifu.shifu.message.ScanStatsRawDataMessage)3 AkkaActorInputMessage (ml.shifu.shifu.message.AkkaActorInputMessage)2 ExceptionMessage (ml.shifu.shifu.message.ExceptionMessage)2 File (java.io.File)1 ArrayList (java.util.ArrayList)1 LinkedList (java.util.LinkedList)1 List (java.util.List)1 ColumnConfig (ml.shifu.shifu.container.obj.ColumnConfig)1 PathFinder (ml.shifu.shifu.fs.PathFinder)1 NormPartRawDataMessage (ml.shifu.shifu.message.NormPartRawDataMessage)1 ScanEvalDataMessage (ml.shifu.shifu.message.ScanEvalDataMessage)1 ScanNormInputDataMessage (ml.shifu.shifu.message.ScanNormInputDataMessage)1 ScanTrainDataMessage (ml.shifu.shifu.message.ScanTrainDataMessage)1 StatsPartRawDataMessage (ml.shifu.shifu.message.StatsPartRawDataMessage)1 StatsResultMessage (ml.shifu.shifu.message.StatsResultMessage)1 TrainPartDataMessage (ml.shifu.shifu.message.TrainPartDataMessage)1 TrainResultMessage (ml.shifu.shifu.message.TrainResultMessage)1 MLDataPair (org.encog.ml.data.MLDataPair)1 BasicMLDataPair (org.encog.ml.data.basic.BasicMLDataPair)1