Search in sources :

Example 1 with ScanTrainDataMessage

use of ml.shifu.shifu.message.ScanTrainDataMessage in project shifu by ShifuML.

the class DataLoadWorker method handleMsg.

/*
     * (non-Javadoc)
     * 
     * @see akka.actor.UntypedActor#onReceive(java.lang.Object)
     */
@Override
public void handleMsg(Object message) {
    if (message instanceof ScanStatsRawDataMessage) {
        log.info("DataLoaderActor Starting ...");
        ScanStatsRawDataMessage msg = (ScanStatsRawDataMessage) message;
        Scanner scanner = msg.getScanner();
        int totalMsgCnt = msg.getTotalMsgCnt();
        List<String> rawDataList = readDataIntoList(scanner);
        log.info("DataLoaderActor Finished: Loaded " + rawDataList.size() + " Records.");
        nextActorRef.tell(new StatsPartRawDataMessage(totalMsgCnt, rawDataList), getSelf());
    } else if (message instanceof ScanNormInputDataMessage) {
        log.info("DataLoaderActor Starting ...");
        ScanNormInputDataMessage msg = (ScanNormInputDataMessage) message;
        Scanner scanner = msg.getScanner();
        int totalMsgCnt = msg.getTotalMsgCnt();
        List<String> rawDataList = readDataIntoList(scanner);
        log.info("DataLoaderActor Finished: Loaded " + rawDataList.size() + " Records.");
        nextActorRef.tell(new NormPartRawDataMessage(totalMsgCnt, rawDataList), getSelf());
    } else if (message instanceof ScanTrainDataMessage) {
        ScanTrainDataMessage msg = (ScanTrainDataMessage) message;
        Scanner scanner = msg.getScanner();
        int totalMsgCnt = msg.getTotalMsgCnt();
        List<MLDataPair> mlDataPairList = readTrainingData(scanner, msg.isDryRun());
        log.info("DataLoaderActor Finished: Loaded " + mlDataPairList.size() + " Records for Training.");
        nextActorRef.tell(new TrainPartDataMessage(totalMsgCnt, msg.isDryRun(), mlDataPairList), getSelf());
    } else if (message instanceof ScanEvalDataMessage) {
        log.info("DataLoaderActor Starting ...");
        ScanEvalDataMessage msg = (ScanEvalDataMessage) message;
        Scanner scanner = msg.getScanner();
        int streamId = msg.getStreamId();
        int totalStreamCnt = msg.getTotalStreamCnt();
        splitDataIntoMultiMessages(streamId, totalStreamCnt, scanner, Environment.getInt(Environment.RECORD_CNT_PER_MESSAGE, 100000));
    /*
             * List<String> evalDataList = readDataIntoList(scanner);
             * 
             * log.info("DataLoaderActor Finished: Loaded " + evalDataList.size() + " Records.");
             * nextActorRef.tell( new RunModelDataMessage(totalMsgCnt, evalDataList), getSelf());
             */
    } else {
        unhandled(message);
    }
}
Also used : BasicMLDataPair(org.encog.ml.data.basic.BasicMLDataPair) MLDataPair(org.encog.ml.data.MLDataPair) Scanner(java.util.Scanner) StatsPartRawDataMessage(ml.shifu.shifu.message.StatsPartRawDataMessage) ScanTrainDataMessage(ml.shifu.shifu.message.ScanTrainDataMessage) ScanNormInputDataMessage(ml.shifu.shifu.message.ScanNormInputDataMessage) ScanStatsRawDataMessage(ml.shifu.shifu.message.ScanStatsRawDataMessage) TrainPartDataMessage(ml.shifu.shifu.message.TrainPartDataMessage) NormPartRawDataMessage(ml.shifu.shifu.message.NormPartRawDataMessage) ScanEvalDataMessage(ml.shifu.shifu.message.ScanEvalDataMessage) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) List(java.util.List)

Example 2 with ScanTrainDataMessage

use of ml.shifu.shifu.message.ScanTrainDataMessage in project shifu by ShifuML.

the class TrainModelActor method onReceive.

/* (non-Javadoc)
     * @see akka.actor.UntypedActor#onReceive(java.lang.Object)
     */
@Override
public void onReceive(Object message) throws Exception {
    if (message instanceof AkkaActorInputMessage) {
        resultCnt = 0;
        AkkaActorInputMessage msg = (AkkaActorInputMessage) message;
        List<Scanner> scanners = msg.getScanners();
        log.debug("Num of Scanners: " + scanners.size());
        for (Scanner scanner : scanners) {
            dataLoadRef.tell(new ScanTrainDataMessage(scanners.size(), isDryRun, scanner), getSelf());
        }
    } else if (message instanceof TrainResultMessage) {
        resultCnt++;
        if (resultCnt == trainerCnt) {
            log.info("Received " + resultCnt + " finish message. Close System.");
            Encog.getInstance().shutdown();
            getContext().system().shutdown();
        }
    } else if (message instanceof ExceptionMessage) {
        // since some children actors meet some exception, shutdown the system
        ExceptionMessage msg = (ExceptionMessage) message;
        getContext().system().shutdown();
        // and wrapper the exception into Return status
        addExceptionIntoCondition(msg.getException());
    } else {
        unhandled(message);
    }
}
Also used : AkkaActorInputMessage(ml.shifu.shifu.message.AkkaActorInputMessage) Scanner(java.util.Scanner) ExceptionMessage(ml.shifu.shifu.message.ExceptionMessage) ScanTrainDataMessage(ml.shifu.shifu.message.ScanTrainDataMessage) TrainResultMessage(ml.shifu.shifu.message.TrainResultMessage)

Aggregations

Scanner (java.util.Scanner)2 ScanTrainDataMessage (ml.shifu.shifu.message.ScanTrainDataMessage)2 ArrayList (java.util.ArrayList)1 LinkedList (java.util.LinkedList)1 List (java.util.List)1 AkkaActorInputMessage (ml.shifu.shifu.message.AkkaActorInputMessage)1 ExceptionMessage (ml.shifu.shifu.message.ExceptionMessage)1 NormPartRawDataMessage (ml.shifu.shifu.message.NormPartRawDataMessage)1 ScanEvalDataMessage (ml.shifu.shifu.message.ScanEvalDataMessage)1 ScanNormInputDataMessage (ml.shifu.shifu.message.ScanNormInputDataMessage)1 ScanStatsRawDataMessage (ml.shifu.shifu.message.ScanStatsRawDataMessage)1 StatsPartRawDataMessage (ml.shifu.shifu.message.StatsPartRawDataMessage)1 TrainPartDataMessage (ml.shifu.shifu.message.TrainPartDataMessage)1 TrainResultMessage (ml.shifu.shifu.message.TrainResultMessage)1 MLDataPair (org.encog.ml.data.MLDataPair)1 BasicMLDataPair (org.encog.ml.data.basic.BasicMLDataPair)1