Search in sources :

Example 1 with StatsPartRawDataMessage

use of ml.shifu.shifu.message.StatsPartRawDataMessage in project shifu by ShifuML.

the class TrainDataPrepWorker method handleMsg.

/*
     * (non-Javadoc)
     * 
     * @see akka.actor.UntypedActor#onReceive(java.lang.Object)
     */
@Override
public void handleMsg(Object message) throws IOException {
    if (message instanceof TrainPartDataMessage) {
        log.debug("Received value object list for training model.");
        TrainPartDataMessage msg = (TrainPartDataMessage) message;
        for (MLDataPair mlDataPir : msg.getMlDataPairList()) {
            if (modelConfig.isTrainOnDisk() && !initialized) {
                int inputSize = mlDataPir.getInput().size();
                int idealSize = mlDataPir.getIdeal().size();
                ((BufferedMLDataSet) masterDataSet).beginLoad(inputSize, idealSize);
                initialized = true;
            }
            masterDataSet.add(mlDataPir);
        }
        receivedMsgCnt++;
        log.debug("Expected " + msg.getTotalMsgCnt() + " messages, received " + receivedMsgCnt + " message(s).");
        if (receivedMsgCnt == msg.getTotalMsgCnt()) {
            if (modelConfig.isTrainOnDisk() && initialized) {
                ((BufferedMLDataSet) masterDataSet).endLoad();
            }
            for (AbstractTrainer trainer : trainers) {
                // if the trainOnDisk is true, setting the "D" option
                if (modelConfig.isTrainOnDisk()) {
                    trainer.setTrainingOption("D");
                }
                trainer.setDataSet(masterDataSet);
                nextActorRef.tell(new TrainInstanceMessage(trainer), this.getSelf());
            }
            if (modelConfig.isTrainOnDisk() && initialized) {
                masterDataSet.close();
                masterDataSet = null;
            }
        }
    } else if (message instanceof StatsPartRawDataMessage) {
        StatsPartRawDataMessage msg = (StatsPartRawDataMessage) message;
        receivedMsgCnt++;
        log.debug("Expected " + msg.getTotalMsgCnt() + " messages, received " + receivedMsgCnt + " message(s).");
        if (receivedMsgCnt == msg.getTotalMsgCnt()) {
            for (AbstractTrainer trainer : trainers) {
                // ((DecisionTreeTrainer)trainer).setDataSet(rawInstanceList);
                nextActorRef.tell(new TrainInstanceMessage(trainer), this.getSelf());
            }
        }
    } else {
        unhandled(message);
    }
}
Also used : MLDataPair(org.encog.ml.data.MLDataPair) TrainInstanceMessage(ml.shifu.shifu.message.TrainInstanceMessage) StatsPartRawDataMessage(ml.shifu.shifu.message.StatsPartRawDataMessage) TrainPartDataMessage(ml.shifu.shifu.message.TrainPartDataMessage) AbstractTrainer(ml.shifu.shifu.core.AbstractTrainer) BufferedMLDataSet(org.encog.ml.data.buffer.BufferedMLDataSet)

Example 2 with StatsPartRawDataMessage

use of ml.shifu.shifu.message.StatsPartRawDataMessage in project shifu by ShifuML.

the class DataLoadWorker method handleMsg.

/*
     * (non-Javadoc)
     * 
     * @see akka.actor.UntypedActor#onReceive(java.lang.Object)
     */
@Override
public void handleMsg(Object message) {
    if (message instanceof ScanStatsRawDataMessage) {
        log.info("DataLoaderActor Starting ...");
        ScanStatsRawDataMessage msg = (ScanStatsRawDataMessage) message;
        Scanner scanner = msg.getScanner();
        int totalMsgCnt = msg.getTotalMsgCnt();
        List<String> rawDataList = readDataIntoList(scanner);
        log.info("DataLoaderActor Finished: Loaded " + rawDataList.size() + " Records.");
        nextActorRef.tell(new StatsPartRawDataMessage(totalMsgCnt, rawDataList), getSelf());
    } else if (message instanceof ScanNormInputDataMessage) {
        log.info("DataLoaderActor Starting ...");
        ScanNormInputDataMessage msg = (ScanNormInputDataMessage) message;
        Scanner scanner = msg.getScanner();
        int totalMsgCnt = msg.getTotalMsgCnt();
        List<String> rawDataList = readDataIntoList(scanner);
        log.info("DataLoaderActor Finished: Loaded " + rawDataList.size() + " Records.");
        nextActorRef.tell(new NormPartRawDataMessage(totalMsgCnt, rawDataList), getSelf());
    } else if (message instanceof ScanTrainDataMessage) {
        ScanTrainDataMessage msg = (ScanTrainDataMessage) message;
        Scanner scanner = msg.getScanner();
        int totalMsgCnt = msg.getTotalMsgCnt();
        List<MLDataPair> mlDataPairList = readTrainingData(scanner, msg.isDryRun());
        log.info("DataLoaderActor Finished: Loaded " + mlDataPairList.size() + " Records for Training.");
        nextActorRef.tell(new TrainPartDataMessage(totalMsgCnt, msg.isDryRun(), mlDataPairList), getSelf());
    } else if (message instanceof ScanEvalDataMessage) {
        log.info("DataLoaderActor Starting ...");
        ScanEvalDataMessage msg = (ScanEvalDataMessage) message;
        Scanner scanner = msg.getScanner();
        int streamId = msg.getStreamId();
        int totalStreamCnt = msg.getTotalStreamCnt();
        splitDataIntoMultiMessages(streamId, totalStreamCnt, scanner, Environment.getInt(Environment.RECORD_CNT_PER_MESSAGE, 100000));
    /*
             * List<String> evalDataList = readDataIntoList(scanner);
             * 
             * log.info("DataLoaderActor Finished: Loaded " + evalDataList.size() + " Records.");
             * nextActorRef.tell( new RunModelDataMessage(totalMsgCnt, evalDataList), getSelf());
             */
    } else {
        unhandled(message);
    }
}
Also used : BasicMLDataPair(org.encog.ml.data.basic.BasicMLDataPair) MLDataPair(org.encog.ml.data.MLDataPair) Scanner(java.util.Scanner) StatsPartRawDataMessage(ml.shifu.shifu.message.StatsPartRawDataMessage) ScanTrainDataMessage(ml.shifu.shifu.message.ScanTrainDataMessage) ScanNormInputDataMessage(ml.shifu.shifu.message.ScanNormInputDataMessage) ScanStatsRawDataMessage(ml.shifu.shifu.message.ScanStatsRawDataMessage) TrainPartDataMessage(ml.shifu.shifu.message.TrainPartDataMessage) NormPartRawDataMessage(ml.shifu.shifu.message.NormPartRawDataMessage) ScanEvalDataMessage(ml.shifu.shifu.message.ScanEvalDataMessage) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) List(java.util.List)

Example 3 with StatsPartRawDataMessage

use of ml.shifu.shifu.message.StatsPartRawDataMessage in project shifu by ShifuML.

the class DataPrepareWorker method handleMsg.

/*
     * (non-Javadoc)
     * 
     * @see akka.actor.UntypedActor#onReceive(java.lang.Object)
     */
@Override
public void handleMsg(Object message) {
    if (message instanceof StatsPartRawDataMessage) {
        StatsPartRawDataMessage partData = (StatsPartRawDataMessage) message;
        Map<Integer, List<ValueObject>> columnVoListMap = buildColumnVoListMap(partData.getRawDataList().size());
        DataPrepareStatsResult rt = convertRawDataIntoValueObject(partData.getRawDataList(), columnVoListMap);
        int totalMsgCnt = partData.getTotalMsgCnt();
        for (Map.Entry<Integer, List<ValueObject>> entry : columnVoListMap.entrySet()) {
            Integer columnNum = entry.getKey();
            log.info("send {} with {} value object", columnNum, entry.getValue().size());
            columnNumToActorMap.get(columnNum).tell(new StatsValueObjectMessage(totalMsgCnt, columnNum, entry.getValue(), rt.getMissingMap().containsKey(columnNum) ? rt.getMissingMap().get(columnNum) : 0, rt.getTotal()), getSelf());
        }
    } else if (message instanceof RunModelResultMessage) {
        RunModelResultMessage msg = (RunModelResultMessage) message;
        Map<Integer, List<ColumnScoreObject>> columnScoreListMap = buildColumnScoreListMap();
        convertModelResultIntoColScore(msg.getScoreResultList(), columnScoreListMap);
        int totalMsgCnt = msg.getTotalStreamCnt();
        for (Entry<Integer, List<ColumnScoreObject>> column : columnScoreListMap.entrySet()) {
            columnNumToActorMap.get(column.getKey()).tell(new ColumnScoreMessage(totalMsgCnt, column.getKey(), column.getValue()), getSelf());
        }
    } else {
        unhandled(message);
    }
}
Also used : ColumnScoreObject(ml.shifu.shifu.container.ColumnScoreObject) StatsPartRawDataMessage(ml.shifu.shifu.message.StatsPartRawDataMessage) StatsValueObjectMessage(ml.shifu.shifu.message.StatsValueObjectMessage) Entry(java.util.Map.Entry) ArrayList(java.util.ArrayList) List(java.util.List) RunModelResultMessage(ml.shifu.shifu.message.RunModelResultMessage) ColumnScoreMessage(ml.shifu.shifu.message.ColumnScoreMessage) HashMap(java.util.HashMap) Map(java.util.Map)

Example 4 with StatsPartRawDataMessage

use of ml.shifu.shifu.message.StatsPartRawDataMessage in project shifu by ShifuML.

the class DataFilterWorker method handleMsg.

/* (non-Javadoc)
     * @see ml.shifu.shifu.actor.worker.AbstractWorkerActor#handleMsg(java.lang.Object)

     */
@Override
public void handleMsg(Object message) throws Exception {
    if (message instanceof StatsPartRawDataMessage) {
        StatsPartRawDataMessage msg = (StatsPartRawDataMessage) message;
        purifyData(msg.getRawDataList());
        nextActorRef.tell(msg, getSelf());
    } else if (message instanceof NormPartRawDataMessage) {
        NormPartRawDataMessage msg = (NormPartRawDataMessage) message;
        purifyData(msg.getRawDataList());
        nextActorRef.tell(msg, getSelf());
    } else if (message instanceof RunModelDataMessage) {
        RunModelDataMessage msg = (RunModelDataMessage) message;
        purifyData(msg.getEvalDataList());
        nextActorRef.tell(msg, getSelf());
    } else {
        unhandled(message);
    }
}
Also used : StatsPartRawDataMessage(ml.shifu.shifu.message.StatsPartRawDataMessage) NormPartRawDataMessage(ml.shifu.shifu.message.NormPartRawDataMessage) RunModelDataMessage(ml.shifu.shifu.message.RunModelDataMessage)

Aggregations

StatsPartRawDataMessage (ml.shifu.shifu.message.StatsPartRawDataMessage)4 ArrayList (java.util.ArrayList)2 List (java.util.List)2 NormPartRawDataMessage (ml.shifu.shifu.message.NormPartRawDataMessage)2 TrainPartDataMessage (ml.shifu.shifu.message.TrainPartDataMessage)2 MLDataPair (org.encog.ml.data.MLDataPair)2 HashMap (java.util.HashMap)1 LinkedList (java.util.LinkedList)1 Map (java.util.Map)1 Entry (java.util.Map.Entry)1 Scanner (java.util.Scanner)1 ColumnScoreObject (ml.shifu.shifu.container.ColumnScoreObject)1 AbstractTrainer (ml.shifu.shifu.core.AbstractTrainer)1 ColumnScoreMessage (ml.shifu.shifu.message.ColumnScoreMessage)1 RunModelDataMessage (ml.shifu.shifu.message.RunModelDataMessage)1 RunModelResultMessage (ml.shifu.shifu.message.RunModelResultMessage)1 ScanEvalDataMessage (ml.shifu.shifu.message.ScanEvalDataMessage)1 ScanNormInputDataMessage (ml.shifu.shifu.message.ScanNormInputDataMessage)1 ScanStatsRawDataMessage (ml.shifu.shifu.message.ScanStatsRawDataMessage)1 ScanTrainDataMessage (ml.shifu.shifu.message.ScanTrainDataMessage)1