Examples with LogisticRegressionTrainer ml.shifu.shifu.core.alg.LogisticRegressionTrainer used on opensource projects

Example 1 with LogisticRegressionTrainer

use of ml.shifu.shifu.core.alg.LogisticRegressionTrainer in project shifu by ShifuML.

the class TrainModelProcessor method runAkkaTrain.

/**
 * run training process with number of bags
 *
 * @param numBags
 *            number of bags, it decide how much trainer will start training
 * @throws IOException
 */
private void runAkkaTrain(int numBags) throws IOException {
    File models = new File("models");
    FileUtils.deleteDirectory(models);
    FileUtils.forceMkdir(models);
    trainers.clear();
    for (int i = 0; i < numBags; i++) {
        AbstractTrainer trainer;
        if (modelConfig.getAlgorithm().equalsIgnoreCase("NN")) {
            trainer = new NNTrainer(modelConfig, i, isDryTrain);
        } else if (modelConfig.getAlgorithm().equalsIgnoreCase("SVM")) {
            trainer = new SVMTrainer(this.modelConfig, i, isDryTrain);
        } else if (modelConfig.getAlgorithm().equalsIgnoreCase("LR")) {
            trainer = new LogisticRegressionTrainer(this.modelConfig, i, isDryTrain);
        } else {
            throw new ShifuException(ShifuErrorCode.ERROR_UNSUPPORT_ALG);
        }
        trainers.add(trainer);
    }
    List<Scanner> scanners = null;
    if (modelConfig.getAlgorithm().equalsIgnoreCase("DT")) {
        LOG.info("Raw Data: " + pathFinder.getNormalizedDataPath());
        try {
            scanners = ShifuFileUtils.getDataScanners(modelConfig.getDataSetRawPath(), modelConfig.getDataSet().getSource());
        } catch (IOException e) {
            throw new ShifuException(ShifuErrorCode.ERROR_INPUT_NOT_FOUND, e, pathFinder.getNormalizedDataPath());
        }
        if (CollectionUtils.isNotEmpty(scanners)) {
            AkkaSystemExecutor.getExecutor().submitDecisionTreeTrainJob(modelConfig, columnConfigList, scanners, trainers);
        }
    } else {
        LOG.info("Normalized Data: " + pathFinder.getNormalizedDataPath());
        try {
            scanners = ShifuFileUtils.getDataScanners(pathFinder.getNormalizedDataPath(), modelConfig.getDataSet().getSource());
        } catch (IOException e) {
            throw new ShifuException(ShifuErrorCode.ERROR_INPUT_NOT_FOUND, e, pathFinder.getNormalizedDataPath());
        }
        if (CollectionUtils.isNotEmpty(scanners)) {
            AkkaSystemExecutor.getExecutor().submitModelTrainJob(modelConfig, columnConfigList, scanners, trainers);
        }
    }
    // release
    closeScanners(scanners);
}

Example 2 with LogisticRegressionTrainer

use of ml.shifu.shifu.core.alg.LogisticRegressionTrainer in project shifu by ShifuML.

the class LogisticRegressionTest method setUp.

@BeforeClass
public void setUp() throws IOException {
    random = new Random();
    config = ModelConfig.createInitModelConfig("test", ALGORITHM.LR, "test", false);
    config.getVarSelect().setFilterNum(5);
    config.getTrain().setAlgorithm("LR");
    // config.
    config.getTrain().setNumTrainEpochs(100);
    config.getTrain().setParams(new HashMap<String, Object>());
    config.getTrain().getParams().put("LearningRate", 0.1);
    trainer = new LogisticRegressionTrainer(config, 0, false);
    trainSet = new BasicMLDataSet();
    for (int i = 0; i < 1000; i++) {
        double[] input = new double[5];
        double[] ideal = new double[1];
        for (int j = 0; j < 5; j++) {
            input[j] = random.nextDouble();
        }
        ideal[0] = random.nextInt(2);
        MLDataPair pair = new BasicMLDataPair(new BasicMLData(input), new BasicMLData(ideal));
        trainSet.add(pair);
    }
    trainer.setDataSet(trainSet);
    trainer.setValidSet(trainSet);
}

Example 3 with LogisticRegressionTrainer

use of ml.shifu.shifu.core.alg.LogisticRegressionTrainer in project shifu by ShifuML.

the class TrainModelActorTest method testActor.

@Test
public void testActor() throws IOException, InterruptedException {
    File tmpDir = new File("./tmp");
    FileUtils.forceMkdir(tmpDir);
    // create normalize data
    actorSystem = ActorSystem.create("shifuActorSystem");
    ActorRef normalizeRef = actorSystem.actorOf(new Props(new UntypedActorFactory() {

        private static final long serialVersionUID = 6777309320338075269L;

        public UntypedActor create() throws IOException {
            return new NormalizeDataActor(modelConfig, columnConfigList, new AkkaExecStatus(true));
        }
    }), "normalize-calculator");
    List<Scanner> scanners = ShifuFileUtils.getDataScanners("src/test/resources/example/cancer-judgement/DataStore/DataSet1", SourceType.LOCAL);
    normalizeRef.tell(new AkkaActorInputMessage(scanners), normalizeRef);
    while (!normalizeRef.isTerminated()) {
        Thread.sleep(5000);
    }
    File outputFile = new File("./tmp/NormalizedData");
    Assert.assertTrue(outputFile.exists());
    // start to run trainer
    actorSystem = ActorSystem.create("shifuActorSystem");
    File models = new File("./models");
    FileUtils.forceMkdir(models);
    final List<AbstractTrainer> trainers = new ArrayList<AbstractTrainer>();
    for (int i = 0; i < 5; i++) {
        AbstractTrainer trainer;
        if (modelConfig.getAlgorithm().equalsIgnoreCase("NN")) {
            trainer = new NNTrainer(this.modelConfig, i, false);
        } else if (modelConfig.getAlgorithm().equalsIgnoreCase("SVM")) {
            trainer = new SVMTrainer(this.modelConfig, i, false);
        } else if (modelConfig.getAlgorithm().equalsIgnoreCase("LR")) {
            trainer = new LogisticRegressionTrainer(this.modelConfig, i, false);
        } else {
            throw new RuntimeException("unsupport algorithm");
        }
        trainers.add(trainer);
    }
    // train model
    ActorRef modelTrainRef = actorSystem.actorOf(new Props(new UntypedActorFactory() {

        private static final long serialVersionUID = 6777309320338075269L;

        public UntypedActor create() throws IOException {
            return new TrainModelActor(modelConfig, columnConfigList, new AkkaExecStatus(true), trainers);
        }
    }), "trainer");
    scanners = ShifuFileUtils.getDataScanners("./tmp/NormalizedData", SourceType.LOCAL);
    modelTrainRef.tell(new AkkaActorInputMessage(scanners), modelTrainRef);
    while (!modelTrainRef.isTerminated()) {
        Thread.sleep(5000);
    }
    for (Scanner scanner : scanners) {
        scanner.close();
    }
    File model0 = new File("./models/model0.nn");
    File model1 = new File("./models/model0.nn");
    File model2 = new File("./models/model0.nn");
    File model3 = new File("./models/model0.nn");
    File model4 = new File("./models/model0.nn");
    Assert.assertTrue(model0.exists());
    Assert.assertTrue(model1.exists());
    Assert.assertTrue(model2.exists());
    Assert.assertTrue(model3.exists());
    Assert.assertTrue(model4.exists());
    File modelsTemp = new File("./modelsTmp");
    FileUtils.deleteDirectory(modelsTemp);
    FileUtils.deleteDirectory(models);
    FileUtils.deleteDirectory(tmpDir);
}