Examples with OptProblem org.vcell.optimization.thrift.OptProblem used on opensource projects

Example 1 with OptProblem

use of org.vcell.optimization.thrift.OptProblem in project vcell by virtualcell.

the class OptServerImplTest method main.

public static void main(String[] args) {
    try {
        System.setProperty(PropertyLoader.installationRoot, "/Users/schaff/Documents/workspace-modular/vcell");
        System.setProperty(PropertyLoader.pythonExe, "/Users/schaff/anaconda/bin/python");
        PythonSupport.verifyInstallation(new PythonPackage[] { PythonPackage.COPASI, PythonPackage.LIBSBML, PythonPackage.THRIFT });
        try {
            Thread.sleep(2000);
        } catch (InterruptedException e) {
        }
        File optProbFile = new File(ResourceUtil.getVCellOptPythonDir(), "optprob.bin");
        System.out.println("using optProblem: " + optProbFile.getAbsolutePath());
        OptServerImpl optServerImpl = new OptServerImpl();
        OptProblem optProblem = CopasiServicePython.readOptProblem(optProbFile);
        ArrayList<String> jobIDs = new ArrayList<String>();
        jobIDs.add(optServerImpl.submit(optProblem));
        jobIDs.add(optServerImpl.submit(optProblem));
        optServerImpl.start();
        jobIDs.add(optServerImpl.submit(optProblem));
        jobIDs.add(optServerImpl.submit(optProblem));
        boolean done = false;
        while (!done) {
            done = true;
            for (String jobID : jobIDs) {
                OptRunContext optRunContext = optServerImpl.getOptRunContextByOptimizationId(jobID);
                if (optRunContext == null) {
                    throw new RuntimeException("optRunContext was null for id = " + jobID);
                }
                OptRunStatus status = optRunContext.getStatus();
                if (status != OptRunStatus.Complete && status != OptRunStatus.Failed) {
                    done = false;
                }
                if (status == OptRunStatus.Complete) {
                    OptRun optRun = CopasiServicePython.readOptRun(optRunContext.getOptRunBinaryFile());
                    System.out.println("job " + jobID + ": status " + status + " " + optRun.getOptResultSet().toString());
                } else {
                    System.out.println("job " + jobID + ": status " + status);
                }
            }
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
            }
        }
        System.out.println("done with all jobs");
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 2 with OptProblem

use of org.vcell.optimization.thrift.OptProblem in project vcell by virtualcell.

the class OptimizationApiTest method readOptProblem.

public static OptProblem readOptProblem(File optProblemFile) throws IOException {
    TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory());
    try {
        OptProblem optProblem = new OptProblem();
        byte[] bytes = FileUtils.readFileToByteArray(optProblemFile);
        deserializer.deserialize(optProblem, bytes);
        return optProblem;
    } catch (TException e) {
        e.printStackTrace();
        throw new IOException("error reading optProblem from file " + optProblemFile.getPath() + ": " + e.getMessage(), e);
    }
}

Example 3 with OptProblem

use of org.vcell.optimization.thrift.OptProblem in project vcell by virtualcell.

the class OptimizationApiTest method main.

public static void main(String[] args) {
    try {
        boolean bIgnoreCertProblems = true;
        boolean bIgnoreHostMismatch = true;
        VCellApiClient apiClient = new VCellApiClient(host, port, bIgnoreCertProblems, bIgnoreHostMismatch);
        File optProbFile = new File("../pythonScripts/VCell_Opt/optprob.bin");
        System.out.println("using optProblem: " + optProbFile.getAbsolutePath());
        OptProblem optProblem = readOptProblem(optProbFile);
        TSerializer serializer = new TSerializer(new TJSONProtocol.Factory());
        String optProblemJson = serializer.toString(optProblem);
        ArrayList<String> jobIDs = new ArrayList<String>();
        jobIDs.add(apiClient.submitOptimization(optProblemJson));
        jobIDs.add(apiClient.submitOptimization(optProblemJson));
        jobIDs.add(apiClient.submitOptimization(optProblemJson));
        jobIDs.add(apiClient.submitOptimization(optProblemJson));
        boolean done = false;
        while (!done) {
            done = true;
            for (String jobID : jobIDs) {
                String optRunJson = apiClient.getOptRunJson(jobID);
                TDeserializer deserializer = new TDeserializer(new TJSONProtocol.Factory());
                OptRun optRun = new OptRun();
                deserializer.deserialize(optRun, optRunJson.getBytes());
                OptRunStatus status = optRun.status;
                if (status != OptRunStatus.Complete && status != OptRunStatus.Failed) {
                    done = false;
                }
                if (status == OptRunStatus.Complete) {
                    System.out.println("job " + jobID + ": status " + status + " " + optRun.getOptResultSet().toString());
                } else {
                    System.out.println("job " + jobID + ": status " + status);
                }
            }
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
            }
        }
        System.out.println("done with all jobs");
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 4 with OptProblem

use of org.vcell.optimization.thrift.OptProblem in project vcell by virtualcell.

the class OptimizationRunServerResource method doOP.

private Object doOP(Object input, ServerResource serverResource) throws ResourceException {
    synchronized (paramOptActiveSockets) {
        if (input instanceof String) {
            String optID = (String) input;
            if (paramOptResults.containsKey(optID)) {
                // return cached results, socket already closed
                return paramOptResults.remove(optID);
            }
            boolean bStop = Boolean.parseBoolean(serverResource.getQueryValue("bStop"));
            OptSocketStreams optSocketStreams = paramOptActiveSockets.get(optID);
            if (optSocketStreams != null) {
                try {
                    Object mesg = optSocketStreams.writeObject(optID, bStop);
                    if (mesg instanceof Boolean) {
                        // return status of stop command
                        optSocketStreams.closeAll(optID);
                        return new JsonRepresentation(mesg.toString());
                    } else if (mesg instanceof String) {
                        // return status of optID query
                        return new JsonRepresentation(mesg.toString());
                    } else if (mesg instanceof byte[]) {
                        // return of optResults when done
                        optSocketStreams.closeAll(optID);
                        String base64String = java.util.Base64.getEncoder().encodeToString((byte[]) mesg);
                        return new JsonRepresentation(new StringRepresentation(base64String));
                    }
                    throw new ResourceException(Status.SERVER_ERROR_INTERNAL, "Unexpected return message " + mesg.getClass().getName() + " for optimization ID=" + optID);
                } catch (Exception e) {
                    e.printStackTrace();
                    optSocketStreams.closeAll(optID);
                    throw new ResourceException(Status.SERVER_ERROR_INTERNAL, e.getMessage() + " optimization ID=" + optID);
                }
            } else {
                throw new ResourceException(Status.SERVER_ERROR_INTERNAL, "Can't find connection for optimization ID=" + optID);
            }
        } else if (input instanceof Representation) {
            Representation optProblemJson = (Representation) input;
            if (paramOptActiveSockets.size() >= 5) {
                String[] keys = paramOptActiveSockets.keySet().toArray(new String[0]);
                for (int i = 0; i < keys.length; i++) {
                    OptSocketStreams optSocketStreams = paramOptActiveSockets.get(keys[i]);
                    try {
                        // Check connection
                        Object obj = optSocketStreams.writeObject("checkIfDone", false);
                        if (obj instanceof Boolean && ((Boolean) obj)) {
                        // pending
                        // optSocketStreams.closeAll(optSocketStreams.optID);
                        } else if (obj instanceof byte[]) {
                            optSocketStreams.closeAll(optSocketStreams.optID);
                            String base64String = java.util.Base64.getEncoder().encodeToString((byte[]) obj);
                            paramOptResults.put(optSocketStreams.optID, new JsonRepresentation(new StringRepresentation(base64String)));
                            break;
                        } else if (obj instanceof String) {
                            if (obj.toString().startsWith(OptRunStatus.Failed.name())) {
                                throw new Exception(obj.toString());
                            }
                        }
                    } catch (Exception e) {
                        // ioError socket  closed
                        e.printStackTrace();
                        optSocketStreams.closeAll(optSocketStreams.optID);
                    }
                }
                if (paramOptActiveSockets.size() >= 5) {
                    throw new ResourceException(Status.SERVER_ERROR_INTERNAL, "Too many active optimization jobs, try again later");
                }
            }
            // User vcellUser = application.getVCellUser(getChallengeResponse(),AuthenticationPolicy.ignoreInvalidCredentials);
            if (optProblemJson != null && optProblemJson.getMediaType().isCompatible(MediaType.APPLICATION_JSON)) {
                try {
                    JsonRepresentation jsonRep = new JsonRepresentation(optProblemJson);
                    JSONObject json = jsonRep.getJsonObject();
                    TDeserializer deserializer = new TDeserializer(new TJSONProtocol.Factory());
                    OptProblem optProblem = new OptProblem();
                    deserializer.deserialize(optProblem, json.toString().getBytes());
                    // Server:		127.0.0.11
                    // justamq_api.1.16o695tgthpt@dockerbuild    | Address:	127.0.0.11#53
                    // justamq_api.1.16o695tgthpt@dockerbuild    |
                    // justamq_api.1.16o695tgthpt@dockerbuild    | Non-authoritative answer:
                    // justamq_api.1.16o695tgthpt@dockerbuild    | Name:	tasks.justamq_submit
                    // justamq_api.1.16o695tgthpt@dockerbuild    | Address: 10.0.7.14
                    // VCELL_SITE defined manually during deploy
                    // stackname = {"vcell"} + {$VCELL_SITE (from swarm *.config)}
                    // see vcell/docker/swarm/README.md "CLIENT and SERVER deploy commands" and "To create deploy configuration file"
                    // tasks.{taskName}, taskName comes from combining stackname + {taskname defined by docker}
                    // Container gets vcell.server.id from vcell:docker:swarm:deploy.sh and *.config variable VCELL_SITE
                    // see vcell/docker/swarm/deploy.sh -> echo "env \$(cat $remote_config_file | xargs) docker stack deploy -c $remote_compose_file $stack_name"
                    // lookup swarm ip number for task
                    String swarmSubmitTaskName = "tasks." + "vcell" + System.getProperty("vcell.server.id").toLowerCase() + "_submit";
                    ProcessBuilder pb = new ProcessBuilder("nslookup", swarmSubmitTaskName);
                    pb.redirectErrorStream(true);
                    Process process = pb.start();
                    java.io.InputStream is = process.getInputStream();
                    java.io.InputStreamReader isr = new java.io.InputStreamReader(is);
                    java.io.BufferedReader br = new java.io.BufferedReader(isr);
                    String line;
                    String ipnum = null;
                    boolean bFound = false;
                    while ((line = br.readLine()) != null) {
                        // System.out.println(line);
                        if (line.contains(swarmSubmitTaskName)) {
                            bFound = true;
                        } else if (bFound && line.trim().startsWith("Address:")) {
                            ipnum = line.trim().substring("Address:".length()).trim();
                            break;
                        }
                    }
                    br.close();
                    int errCode = process.waitFor();
                    System.out.println("----------nslookup errcode=" + errCode);
                    OptSocketStreams optSocketStreams = OptSocketStreams.create(ipnum);
                    optSocketStreams.optID = (String) optSocketStreams.writeObject(optProblem, false);
                    System.out.println("----------optimizationJobID=" + optSocketStreams.optID + " created socket connect to submit");
                    paramOptActiveSockets.put(optSocketStreams.optID, optSocketStreams);
                    return optSocketStreams;
                } catch (Exception e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                    throw new ResourceException(Status.SERVER_ERROR_INTERNAL, e.getMessage());
                }
            } else {
                throw new RuntimeException("unexpected post representation " + optProblemJson);
            }
        }
        throw new ResourceException(Status.SERVER_ERROR_INTERNAL, "Unexpected operation type=" + input.getClass().getName());
    }
}

Example 5 with OptProblem

use of org.vcell.optimization.thrift.OptProblem in project vcell by virtualcell.

the class HtcSimulationWorker method initOptimizationSocket.

private void initOptimizationSocket() {
    Thread optThread = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                optimizationServersocket = new ServerSocket(8877);
                while (true) {
                    Socket optSocket = optimizationServersocket.accept();
                    Thread thread = new Thread(new Runnable() {

                        @Override
                        public void run() {
                            OptServerJobInfo optServerJobInfo = null;
                            try (ObjectInputStream is = new ObjectInputStream(optSocket.getInputStream());
                                ObjectOutputStream oos = new ObjectOutputStream(optSocket.getOutputStream());
                                Socket myOptSocket = optSocket) {
                                long jobStart = 0;
                                while (true) {
                                    try {
                                        Object obj = is.readObject();
                                        Boolean bStop = (Boolean) is.readObject();
                                        if (bStop) {
                                            optServerStopJob(optServerJobInfo);
                                            oos.writeObject(new Boolean(true));
                                            return;
                                        }
                                        if (obj instanceof String && obj.toString().equals("checkIfDone")) {
                                            // check and remove connection and job status
                                            if (sendOptResults(optServerJobInfo.getOptID(), oos)) {
                                                return;
                                            }
                                            HtcJobStatus htcJobStatus = optServerGetJobStatus(optServerJobInfo.getHtcJobInfo());
                                            if (htcJobStatus == null) {
                                                // pending
                                                oos.writeObject(new Boolean(true));
                                            } else {
                                                if (htcJobStatus.isFailed()) {
                                                    throw new Exception("slurm job " + optServerJobInfo.getHtcJobInfo().getHtcJobID() + " failed");
                                                } else if (htcJobStatus.isComplete()) {
                                                    if (!sendOptResults(optServerJobInfo.optID, oos)) {
                                                        throw new Exception("job done but results missing");
                                                    } else {
                                                        return;
                                                    }
                                                } else {
                                                    // running
                                                    oos.writeObject(OptRunStatus.Running.name() + ":");
                                                }
                                            }
                                        } else if (obj instanceof OptProblem) {
                                            // Start opt job
                                            OptProblem optProblem = (OptProblem) obj;
                                            optServerJobInfo = submitOptProblem(optProblem);
                                            oos.writeObject(optServerJobInfo.getOptID());
                                            jobStart = System.currentTimeMillis();
                                        } else if (obj instanceof String) {
                                            // Get opt job status
                                            String optID = obj.toString();
                                            if (sendOptResults(optID, oos)) {
                                                return;
                                            }
                                            File f = generateOptInterresultsFilePath(optID);
                                            // make container read results status
                                            boolean bExist = hackFileExists(f);
                                            long lastModified = f.lastModified();
                                            if (lastModified == 0 && (System.currentTimeMillis() - jobStart) > 60000) {
                                                throw new Exception("results progress timed out");
                                            } else if (lastModified != 0 && (lastModified - jobStart) > 60000) {
                                                throw new Exception("results progress timed out");
                                            } else if (lastModified != 0) {
                                                jobStart = lastModified;
                                            }
                                            if (bExist) /*f.exists()*/
                                            {
                                                List<String> progressLines = Files.readAllLines(f.toPath());
                                                if (progressLines != null && progressLines.size() > 0) {
                                                    String optRunstatus = progressLines.get(progressLines.size() - 1);
                                                    if (optRunstatus.toLowerCase().trim().startsWith("except")) {
                                                        throw new Exception("python script error " + optRunstatus);
                                                    }
                                                    optRunstatus = OptRunStatus.Running.name() + ":" + optRunstatus;
                                                    oos.writeObject(optRunstatus);
                                                } else {
                                                    oos.writeObject(OptRunStatus.Queued.name() + ":0:0:0");
                                                }
                                            } else {
                                                oos.writeObject(OptRunStatus.Queued.name() + ":0:0:0");
                                            }
                                        } else {
                                            throw new Exception("Unexpected paramOpt command " + obj.getClass().getName());
                                        }
                                    } catch (Exception e) {
                                        oos.writeObject(OptRunStatus.Failed.name() + ":" + e.getMessage());
                                        throw e;
                                    }
                                }
                            } catch (Exception e) {
                                e.printStackTrace();
                            } finally {
                                // cleanup
                                if (optServerJobInfo != null && optServerJobInfo.getOptID() != null) {
                                    File optDir = generateOptimizeDirName(optServerJobInfo.getOptID());
                                    if (optDir.exists()) {
                                        generateOptProblemFilePath(optServerJobInfo.getOptID()).delete();
                                        generateOptOutputFilePath(optServerJobInfo.getOptID()).delete();
                                        generateOptInterresultsFilePath(optServerJobInfo.getOptID()).delete();
                                        optDir.delete();
                                    }
                                }
                            }
                        }
                    }, "paramOptProblem");
                    thread.setDaemon(true);
                    thread.start();
                }
            } catch (Exception e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
    });
    optThread.setDaemon(true);
    optThread.start();
}