Examples with AmplistorCredential cbit.vcell.util.AmplistorUtils.AmplistorCredential used on opensource projects

Example 1 with AmplistorCredential

use of cbit.vcell.util.AmplistorUtils.AmplistorCredential in project vcell by virtualcell.

the class TestMissingSimData method checkDataExists.

// private static void runSimsNew(String connectURL,String dbSchemaUser, String dbPassword) throws Exception{
// 
// //		VCellBootstrap vCellBootstrap = getVCellBootstrap("rmi-beta.cam.uchc.edu", 40105, "VCellBootstrapServer", 12, false);
// //		VCellBootstrap vCellBootstrap = getVCellBootstrap("rmi-alpha.cam.uchc.edu", 40106, "VCellBootstrapServer", 12, false);
// VCellBootstrap vCellBootstrap = getVCellBootstrap("rmi-alpha.cam.uchc.edu", 40111, "VCellBootstrapServer", 12, false);//Test2
// 
// if(true){
// Hashtable<KeyValue, UserLoginInfo> keyUserLoginInfo = doQuery(connectURL, dbSchemaUser, dbPassword);
// Enumeration<KeyValue> keys = keyUserLoginInfo.keys();
// while(keys.hasMoreElements()){
// KeyValue simKey = keys.nextElement();
// UserLoginInfo userLoginInfo = keyUserLoginInfo.get(simKey);
// VCSimulationIdentifier vcSimulationIdentifier = new VCSimulationIdentifier(simKey, userLoginInfo.getUser());
// acquireThread(vcSimulationIdentifier, vCellBootstrap, userLoginInfo, connectURL, dbSchemaUser, dbPassword);
// }
// return;
// }
// 
// 
// String itemSelectSQL = " select vc_userinfo.userid,vc_userinfo.id userkey,vc_userinfo.digestpw,missingdata.simjobsimref,vc_softwareversion.softwareversion ";
// 
// String sqlPart =
// " from missingdata,vc_simulation,vc_userinfo,vc_softwareversion "+
// " where "+
// " (vc_simulation.id in (select simref from vc_biomodelsim)) and " +
// //				" (vc_simulation.id in (select simref from vc_mathmodelsim)) and " +
// " vc_userinfo.userid='fgao5' and "+
// " vc_userinfo.id = vc_simulation.ownerref and "+
// " missingdata.simjobsimref = vc_simulation.id and "+
// //				" missingdata.dataexists not like 'readable%' and "+
// " (missingdata.dataexists = 'false') "+
// " and missingdata.notes is not null and " +
// " (missingdata.notes like '%Compiled_solvers_no_longer%' or missingdata.notes like '%Connection_refused%')" +
// //				" and missingdata.simjobsimref = 34080002 " +
// //				" or missingdata.dataexists like 'error - %') "+
// //				" and (missingdata.notes is null ) "+
// //				" or missingdata.notes not like 'reran OK%')" +
// " and vc_simulation.parentsimref is null and "+
// " (softwareversion is null or regexp_substr(softwareversion,'^((release)|(rel)|(alpha)|(beta))_version_([[:digit:]]+\\.?)+_build_([[:digit:]]+\\.?)+',1,1,'i') is not null) and "+
// " vc_softwareversion.versionableref (+) = vc_simulation.id " +
// //				" and rownum = 1 ";
// //				" and vc_simulation.id=39116536"
// " order by vc_userinfo.userid";
// 
// //		(mdt.dataexists = 'false' or mdt.dataexists like 'error - %') and
// //		(mdt.notes is null or mdt.notes not like 'reran OK%') and
// 
// //Create hash of sims and userlogininfo
// Hashtable<KeyValue, UserLoginInfo> simToUserLoginInfoHash = new Hashtable<>();
// Statement updateStatement = con.createStatement();
// Statement queryStatement = con.createStatement();
// //Get Count
// ResultSet rset = queryStatement.executeQuery("select count(*) "+sqlPart);
// rset.next();
// int totalCount = rset.getInt(1);
// rset.close();
// //Get sims
// rset = queryStatement.executeQuery(itemSelectSQL+sqlPart);
// UserLoginInfo userLoginInfo = null;
// VCellConnection vcellConnection = null;
// int currentCount = 1;
// while(rset.next()){
// KeyValue simJobSimRef = new KeyValue(rset.getString("simjobsimref"));
// try{
// String softwareVersion = rset.getString("softwareversion");
// String userid = rset.getString("userid");
// System.out.println("-----");
// System.out.println("-----running "+currentCount+" of "+totalCount+"   user="+userid+" simjobsimref="+simJobSimRef);
// currentCount+= 1;
// System.out.println("-----");
// 
// if(!rset.wasNull() && softwareVersion != null){
// StringTokenizer st = new StringTokenizer(softwareVersion, "_");
// st.nextToken();//site name
// st.nextToken();//'Version' literal string
// String majorVersion = st.nextToken();//major version number
// if(majorVersion.equals("5.4")){
// throw new Exception("Alpha-5.4 sims are not being re-run using Beta-5.3 code");
// }
// }
// 
// if(userid.toLowerCase().equals("vcelltestaccount")
// //					 || userid.toLowerCase().equals("anu")
// //					 || userid.toLowerCase().equals("fgao")
// //					 || userid.toLowerCase().equals("liye")
// //					 || userid.toLowerCase().equals("schaff")
// //					 || userid.toLowerCase().equals("ignovak")
// //					 || userid.toLowerCase().equals("jditlev")
// //					 || userid.toLowerCase().equals("sensation")
// ){
// continue;
// }
// String userkey = rset.getString("userkey");
// if(userLoginInfo == null || !userLoginInfo.getUserName().equals(userid)){
// userLoginInfo = new UserLoginInfo(userid,DigestedPassword.createAlreadyDigested(rset.getString("digestpw")));
// userLoginInfo.setUser(new User(userid, new KeyValue(userkey)));
// vcellConnection = null;
// }
// VCSimulationIdentifier vcSimulationIdentifier = new VCSimulationIdentifier(simJobSimRef, userLoginInfo.getUser());
// 
// 
// //
// //
// //
// if(true){
// acquireThread(vcSimulationIdentifier, vCellBootstrap, userLoginInfo, connectURL, dbSchemaUser, dbPassword);
// continue;
// }
// 
// try{
// if(vcellConnection != null){
// vcellConnection.getMessageEvents();
// }
// }catch(Exception e){
// e.printStackTrace();
// //assume disconnected
// vcellConnection = null;
// }
// if(vcellConnection == null){
// vcellConnection = vCellBootstrap.getVCellConnection(userLoginInfo);
// vcellConnection.getMessageEvents();
// }
// 
// SimulationStatusPersistent initSimulationStatus = vcellConnection.getUserMetaDbServer().getSimulationStatus(vcSimulationIdentifier.getSimulationKey());
// 
// System.out.println("initial status="+initSimulationStatus);
// //				if(!initSimulationStatus.isCompleted() || !initSimulationStatus.getHasData()){
// //					continue;
// //				}
// 
// 
// BigString simXML = vcellConnection.getUserMetaDbServer().getSimulationXML(simJobSimRef);
// Simulation sim = XmlHelper.XMLToSim(simXML.toString());
// //				SolverDescription solverDescription = sim.getSolverTaskDescription().getSolverDescription();
// //				if(solverDescription.equals(SolverDescription.StochGibson)  || solverDescription.equals(SolverDescription.FiniteVolume)){
// //					//These 2 solvers give too much trouble so skip
// //					System.out.println("--skipping solver");
// ////					notCompletedSimIDs.add(simIDAndJobID.simID.toString());
// //					return;
// //				}
// 
// 
// //				if(!sim.isSpatial()){
// //					continue;
// //				}
// //				if(sim.getSolverTaskDescription().isSerialParameterScan()/* || sim.getSolverTaskDescription().getExpectedNumTimePoints() > 20*/){
// //					continue;
// //				}
// 
// int scanCount = sim.getScanCount();
// //				if(true){return;}
// SimulationStatusPersistent simulationStatus = null;
// SimulationInfo simulationInfo = sim.getSimulationInfo();
// if(!simulationInfo.getAuthoritativeVCSimulationIdentifier().getSimulationKey().equals(vcSimulationIdentifier.getSimulationKey())){
// throw new Exception("Unexpected authoritative and sim id are not the same");
// }
// vcellConnection.getSimulationController().startSimulation(vcSimulationIdentifier, scanCount);
// long startTime = System.currentTimeMillis();
// while(simulationStatus == null || simulationStatus.isStopped() || simulationStatus.isCompleted() || simulationStatus.isFailed()){
// Thread.sleep(2000);
// simulationStatus = vcellConnection.getUserMetaDbServer().getSimulationStatus(vcSimulationIdentifier.getSimulationKey());
// if(simulationStatus.isFailed() && !initSimulationStatus.isFailed()){
// break;
// }
// MessageEvent[] messageEvents = vcellConnection.getMessageEvents();
// if((System.currentTimeMillis()-startTime) > 30000){
// throw new Exception("-----Sim finished too fast or took too long to start, status= "+simulationStatus);
// }
// System.out.println(simulationStatus);
// }
// SimulationStatusPersistent lastSimStatus = simulationStatus;
// while(!simulationStatus.isStopped() && !simulationStatus.isCompleted() && !simulationStatus.isFailed()){
// for(int i = 0;i<3;i++){
// MessageEvent[] messageEvents = vcellConnection.getMessageEvents();
// Thread.sleep(1000);
// }
// simulationStatus = vcellConnection.getUserMetaDbServer().getSimulationStatus(vcSimulationIdentifier.getSimulationKey());
// if(!simulationStatus.toString().equals(lastSimStatus.toString())){
// lastSimStatus = simulationStatus;
// System.out.println("running status="+simulationStatus);
// }
// //					MessageEvent[] messageEvents = vcellConnection.getMessageEvents();
// //					for (int i = 0; messageEvents != null && i < messageEvents.length; i++) {
// //						System.out.println(messageEvents[i]);
// //					}
// }
// 
// if(!lastSimStatus.isCompleted()){
// throw new Exception("Unexpected run status: "+lastSimStatus.toString());
// }
// 
// 
// String updatestr = "update missingdata set notes='"+DB_NOTES_SIM_RUNOK_CODE+"' where simjobsimref="+simJobSimRef;
// System.out.println(updatestr);
// updateStatement.executeUpdate(updatestr);
// con.commit();
// }catch(Exception e){
// e.printStackTrace();
// String errString = TokenMangler.fixTokenStrict(DB_NOTES_SIM_ERROR_CODE+e.getClass().getSimpleName()+" "+e.getMessage());
// if(errString.length() > 256){
// errString = errString.substring(0, 256);
// }
// String updatestr = "update missingdata set notes='"+errString+"' where simjobsimref="+simJobSimRef;
// System.out.println(updatestr);
// updateStatement.executeUpdate(updatestr);
// con.commit();
// }
// }
// 
// }
private static void checkDataExists(Connection con, boolean bExistOnly) throws SQLException {
    AmplistorCredential amplistorCredential = AmplistorUtilsTest.getAmplistorCredential();
    Statement queryStatement = con.createStatement();
    Statement updateStatement = con.createStatement();
    // Hashtable<KeyValue, Exception> errorHash = new Hashtable<>();
    String sql = "select missingdata.*,parentsimref " + " from missingdata,vc_simulation,vc_userinfo" + " where vc_simulation.id = missingdata.simjobsimref and" + " vc_simulation.parentsimref is null and " + " vc_simulation.ownerref=vc_userinfo.id and " + " missingdata.notes is not null and " + " (missingdata.notes like '%exceeded_maximum%' " + " or missingdata.notes ='recheck dataexists' )" + // " vc_userinfo.userid='schaff' "+
    " order by missingdata.userid";
    ResultSet rset = queryStatement.executeQuery(sql);
    while (rset.next()) {
        // if(!rset.getString("dataexists").equals("tbd")){
        // continue;
        // }
        KeyValue simJobSimRef = null;
        KeyValue parentsimref = null;
        User user = null;
        try {
            simJobSimRef = new KeyValue(rset.getString("simjobsimref"));
            parentsimref = (rset.getString("parentsimref") == null ? null : new KeyValue(rset.getString("parentsimref")));
            user = new User(rset.getString("userid"), new KeyValue(rset.getString("userinfoid")));
            int jobIndex = rset.getInt("jobindex");
            File primaryDataDir = new File("\\\\cfs02\\ifs\\raid\\vcell\\users\\" + user.getName());
            String updatestr = null;
            File filePathNamePrime = new File(primaryDataDir, SimulationData.createCanonicalSimLogFileName(simJobSimRef, 0, false));
            if (filePathNamePrime.exists()) {
                updatestr = "fileNewPrime";
            } else if (new File(primaryDataDir, SimulationData.createCanonicalSimLogFileName(simJobSimRef, 0, true)).exists()) {
                updatestr = "fileOldPrime";
            } else if (parentsimref != null && new File(primaryDataDir, SimulationData.createCanonicalSimLogFileName(parentsimref, 0, false)).exists()) {
                updatestr = "fileNewParent";
            } else if (parentsimref != null && new File(primaryDataDir, SimulationData.createCanonicalSimLogFileName(parentsimref, 0, true)).exists()) {
                updatestr = "fileOldParent";
            } else if (AmplistorUtils.bFileExists(new URL(AmplistorUtils.DEFAULT_AMPLI_SERVICE_VCELL_URL + user.getName() + "/" + filePathNamePrime.getName()), amplistorCredential)) {
                updatestr = "ampliNewPrime";
            } else if (AmplistorUtils.bFileExists(new URL(AmplistorUtils.DEFAULT_AMPLI_SERVICE_VCELL_URL + user.getName() + "/" + SimulationData.createCanonicalSimLogFileName(simJobSimRef, 0, true)), amplistorCredential)) {
                updatestr = "ampliOldPrime";
            } else if (parentsimref != null && AmplistorUtils.bFileExists(new URL(AmplistorUtils.DEFAULT_AMPLI_SERVICE_VCELL_URL + user.getName() + "/" + SimulationData.createCanonicalSimLogFileName(parentsimref, 0, false)), amplistorCredential)) {
                updatestr = "ampliNewParent";
            } else if (parentsimref != null && AmplistorUtils.bFileExists(new URL(AmplistorUtils.DEFAULT_AMPLI_SERVICE_VCELL_URL + user.getName() + "/" + SimulationData.createCanonicalSimLogFileName(parentsimref, 0, true)), amplistorCredential)) {
                updatestr = "ampliOldParent";
            } else {
                updatestr = "false";
            }
            if (bExistOnly || updatestr.equals("false")) {
                updateStatement.executeUpdate("update missingdata set dataexists='" + updatestr + "' where simjobsimref=" + simJobSimRef.toString());
                con.commit();
                continue;
            }
            // Log file exists, now check if the data can really be read
            VCSimulationIdentifier vcSimulationIdentifier = new VCSimulationIdentifier(simJobSimRef, user);
            VCSimulationDataIdentifier vcSimulationDataIdentifier = new VCSimulationDataIdentifier(vcSimulationIdentifier, jobIndex);
            // Try to read log,times and simdata to see if this data is well formed
            SimDataAmplistorInfo simDataAmplistorInfo = AmplistorUtils.getSimDataAmplistorInfoFromPropertyLoader();
            SimulationData simData = new SimulationData(vcSimulationDataIdentifier, primaryDataDir, null, simDataAmplistorInfo);
            double[] dataTimes = simData.getDataTimes();
            DataIdentifier[] dataIdentifiers = simData.getVarAndFunctionDataIdentifiers(null);
            DataIdentifier readDataIdentifier = null;
            for (DataIdentifier dataIdentifier : dataIdentifiers) {
                if (!dataIdentifier.isFunction()) {
                    if (simData.getIsODEData()) {
                        ODEDataBlock odeDataBlock = simData.getODEDataBlock();
                        odeDataBlock.getODESimData().getRow(dataTimes.length - 1);
                    } else {
                        simData.getSimDataBlock(null, dataIdentifier.getName(), dataTimes[dataTimes.length - 1]);
                    }
                    readDataIdentifier = dataIdentifier;
                    break;
                }
            }
            System.out.println(BeanUtils.forceStringSize("user= " + user.getName(), 20, " ", false) + " simref= " + BeanUtils.forceStringSize(simJobSimRef.toString(), 14, " ", false) + " numTimes= " + BeanUtils.forceStringSize(dataTimes.length + "", 8, " ", true) + " readDataID= " + BeanUtils.forceStringSize(readDataIdentifier.getName() + "", 20, " ", true));
            updatestr = "update missingdata set dataexists='readable' where simjobsimref=" + simJobSimRef.toString();
        // String updatestr = "update missingdata set dataexists='true' where"+
        // " userinfoid="+user.getID().toString()+
        // " and simjobsimref="+simJobSimRef.toString()+
        // " and maxtaskid="+rset.getString("maxtaskid")+
        // " and jobindex="+jobIndex;
        // updateStatement.executeUpdate(updatestr);
        // con.commit();
        } catch (Exception e) {
            if (simJobSimRef == null) {
                e.printStackTrace();
            // throw new SQLException("Error querying",e);
            } else {
                String errString = e.getClass().getSimpleName() + " " + e.getMessage();
                if (errString.length() > 512) {
                    errString = errString.substring(0, 512);
                }
                String updatestr = "update missingdata set dataexists='error - " + TokenMangler.fixTokenStrict(errString) + "' where simjobsimref=" + simJobSimRef.toString();
                // updateStatement.executeUpdate(updatestr);
                // con.commit();
                System.out.println(BeanUtils.forceStringSize("user= " + (user == null ? "unavailable" : user.getName()), 20, " ", false) + " simref= " + BeanUtils.forceStringSize(simJobSimRef.toString(), 14, " ", false) + " parentsimref= " + BeanUtils.forceStringSize((parentsimref == null ? "NULL" : parentsimref.toString()), 14, " ", false) + " failed= " + e.getMessage());
            // errorHash.put(simJobSimRef,e);
            }
        }
    }
    rset.close();
// return errorHash;
}

Example 2 with AmplistorCredential

use of cbit.vcell.util.AmplistorUtils.AmplistorCredential in project vcell by virtualcell.

the class UnarchiveAmplistor method main.

public static void main(String[] args) {
    try {
        // can set directly here, or
        String amplistorCredentialCleartextPassword = null;
        if (amplistorCredentialCleartextPassword == null) {
            amplistorCredentialCleartextPassword = readFromHomeDirectory();
        }
        if (amplistorCredentialCleartextPassword == null) {
            System.err.println("need password");
            return;
        }
        // /share/apps/vcell2/users/boris/SimID_87497385_0__0.simtask.xml
        // /share/apps/vcell2/users/boris/SimID_87497385_0_.fvinput
        // /share/apps/vcell2/users/boris/SimID_87497385_0_.smoldynInput
        // /share/apps/vcell2/users/boris/SimID_87497385_0_.smoldynOutput
        // /share/apps/vcell2/users/boris/SimID_87497385_0_.tid
        // /share/apps/vcell2/users/boris/SimID_87497385_0_.vcg
        String amplistorCredentialUser = "vcell";
        AmplistorCredential amplistorCredential = new AmplistorCredential(amplistorCredentialUser, amplistorCredentialCleartextPassword);
        for (String fileName : USER_FILE_NAMES) {
            // not sure where isolon is anymore
            File destinationFile = new File("\\\\cfs02\\raid\\vcell\\" + USER + '\\' + fileName);
            AmplistorUtils.getObjectDataPutInFile(AmplistorUtils.DEFAULT_AMPLI_SERVICE_VCELL_URL + '/' + USER + '/' + fileName, amplistorCredential, destinationFile);
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 3 with AmplistorCredential

use of cbit.vcell.util.AmplistorUtils.AmplistorCredential in project vcell by virtualcell.

the class AmplistorUtilsTest method getAmplistorCredential.

public static AmplistorCredential getAmplistorCredential() {
    Scanner scanner = null;
    String password = null;
    try {
        if (System.getProperty(AMPLI_REST_TEST_PASSWORD_KEY) == null) {
            scanner = new Scanner(System.in);
            System.out.println("Enter 'camadmin' password for Amplistor (then press enter):");
            password = scanner.nextLine();
        } else {
            password = System.getProperty(AMPLI_REST_TEST_PASSWORD_KEY);
        }
    } finally {
        if (scanner != null) {
            scanner.close();
        }
    }
    return new AmplistorCredential(System.getProperty(AMPLI_REST_TEST_USER_KEY, "camadmin"), password);
}

Example 4 with AmplistorCredential

use of cbit.vcell.util.AmplistorUtils.AmplistorCredential in project vcell by virtualcell.

the class AmplistorUtilsTest method test.

@Test
public void test() throws Exception {
    // 
    // Create amplistor full access credential
    // 
    AmplistorCredential amplistorCredential = getAmplistorCredential();
    // must set this property in eclipse debug configuration
    if (amplistorCredential.userName == null || amplistorCredential.password == null) {
        throw new Exception("Amplistor full access credential required for test");
    }
    // 
    // Define Amplistor test dir
    // 
    final String AMPLI_REST_TEST_DIR = "Ampli_REST_Test_Dir";
    String dirNameURL = AmplistorUtils.DEFAULT_AMPLI_SERVICE_VCELL_URL + AMPLI_REST_TEST_DIR;
    // 
    // Define Local test dir and file
    // 
    Random rand = new Random();
    byte[] rndBytes = new byte[10000];
    rand.nextBytes(rndBytes);
    File tmpDir = Files.createTempDirectory("rnd_").toFile();
    File[] tmpFiles = new File[NUMFILES];
    final String SIMID_PREFIX = "SimID_";
    for (int i = 0; i < NUMFILES; i++) {
        tmpFiles[i] = File.createTempFile(SIMID_PREFIX + i + "_0_", ".rndbin", tmpDir);
        FileOutputStream fos = new FileOutputStream(tmpFiles[i]);
        BufferedOutputStream bos = new BufferedOutputStream(fos);
        bos.write(rndBytes);
        bos.close();
    }
    // 
    try {
        deleteDirAndAllFiles(dirNameURL, amplistorCredential);
    } catch (FileNotFoundException e) {
    // this is OK, ignore
    }
    // 
    // Create test dir on amplistor
    // 
    AmplistorUtils.createDir(dirNameURL, amplistorCredential);
    // 
    // Upload rnd file
    // 
    Hashtable<File, Exception> failures = AmplistorUtils.uploadFilesOperation(tmpDir, new URL(dirNameURL), amplistorCredential);
    if (failures != null) {
        throw new Exception("Upload failed " + failures.values().iterator().next().getMessage());
    }
    // 
    // List amplistor test dir we created and populated
    // 
    ArrayList<String> fileNames = AmplistorUtils.listDir(dirNameURL, null, amplistorCredential);
    if (fileNames.size() != NUMFILES) {
        throw new Exception("Expected " + NUMFILES + " files but got " + fileNames.size());
    }
    for (String fileName : fileNames) {
        System.out.println("listing:" + fileName);
    }
    String file0url = dirNameURL + "/" + fileNames.get(0);
    // 
    if (!AmplistorUtils.bFileExists(new URL(file0url), amplistorCredential)) {
        throw new Exception("Expected " + file0url + " 'bFileExist' to be true");
    }
    // 
    // check filter returns correct results
    // 
    AmplistorUtils.AmplistorFileNameMatcher onlyTheseMatchingFiles = new AmplistorUtils.AmplistorFileNameMatcher() {

        @Override
        public boolean accept(String fileName) {
            return fileName.equals(tmpFiles[0].getName());
        }
    };
    ArrayList<String> filteredNames = AmplistorUtils.listDir(dirNameURL, onlyTheseMatchingFiles, amplistorCredential);
    if (filteredNames.size() != 1 || !filteredNames.get(0).equals(tmpFiles[0].getName())) {
        throw new Exception("Filter test failed, expecting 1==" + filteredNames.size() + " and " + tmpFiles[0].getName() + "==" + (filteredNames.size() == 0 ? "null" : filteredNames.get(0)));
    }
    // 
    // set metadata on first file (this is done during upload using lastmodified of file but do it again to demonstrate)
    // 
    Calendar calendar = Calendar.getInstance();
    calendar.set(2010, 2, 3, 5, 15, 30);
    calendar.set(Calendar.MILLISECOND, 0);
    System.out.println("Set Custom Modification time----- " + calendar.getTime());
    // AmplistorUtils.setFileMetaData(dirNameURL+"/"+tmpFiles[0].getName(), amplistorCredential, SimulationData.AmplistorHelper.CUSTOM_FILE_MODIFICATION_DATE, tmpFiles[0].lastModified()/1000+".0");
    AmplistorUtils.setFileMetaData(file0url, amplistorCredential, AmplistorUtils.CUSTOM_FILE_MODIFICATION_DATE, calendar.getTime().getTime() / 1000 + ".0");
    // 
    // Print the http header info for one of the uploaded files
    // 
    AmplistorUtils.AmpliCustomHeaderHelper ampliCustomHeaderHelper = AmplistorUtils.printHeaderFields(file0url, amplistorCredential);
    if (ampliCustomHeaderHelper.customModification == null || !ampliCustomHeaderHelper.customModification.equals(calendar.getTime())) {
        throw new Exception("Queried custom modification date " + ampliCustomHeaderHelper.customModification + " does not match set custom modification date " + calendar.getTime());
    }
    // 
    // Download 1 of the files
    // 
    byte[] downloadBytes = AmplistorUtils.getObjectData(file0url, amplistorCredential);
    // 
    if (downloadBytes.length != tmpFiles[0].length() || rndBytes.length != downloadBytes.length) {
        throw new Exception("round trip file sizes are different local = " + tmpFiles[0].length() + " remote = " + downloadBytes.length + " memory = " + rndBytes.length);
    }
    FileInputStream fis = new FileInputStream(tmpFiles[0]);
    BufferedInputStream bis = new BufferedInputStream(fis);
    DataInputStream dis = new DataInputStream(bis);
    byte[] localBytes = new byte[rndBytes.length];
    dis.readFully(localBytes);
    dis.close();
    if (!Arrays.equals(localBytes, downloadBytes) || !Arrays.equals(rndBytes, downloadBytes)) {
        throw new Exception("download bytes not match local bytes after round trip");
    }
    // 
    // Check special SimID parsing for delete
    // 
    HashSet<KeyValue> doNotDeleteTheseSimKeys = new HashSet<KeyValue>();
    doNotDeleteTheseSimKeys.add(new KeyValue("0"));
    doNotDeleteTheseSimKeys.add(new KeyValue("1"));
    long numDeleted = AmplistorUtils.deleteSimFilesNotInHash(dirNameURL, doNotDeleteTheseSimKeys, false, amplistorCredential).size();
    if (numDeleted != (NUMFILES - doNotDeleteTheseSimKeys.size())) {
        throw new Exception("Expected to delete " + (NUMFILES - doNotDeleteTheseSimKeys.size()) + " but deleted " + numDeleted);
    }
    // 
    // remove one of the remaining files on amplistor
    // 
    AmplistorUtils.deleteFilesOperation(new String[] { tmpFiles[0].getName() }, dirNameURL, amplistorCredential);
    // 
    if (AmplistorUtils.bFileExists(new URL(dirNameURL + "/" + tmpFiles[0].getName()), amplistorCredential)) {
        throw new Exception("Expected " + dirNameURL + "/" + tmpFiles[0].getName() + " 'bFileExist' to be false");
    }
    // 
    try {
        AmplistorUtils.deleteFilesOperation(new String[] { "blahblah" }, dirNameURL, amplistorCredential);
        throw new Exception("We shouldn't have gotten here, Expecting FileNotFoundException");
    } catch (FileNotFoundException e) {
    // ignore, this should happen
    }
    // 
    // there should be 1 file left
    // 
    int numleft = AmplistorUtils.listDir(dirNameURL, null, amplistorCredential).size();
    if (numleft != 1) {
        throw new Exception("Expecting 1 file in list but got " + numleft);
    }
    // 
    // Remove test from amplistor
    // 
    // do not ignore FileNotFound
    deleteDirAndAllFiles(dirNameURL, amplistorCredential);
    // 
    // vcell_logs Amplistor test
    // 
    // 
    // try to upload to vcell_logs without credentials
    // 
    AmplistorUtils.uploadFile(new URL(AmplistorUtils.DEFAULT_AMPLI_VCELL_LOGS_URL), tmpFiles[0], null);
    // 
    try {
        AmplistorUtils.listDir(AmplistorUtils.DEFAULT_AMPLI_VCELL_LOGS_URL, null, null);
        throw new Exception("Souldn't have gotten here, Expecting failure to get vcell_logs with no authentication");
    } catch (AuthenticationException e) {
    // ignore, this should happen
    }
    // 
    // try to get dir list from vcell_logs with credentials (should succeed)
    // 
    ArrayList<String> vcell_logs_dirlist = AmplistorUtils.listDir(AmplistorUtils.DEFAULT_AMPLI_VCELL_LOGS_URL, null, amplistorCredential);
    System.out.println("Found " + vcell_logs_dirlist.size() + " files in vcell_logs directory");
    // 
    try {
        AmplistorUtils.deleteFilesOperation(new String[] { tmpFiles[0].getName() }, AmplistorUtils.DEFAULT_AMPLI_VCELL_LOGS_URL, null);
        throw new Exception("Souldn't have gotten here, Expecting failure to get vcell_logs with no authentication");
    } catch (AuthenticationException e) {
    // ignore, this should happen
    }
    // 
    // try to delete file from vcell_logs with credentials (should succeed)
    // 
    AmplistorUtils.deleteFilesOperation(new String[] { tmpFiles[0].getName() }, AmplistorUtils.DEFAULT_AMPLI_VCELL_LOGS_URL, amplistorCredential);
    // 
    try {
        for (int i = 0; i < NUMFILES; i++) {
            tmpFiles[i].delete();
        }
        tmpDir.delete();
    } catch (Exception e) {
    // ignore, not part of test
    }
}