Examples with NYCFileUtils org.onebusaway.admin.util.NYCFileUtils used on opensource projects

Example 6 with NYCFileUtils

use of org.onebusaway.admin.util.NYCFileUtils in project onebusaway-application-modules by camsys.

the class BundleRequestServiceImplTest method setup.

@Before
public void setup() {
    BundleValidationServiceImpl validationService = new BundleValidationServiceImpl();
    service = new BundleRequestServiceImpl();
    service.setInstanceId("localhost");
    service.setup();
    FileService fileService = new S3FileServiceImpl() {

        @Override
        public void setup() {
        }

        @Override
        public boolean bundleDirectoryExists(String filename) {
            return !"noSuchDirectory".equals(filename);
        }

        @Override
        public boolean createBundleDirectory(String filename) {
            return true;
        }

        @Override
        public List<String[]> listBundleDirectories(int maxResults) {
            ArrayList<String[]> list = new ArrayList<String[]>();
            String[] columns = { "2012April/", "", "" + System.currentTimeMillis() };
            list.add(columns);
            return list;
        }

        @Override
        public List<String> list(String directory, int maxResults) {
            ArrayList<String> list = new ArrayList<String>();
            list.add("google_transit_brooklyn.zip");
            list.add("google_transit_staten_island.zip");
            return list;
        }

        @Override
        public String get(String key, String tmpDir) {
            InputStream input = this.getClass().getResourceAsStream("empty_feed.zip");
            String destination = "/tmp/empty_feed.zip.html";
            new NYCFileUtils().copy(input, destination);
            return destination;
        }

        @Override
        public String put(String key, String tmpDir) {
            // no op
            return null;
        }
    };
    fileService.setup();
    fileService.setBucketName("obanyc-bundle-data-test");
    BundleServerServiceImpl bundleServer = new BundleServerServiceImpl() {

        @Override
        public void setup() {
        // no op
        }

        @Override
        public String start(String instanceId) {
            return instanceId;
        }

        @Override
        public String pollPublicDns(String instanceId, int maxWaitSeconds) {
            return "localhost";
        }

        @Override
        public String findPublicDns(String instanceId) {
            return "localhost";
        }

        @Override
        public String findPublicIp(String instanceId) {
            return "127.0.0.1";
        }

        @Override
        public String stop(String instanceId) {
            return instanceId;
        }

        @Override
        public boolean ping(String instanceId) {
            return true;
        }

        @SuppressWarnings("unchecked")
        @Override
        public <T> T makeRequest(String instanceId, String apiCall, Object payload, Class<T> returnType, int waitTimeInSeconds, Map params, String sessionId) {
            _log.debug("makeRequest called with apiCall=" + apiCall + " and payload=" + payload);
            if (apiCall.equals("/validate/remote/2012Jan/test_0/1/create")) {
                BundleResponse br = new BundleResponse("1");
                return (T) br;
            } else if (apiCall.equals("/build/remote/2012Jan/test_0/null/1/2012-04-08/2012-07-07/create") || apiCall.equals("/build/remote/create")) {
                BundleBuildResponse br = new BundleBuildResponse("1");
                return (T) br;
            } else if (apiCall.equals("/ping/remote")) {
                return (T) "{1}";
            } else if (apiCall.equals("/validate/remote/1/list")) {
                BundleResponse br = new BundleResponse("1");
                br.addValidationFile("file1");
                br.addValidationFile("file2");
                br.setComplete(true);
                return (T) br;
            } else if (apiCall.equals("/build/remote/1/list")) {
                BundleBuildResponse br = new BundleBuildResponse("1");
                br.addGtfsFile("file1");
                br.addGtfsFile("file2");
                br.setComplete(true);
                return (T) br;
            } else {
                _log.error("unmatched apiCall=|" + apiCall + "|");
            }
            return null;
        }
    };
    bundleServer.setEc2User("user");
    bundleServer.setEc2Password("password");
    bundleServer.setup();
    service.setBundleServerService(bundleServer);
    validationService.setFileService(fileService);
}

Example 7 with NYCFileUtils

use of org.onebusaway.admin.util.NYCFileUtils in project onebusaway-application-modules by camsys.

the class BundleBuildingServiceImpl method prepare.

/**
 * stage file locations for bundle building.
 */
@Override
public void prepare(BundleBuildRequest request, BundleBuildResponse response) {
    response.addStatusMessage("preparing for build");
    NYCFileUtils fs = new NYCFileUtils();
    // copy source data to inputs
    String rootPath = request.getTmpDirectory() + File.separator + request.getBundleName();
    response.setBundleRootDirectory(rootPath);
    File rootDir = new File(rootPath);
    rootDir.mkdirs();
    String inputsPath = request.getTmpDirectory() + File.separator + request.getBundleName() + File.separator + INPUTS_DIR;
    response.setBundleInputDirectory(inputsPath);
    File inputsDir = new File(inputsPath);
    inputsDir.mkdirs();
    String outputsPath = request.getTmpDirectory() + File.separator + request.getBundleName() + File.separator + OUTPUT_DIR;
    response.setBundleOutputDirectory(outputsPath);
    File outputsDir = new File(outputsPath);
    outputsDir.mkdirs();
    String dataPath = request.getTmpDirectory() + File.separator + request.getBundleName() + File.separator + DATA_DIR;
    File dataDir = new File(dataPath);
    response.setBundleDataDirectory(dataPath);
    dataDir.mkdirs();
    for (String gtfs : response.getGtfsList()) {
        String outputFilename = null;
        if (!gtfs.endsWith(".zip")) {
            _log.error("ignoring gtfs path that is not a zip:" + gtfs);
            response.addStatusMessage("ignoring gtfs path that is not a zip:" + gtfs);
        } else {
            outputFilename = inputsPath + File.separator + fs.parseFileName(gtfs);
            _log.debug("prepping gtfs:" + gtfs + " to " + outputFilename);
            fs.copyFiles(new File(gtfs), new File(outputFilename));
        }
    }
    _log.debug("finished prepping gtfs!");
    for (String stif : response.getAuxZipList()) {
        _log.debug("prepping stif:" + stif);
        String outputFilename = inputsPath + File.separator + fs.parseFileName(stif);
        fs.copyFiles(new File(stif), new File(outputFilename));
    }
    for (String stifZip : response.getAuxZipList()) {
        _log.debug("stif copying " + stifZip + " to " + request.getTmpDirectory() + File.separator + "stif");
        new NYCFileUtils().unzip(stifZip, request.getTmpDirectory() + File.separator + "stif");
    }
    _log.debug("stip unzip complete ");
    // stage baseLocations
    InputStream baseLocationsStream = this.getClass().getResourceAsStream("/BaseLocations.txt");
    new NYCFileUtils().copy(baseLocationsStream, dataPath + File.separator + "BaseLocations.txt");
    File configPath = new File(inputsPath + File.separator + "config");
    configPath.mkdirs();
    // stage any configuration files
    for (String config : response.getConfigList()) {
        _log.debug("config copying " + config + " to " + inputsPath + File.separator + "config");
        response.addStatusMessage("found additional configuration file=" + config);
        String outputFilename = inputsPath + File.separator + "config" + File.separator + fs.parseFileName(config);
        fs.copyFiles(new File(config), new File(outputFilename));
    }
    if (isStifTaskApplicable()) {
        _log.info("running stif with auxconfig=" + this._auxConfig);
        prepStif(request, response);
    } else {
        _log.info("running hastus with auxconfig=" + this._auxConfig);
        prepHastus(request, response);
    }
}

Example 8 with NYCFileUtils

use of org.onebusaway.admin.util.NYCFileUtils in project onebusaway-application-modules by camsys.

the class BundleBuildingServiceImpl method download.

/**
 * download from S3 and stage for building
 */
@Override
public void download(BundleBuildRequest request, BundleBuildResponse response) {
    String bundleDir = request.getBundleDirectory();
    String tmpDirectory = request.getTmpDirectory();
    if (tmpDirectory == null) {
        tmpDirectory = new NYCFileUtils().createTmpDirectory();
        request.setTmpDirectory(tmpDirectory);
    }
    response.setTmpDirectory(tmpDirectory);
    // download gtfs
    List<String> gtfs = _fileService.list(bundleDir + "/" + _fileService.getGtfsPath(), -1);
    NYCFileUtils fs = new NYCFileUtils();
    // Parent of agency dir
    String parentDir = _fileService.getGtfsPath();
    for (String file : gtfs) {
        _log.debug("downloading gtfs:" + file);
        response.addStatusMessage("downloading gtfs " + file);
        // write some meta_data into the file name for later use
        String agencyDir = parseAgencyDir(parentDir, file);
        String gtfsFileName = _fileService.get(file, tmpDirectory);
        // if we have metadata, rename file to encode metadata
        if (agencyDir != null) {
            File toRename = new File(gtfsFileName);
            String newNameStr = fs.parseDirectory(gtfsFileName) + File.separator + agencyDir + "_" + toRename.getName();
            try {
                fs.moveFile(gtfsFileName, newNameStr);
                response.addGtfsFile(newNameStr);
                _log.debug("gtfs file " + gtfsFileName + " renamed to " + newNameStr);
            } catch (Exception e) {
                _log.error("exception moving GTFS file:", e);
                // use the old one and carry on
                response.addGtfsFile(gtfsFileName);
            }
        } else {
            response.addGtfsFile(gtfsFileName);
        }
    }
    _log.debug("finished download gtfs");
    // download aux files, which could be stif or hastus, etc
    List<String> aux = _fileService.list(bundleDir + "/" + _fileService.getAuxPath(), -1);
    // Parent of agency dir
    parentDir = _fileService.getAuxPath();
    for (String file : aux) {
        _log.info("downloading aux:" + aux);
        response.addStatusMessage("downloading aux files " + file);
        String agencyDir = parseAgencyDir(parentDir, file);
        if (agencyDir == null) {
            response.addAuxZipFile(_fileService.get(file, tmpDirectory));
        } else {
            String auxFileName = _fileService.get(file, tmpDirectory);
            File toRename = new File(file);
            String newNameStr = fs.parseDirectory(auxFileName) + File.separator + agencyDir + "_" + toRename.getName();
            try {
                _log.info(auxFileName + " moved to " + newNameStr);
                fs.moveFile(auxFileName, newNameStr);
                response.addAuxZipFile(newNameStr);
            } catch (Exception e) {
                _log.error("exception moving AUX file:", e);
                // use the old one and carry on
                response.addAuxZipFile(auxFileName);
            }
        }
    }
    _log.info("finished download aux files");
    // download optional configuration files
    List<String> config = _fileService.list(bundleDir + "/" + _fileService.getConfigPath(), -1);
    for (String file : config) {
        _log.debug("downloading config:" + config);
        response.addStatusMessage("downloading config file " + file);
        response.addConfigFile(_fileService.get(file, tmpDirectory));
    }
    _log.debug("download complete");
    response.addStatusMessage("download complete");
}

Example 9 with NYCFileUtils

use of org.onebusaway.admin.util.NYCFileUtils in project onebusaway-application-modules by camsys.

the class BundleBuildingServiceImpl method assemble.

/**
 * arrange files for tar'ing into bundle format
 */
@Override
public void assemble(BundleBuildRequest request, BundleBuildResponse response) {
    response.addStatusMessage("compressing results");
    NYCFileUtils fs = new NYCFileUtils();
    // build BundleMetaData.json.
    // If this is a rebuild of a bundle, re-use the previous bundle id.
    String bundleId = checkPreviousBundleId(request, response);
    new BundleBuildingUtil().generateJsonMetadata(request, response, bundleId);
    String[] paths = { request.getBundleName() };
    String filename = request.getTmpDirectory() + File.separator + request.getBundleName() + ".tar.gz";
    response.setBundleTarFilename(filename);
    response.addStatusMessage("creating bundle=" + filename + " for root dir=" + request.getTmpDirectory());
    String baseDir = request.getTmpDirectory();
    fs.tarcvf(baseDir, paths, filename);
    // now copy inputs and outputs to root for easy access
    // inputs
    String inputsPath = request.getTmpDirectory() + File.separator + INPUTS_DIR;
    File inputsDestDir = new File(inputsPath);
    inputsDestDir.mkdir();
    File inputsDir = new File(response.getBundleInputDirectory());
    _log.debug("copying input");
    File[] inputFiles = inputsDir.listFiles();
    if (inputFiles != null) {
        for (File input : inputFiles) {
            _log.debug("copying " + input + " to " + inputsPath + File.separator + input.getName());
            fs.copyFiles(input, new File(inputsPath + File.separator + input.getName()));
        }
    }
    _log.debug("copying output");
    // outputs
    String outputsPath = request.getTmpDirectory() + File.separator + OUTPUT_DIR;
    File outputsDestDir = new File(outputsPath);
    outputsDestDir.mkdir();
    // copy log file to outputs
    File outputPath = new File(response.getBundleDataDirectory());
    String logFilename = outputPath + File.separator + "bundleBuilder.out.txt";
    fs.copyFiles(new File(logFilename), new File(response.getBundleOutputDirectory() + File.separator + "bundleBuilder.out.txt"));
    response.addOutputFile("bundleBuilder.out.txt");
    // copy the rest of the bundle content to outputs directory
    File outputsDir = new File(response.getBundleOutputDirectory());
    File[] outputFiles = outputsDir.listFiles();
    if (outputFiles != null) {
        for (File output : outputFiles) {
            response.addOutputFile(output.getName());
            fs.copyFiles(output, new File(outputsPath + File.separator + output.getName()));
        }
    }
}

Example 10 with NYCFileUtils

use of org.onebusaway.admin.util.NYCFileUtils in project onebusaway-application-modules by camsys.

the class BundleBuildingServiceImpl method prepStif.

// clean stifs via STIF_PYTHON_CLEANUP_SCRIPT
private void prepStif(BundleBuildRequest request, BundleBuildResponse response) {
    NYCFileUtils fs = new NYCFileUtils();
    // create STIF dir as well
    String stifPath = request.getTmpDirectory() + File.separator + "stif";
    File stifDir = new File(stifPath);
    _log.info("creating stif directory=" + stifPath);
    stifDir.mkdirs();
    try {
        File[] stifDirectories = stifDir.listFiles();
        if (stifDirectories != null) {
            fs = new NYCFileUtils(request.getTmpDirectory());
            String stifUtilUrl = getStifCleanupUrl();
            response.addStatusMessage("downloading " + stifUtilUrl + " to clean stifs");
            fs.wget(stifUtilUrl);
            String stifUtilName = fs.parseFileName(stifUtilUrl);
            // make executable
            fs.chmod("500", request.getTmpDirectory() + File.separator + stifUtilName);
            // for each subdirectory of stif, run the script
            for (File stifSubDir : stifDirectories) {
                String cmd = request.getTmpDirectory() + File.separator + stifUtilName + " " + stifSubDir.getCanonicalPath();
                // kick off process and collect output
                ProcessUtil pu = new ProcessUtil(cmd);
                pu.exec();
                if (pu.getReturnCode() == null || !pu.getReturnCode().equals(0)) {
                    // obanyc-1692, do not send to client
                    String returnCodeMessage = stifUtilName + " exited with return code " + pu.getReturnCode();
                    _log.info(returnCodeMessage);
                    _log.info("output=" + pu.getOutput());
                    _log.info("error=" + pu.getError());
                }
                if (pu.getException() != null) {
                    response.setException(pu.getException());
                }
            }
            response.addStatusMessage("stif cleaning complete");
        }
    } catch (Exception any) {
        response.setException(any);
    }
}