Examples with NYCFileUtils org.onebusaway.admin.util.NYCFileUtils used on opensource projects

Example 1 with NYCFileUtils

use of org.onebusaway.admin.util.NYCFileUtils in project onebusaway-application-modules by camsys.

the class S3FileServiceImpl method get.

@Override
public /**
 * Retrieve the specified key from S3 and store in the given directory.
 */
String get(String key, String tmpDir) {
    _log.debug("get(" + key + ", " + tmpDir + ")");
    NYCFileUtils fs = new NYCFileUtils();
    String filename = fs.parseFileName(key);
    _log.debug("filename=" + filename);
    GetObjectRequest request = new GetObjectRequest(this._bucketName, key);
    S3Object file = _s3.getObject(request);
    String pathAndFileName = tmpDir + File.separator + filename;
    fs.copy(file.getObjectContent(), pathAndFileName);
    return pathAndFileName;
}

Example 2 with NYCFileUtils

use of org.onebusaway.admin.util.NYCFileUtils in project onebusaway-application-modules by camsys.

the class S3FileServiceImpl method createOutputFilesZip.

@Override
public String createOutputFilesZip(String s3Path) {
    String directoryName = null;
    // create tmp dir
    NYCFileUtils fs = new NYCFileUtils();
    directoryName = fs.createTmpDirectory();
    for (String s3File : list(s3Path, MAX_RESULTS)) {
        get(s3File, directoryName);
    }
    final String zipFileName = directoryName + File.separator + "output.zip";
    File outputDirectory = new File(directoryName);
    String[] outputFiles = outputDirectory.list();
    // Buffer for reading the files
    byte[] buffer = new byte[1024];
    ZipOutputStream zout = null;
    try {
        zout = new ZipOutputStream(new FileOutputStream(zipFileName));
        for (int i = 0; i < outputFiles.length; i++) {
            // Add each file in output directory to the zip
            String entryName = directoryName + File.separator + outputFiles[i];
            FileInputStream in = new FileInputStream(entryName);
            // Add ZIP entry
            zout.putNextEntry(new ZipEntry(outputFiles[i]));
            int len;
            while ((len = in.read(buffer)) > 0) {
                zout.write(buffer, 0, len);
            }
            // Close the zip entry and input stream
            zout.closeEntry();
            in.close();
            // clean up after ourselves
            new File(entryName).delete();
        }
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        // Close the zip
        try {
            zout.close();
            // finally remove the tmp directory
            new File(directoryName).delete();
        } catch (IOException e) {
            _log.error("createOutputFileZip failed:", e);
        }
    }
    return zipFileName;
}

Example 3 with NYCFileUtils

use of org.onebusaway.admin.util.NYCFileUtils in project onebusaway-application-modules by camsys.

the class SyncBundleAction method syncBundle.

public String syncBundle() {
    String syncStatus = "Syncing in progress";
    String apiHost = "http://admin.staging.obast.org:9999/api/bundle/latest";
    JsonObject latestBundle = null;
    try {
        latestBundle = getJsonData(apiHost).getAsJsonObject();
    } catch (Exception e) {
        _log.error("Failed to retrieve name of the latest deployed bundle");
    }
    String datasetName = latestBundle.get("dataset").getAsString();
    String buildName = latestBundle.get("name").getAsString();
    String bundleId = latestBundle.get("id").getAsString();
    String bundleFileName = buildName + ".tar.gz";
    String tmpDir = new NYCFileUtils().createTmpDirectory();
    String bundleDir = "/var/lib/oba/bundles";
    String deployDir = "/var/lib/oba/bundles/active";
    String bundleBuildDir = "/var/lib/oba/bundles/builder";
    try {
        String bundleSourceString = "http://admin.staging.obast.org:9999/api/bundle/archive/get-by-name/" + datasetName + "/" + buildName + "/" + bundleFileName;
        URL bundleSource = new URL(bundleSourceString);
        ReadableByteChannel rbc = Channels.newChannel(bundleSource.openStream());
        FileOutputStream fos = new FileOutputStream(tmpDir + File.separator + bundleFileName);
        fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
    } catch (MalformedURLException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
    try {
        unzipBundle(tmpDir, bundleFileName);
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
    // Copy to extracted files /var/lib/oba/bundles/active
    try {
        // FileUtils.copyDirectory(new File(tmpDir + File.separator + "untarredBundle" + File.separator +  buildName), new File(deployDir));
        FileUtils.copyDirectory(new File(tmpDir + File.separator + "untarredBundle"), new File(deployDir));
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
    // Copy downloaded .tar.gz bundle to bundle builds dir
    String buildDest = bundleBuildDir + File.separator + datasetName + File.separator + "builds" + File.separator + buildName + File.separator + bundleFileName;
    try {
        FileUtils.copyFile(new File(tmpDir + File.separator + bundleFileName), new File(buildDest));
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
    // Copy inputs and outputs dirs
    File srcInputDir = new File(tmpDir + File.separator + "untarredBundle" + File.separator + buildName + File.separator + "inputs");
    File srcOutputDir = new File(tmpDir + File.separator + "untarredBundle" + File.separator + buildName + File.separator + "outputs");
    String destBuildsDir = bundleBuildDir + File.separator + datasetName + File.separator + "builds" + File.separator + buildName;
    try {
        FileUtils.copyDirectory(srcInputDir, new File(destBuildsDir + File.separator + "inputs"));
        FileUtils.copyDirectory(srcOutputDir, new File(destBuildsDir + File.separator + "outputs"));
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
    // Copy gtfs and aux files to aux_latest and gtfs_latest
    String gtfsFileDest = bundleBuildDir + File.separator + datasetName + File.separator + "gtfs_latest";
    String auxFileDest = bundleBuildDir + File.separator + datasetName + File.separator + "aux_latest";
    File[] inputFiles = srcInputDir.listFiles();
    for (File inputFile : inputFiles) {
        try {
            String fileName = inputFile.getName();
            int idx = fileName.indexOf("_");
            if (idx > 0) {
                // Skip over config dir
                String agencyNum = fileName.substring(0, idx);
                String zipFileName = fileName.substring(idx + 1);
                if (agencyNum.equals("29")) {
                    // For CT aux files
                    String fileDest = auxFileDest + File.separator + agencyNum + File.separator + zipFileName;
                    FileUtils.copyFile(inputFile, new File(fileDest));
                } else {
                    String fileDest = gtfsFileDest + File.separator + agencyNum + File.separator + zipFileName;
                    FileUtils.copyFile(inputFile, new File(fileDest));
                }
            }
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }
    syncStatus = "Complete";
    return "syncStatus";
}

Example 4 with NYCFileUtils

use of org.onebusaway.admin.util.NYCFileUtils in project onebusaway-application-modules by camsys.

the class BundleBuildingServiceImplTest method setup.

public void setup() {
    _service = new BundleBuildingServiceImpl() {

        @Override
        public String getDefaultAgencyId() {
            return null;
        }
    };
    _service.setDebug(true);
    FileService fileService;
    fileService = new S3FileServiceImpl() {

        @Override
        public void setup() {
        }

        @Override
        public boolean bundleDirectoryExists(String filename) {
            return !"noSuchDirectory".equals(filename);
        }

        @Override
        public boolean createBundleDirectory(String filename) {
            return true;
        }

        @Override
        public List<String[]> listBundleDirectories(int maxResults) {
            ArrayList<String[]> list = new ArrayList<String[]>();
            String[] columns0 = { "2012April", "", "" + System.currentTimeMillis() };
            list.add(columns0);
            String[] columns1 = { "2012Jan", "", "" + System.currentTimeMillis() };
            list.add(columns1);
            String[] columns2 = { "2011April", "", "" + System.currentTimeMillis() };
            list.add(columns2);
            String[] columns3 = { "2011Jan", "", "" + System.currentTimeMillis() };
            list.add(columns3);
            String[] columns4 = { "2010April", "", "" + System.currentTimeMillis() };
            list.add(columns4);
            String[] columns5 = { "2010Jan", "", "" + System.currentTimeMillis() };
            list.add(columns5);
            return list;
        }

        @Override
        public List<String> list(String directory, int maxResults) {
            _log.debug("list called with " + directory);
            ArrayList<String> list = new ArrayList<String>();
            if (directory.equals("test/gtfs_latest")) {
                list.add("gtfs-m34.zip");
            } else if (directory.equals("test/aux_latest")) {
                if ("true".equals(_service.getAuxConfig())) {
                    list.add(CT_GIS_ZIP);
                    list.add(CT_SCHEDULE_ZIP);
                } else {
                    list.add("stif-m34.zip");
                }
            } else if (directory.equals("test/config")) {
            // do nothing
            } else {
                throw new RuntimeException("file not found for dir=" + directory);
            }
            return list;
        }

        @Override
        public String get(String key, String tmpDir) {
            _log.debug("get called with " + key);
            InputStream source = null;
            if (key.equals("gtfs-m34.zip")) {
                source = this.getClass().getResourceAsStream("gtfs-m34.zip");
            } else if (key.equals("stif-m34.zip")) {
                source = this.getClass().getResourceAsStream("stif-m34.zip");
            } else if (key.equals(CT_GIS_ZIP)) {
                source = this.getClass().getResourceAsStream(CT_GIS_ZIP);
            } else if (key.equals(CT_SCHEDULE_ZIP)) {
                source = this.getClass().getResourceAsStream(CT_SCHEDULE_ZIP);
            } else {
                throw new RuntimeException("unmatched key=" + key + " for tmpDir=" + tmpDir);
            }
            String filename = tmpDir + File.separator + key;
            _log.info("copying " + source + " to " + filename);
            new NYCFileUtils().copy(source, filename);
            return filename;
        }

        @Override
        public String put(String key, String file) {
            // do nothing
            return null;
        }
    };
    fileService.setBucketName("obanyc-bundle-data");
    fileService.setGtfsPath("gtfs_latest");
    fileService.setAuxPath("aux_latest");
    fileService.setBuildPath("builds");
    fileService.setConfigPath("config");
    fileService.setup();
    // uncomment for s3
    // fileService = new FileServiceImpl();
    // fileService.setBucketName("obanyc-bundle-data");
    // fileService.setGtfsPath("gtfs_latest");
    // fileService.setStifPath("stif_latest");
    // fileService.setBuildPath("builds");
    // fileService.setConfigPath("config");
    // fileService.setup();
    _service.setFileService(fileService);
    _service.setup();
}

Example 5 with NYCFileUtils

use of org.onebusaway.admin.util.NYCFileUtils in project onebusaway-application-modules by camsys.

the class BundleBuildingServiceImplTest method testBuildHastus.

private void testBuildHastus() {
    _service.setAuxConfig("true");
    String bundleDir = "test";
    String tmpDir = new NYCFileUtils().createTmpDirectory();
    BundleBuildRequest request = new BundleBuildRequest();
    request.setBundleDirectory(bundleDir);
    request.setBundleName("testnameHastus");
    request.setTmpDirectory(tmpDir);
    request.setBundleStartDate("2012-04-08");
    request.setBundleEndDate("2012-07-07");
    assertNotNull(request.getTmpDirectory());
    assertNotNull(request.getBundleDirectory());
    BundleBuildResponse response = new BundleBuildResponse("" + System.currentTimeMillis());
    assertEquals(0, response.getStatusList().size());
    // step 1
    _service.download(request, response);
    assertNotNull(response.getGtfsList());
    assertEquals(1, response.getGtfsList().size());
    assertNotNull(response.getAuxZipList());
    assertEquals(2, response.getAuxZipList().size());
    assertNotNull(response.getStatusList());
    assertTrue(response.getStatusList().size() > 0);
    assertNotNull(response.getConfigList());
    assertEquals(0, response.getConfigList().size());
    // step 2
    _service.prepare(request, response);
    assertFalse(response.isComplete());
    // step 3
    int rc = _service.build(request, response);
    if (response.getException() != null) {
        _log.error("Failed with exception=" + response.getException(), response.getException());
    }
    assertNull(response.getException());
    assertFalse(response.isComplete());
    assertEquals(0, rc);
    // step 4
    // OBANYC-1451 -- fails on OSX TODO
    // _service.assemble(request, response);
    // step 5
    _service.upload(request, response);
    // set by BundleRequestService
    assertFalse(response.isComplete());
}