Examples with CloudFileManager com.google.samples.apps.iosched.server.schedule.server.cloudstorage.CloudFileManager used on opensource projects

Example 1 with CloudFileManager

use of com.google.samples.apps.iosched.server.schedule.server.cloudstorage.CloudFileManager in project iosched by google.

the class APIUpdater method run.

public void run(boolean force, boolean obfuscate, OutputStream optionalOutput) throws IOException {
    RemoteFilesEntityFetcherFactory.setBuilder(new RemoteFilesEntityFetcherFactory.FetcherBuilder() {

        String[] filenames;

        @Override
        public RemoteFilesEntityFetcherFactory.FetcherBuilder setSourceFiles(String... filenames) {
            this.filenames = filenames;
            return this;
        }

        @Override
        public EntityFetcher build() {
            return new CloudStorageRemoteFilesEntityFetcher(filenames);
        }
    });
    UpdateRunLogger logger = new UpdateRunLogger();
    CloudFileManager fileManager = new CloudFileManager();
    logger.startTimer();
    JsonDataSources sources = new ExtraInput().fetchAllDataSources();
    logger.stopTimer("fetchExtraAPI");
    logger.startTimer();
    sources.putAll(new VendorStaticInput().fetchAllDataSources());
    logger.stopTimer("fetchVendorStaticAPI");
    logger.startTimer();
    JsonObject newData = new DataExtractor(obfuscate).extractFromDataSources(sources);
    logger.stopTimer("extractOurData");
    logger.startTimer();
    byte[] newHash = CloudFileManager.calulateHash(newData);
    logger.stopTimer("calculateHash");
    // compare current Vendor API log with the one from previous run:
    logger.startTimer();
    if (!force && isUpToDate(newHash, logger)) {
        logger.logNoopRun();
        return;
    }
    logger.stopTimer("compareHash");
    logger.startTimer();
    ManifestData dataProduction = extractManifestData(fileManager.readProductionManifest(), null);
    //ManifestData dataStaging = extractManifestData(fileManager.readStagingManifest(), dataProduction);
    logger.stopTimer("readManifest");
    JsonWriter optionalOutputWriter = null;
    logger.startTimer();
    // Upload a new version of the sessions file
    if (optionalOutput != null) {
        // send data to the outputstream
        Writer writer = Channels.newWriter(Channels.newChannel(optionalOutput), "UTF-8");
        optionalOutputWriter = new JsonWriter(writer);
        optionalOutputWriter.setIndent("  ");
        new Gson().toJson(newData, optionalOutputWriter);
        optionalOutputWriter.flush();
    } else {
        // save data to the CloudStorage
        fileManager.createOrUpdate(dataProduction.sessionsFilename, newData, false);
    }
    logger.stopTimer("uploadNewSessionsFile");
    // Check data consistency
    logger.startTimer();
    DataCheck checker = new DataCheck(fileManager);
    CheckResult result = checker.check(sources, newData, dataProduction);
    if (!result.failures.isEmpty()) {
        reportDataCheckFailures(result, optionalOutput);
    }
    logger.stopTimer("runDataCheck");
    if (optionalOutput == null) {
        // Only update manifest and log if saving to persistent storage
        logger.startTimer();
        // Create new manifests
        JsonObject newProductionManifest = new JsonObject();
        newProductionManifest.add("format", new JsonPrimitive(Config.MANIFEST_FORMAT_VERSION));
        newProductionManifest.add("data_files", dataProduction.dataFiles);
        JsonObject newStagingManifest = new JsonObject();
        newStagingManifest.add("format", new JsonPrimitive(Config.MANIFEST_FORMAT_VERSION));
        // newStagingManifest.add("data_files", dataStaging.dataFiles);
        // save manifests to the CloudStorage
        fileManager.createOrUpdateProductionManifest(newProductionManifest);
        fileManager.createOrUpdateStagingManifest(newStagingManifest);
        try {
            // notify production GCM server:
            new GCMPing().notifyGCMServer(Config.GCM_URL, Config.GCM_API_KEY);
        } catch (Throwable t) {
            Logger.getLogger(APIUpdater.class.getName()).log(Level.SEVERE, "Error while pinging GCM server", t);
        }
        logger.stopTimer("uploadManifest");
        logger.logUpdateRun(dataProduction.majorVersion, dataProduction.minorVersion, dataProduction.sessionsFilename, newHash, newData, force);
    }
}

Example 2 with CloudFileManager

use of com.google.samples.apps.iosched.server.schedule.server.cloudstorage.CloudFileManager in project iosched by google.

the class CMSUpdateServlet method process.

private void process(HttpServletResponse resp, boolean showOnly) throws IOException {
    // everything ok, let's update
    StringBuilder summary = new StringBuilder();
    JsonObject contents = new JsonObject();
    JsonDataSources sources = new VendorDynamicInput().fetchAllDataSources();
    for (String entity : sources) {
        JsonArray array = new JsonArray();
        JsonDataSource source = sources.getSource(entity);
        for (JsonObject obj : source) {
            array.add(obj);
        }
        summary.append(entity).append(": ").append(source.size()).append("\n");
        contents.add(entity, array);
    }
    if (showOnly) {
        // Show generated contents to the output
        resp.setContentType("application/json");
        Writer writer = Channels.newWriter(Channels.newChannel(resp.getOutputStream()), "UTF-8");
        JsonWriter outputWriter = new JsonWriter(writer);
        outputWriter.setIndent("  ");
        new Gson().toJson(contents, outputWriter);
        outputWriter.flush();
    } else {
        // Write file to cloud storage
        CloudFileManager fileManager = new CloudFileManager();
        fileManager.createOrUpdate("__raw_session_data.json", contents, true);
        // send email
        Message message = new Message();
        message.setSender(Config.EMAIL_FROM);
        message.setSubject("[iosched-data-update] Manual sync from CMS");
        message.setTextBody("Hey,\n\n" + "(this message is autogenerated)\n" + "This is a heads up that " + userService.getCurrentUser().getEmail() + " has just updated the IOSched 2015 data from the Vendor CMS.\n\n" + "Here is a brief status of what has been extracted from the Vendor API:\n" + summary + "\n\n" + "If you want to check the most current data that will soon be sync'ed to the IOSched Android app, " + "check this link: http://storage.googleapis.com/iosched-updater-dev.appspot.com/__raw_session_data.json\n" + "This data will remain unchanged until someone with proper privileges updates it again on https://iosched-updater-dev.appspot.com/cmsupdate\n\n" + "Thanks!\n\n" + "A robot on behalf of the IOSched team!\n\n" + "PS: you are receiving this either because you are an admin of the IOSched project or " + "because you are in a hard-coded list of I/O organizers. If you don't want to " + "receive it anymore, pay me a beer and ask kindly.");
        // TODO(arthurthompson): Reimplement mailing, it currently fails due to invalid sender.
        //MailServiceFactory.getMailService().sendToAdmins(message);
        resp.sendRedirect("/admin/schedule/updateok.html");
    }
}

Example 3 with CloudFileManager

use of com.google.samples.apps.iosched.server.schedule.server.cloudstorage.CloudFileManager in project iosched by google.

the class LogDataServlet method doGet.

@Override
public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
    resp.setContentType("application/json");
    UpdateRunLogger logger = new UpdateRunLogger();
    JsonObject response = new JsonObject();
    int limitElements = 10;
    if (req.getParameter("limit") != null) {
        limitElements = Integer.parseInt(req.getParameter("limit"));
    }
    List<Entity> lastRunsEntities = logger.getMostRecentRuns(limitElements);
    JsonArray lastRuns = new JsonArray();
    for (Entity run : lastRunsEntities) {
        JsonObject obj = new JsonObject();
        JsonObject timings = new JsonObject();
        TreeMap<String, Object> sortedMap = new TreeMap<String, Object>(run.getProperties());
        for (Entry<String, Object> property : sortedMap.entrySet()) {
            Object value = property.getValue();
            String key = property.getKey();
            if (key.startsWith("time_")) {
                timings.add(key.substring("time_".length()), new JsonPrimitive((Number) value));
            } else {
                JsonPrimitive converted = null;
                if (value instanceof ShortBlob) {
                    converted = new JsonPrimitive(bytesToHex(((ShortBlob) value).getBytes()));
                } else if (value instanceof String) {
                    converted = new JsonPrimitive((String) value);
                } else if (value instanceof Number) {
                    converted = new JsonPrimitive((Number) value);
                } else if (value instanceof Boolean) {
                    converted = new JsonPrimitive((Boolean) value);
                } else if (value instanceof Character) {
                    converted = new JsonPrimitive((Character) value);
                } else if (value instanceof Date) {
                    converted = new JsonPrimitive(DateFormat.getDateTimeInstance().format((Date) value));
                }
                if (converted != null) {
                    obj.add(key, converted);
                }
            }
        }
        obj.add("timings", timings);
        lastRuns.add(obj);
    }
    response.add("lastruns", lastRuns);
    CloudFileManager cloudManager = new CloudFileManager();
    response.add("bucket", new JsonPrimitive(cloudManager.getBucketName()));
    response.add("productionManifest", new JsonPrimitive(cloudManager.getProductionManifestURL()));
    response.add("stagingManifest", new JsonPrimitive(cloudManager.getStagingManifestURL()));
    new GsonBuilder().setPrettyPrinting().disableHtmlEscaping().create().toJson(response, resp.getWriter());
}