Search in sources :

Example 1 with EntityFetcher

use of com.google.samples.apps.iosched.server.schedule.input.fetcher.EntityFetcher in project iosched by google.

the class DataExtractorTest method setUp.

/**
   * @throws java.lang.Exception
   */
@Before
public void setUp() throws Exception {
    fakeFetcher = new EntityFetcher() {

        @Override
        public JsonElement fetch(Enum<?> entityType, Map<String, String> params) throws IOException {
            String filename = "sample_" + entityType.name();
            if (params != null && params.get("page") != null && Integer.parseInt(params.get("page")) > 1) {
                filename += "_page" + params.get("page");
            }
            filename += ".json";
            InputStream stream = TestHelper.openTestDataFileStream(filename);
            JsonReader reader = new JsonReader(new InputStreamReader(stream, Charset.forName("UTF-8")));
            return new JsonParser().parse(reader);
        }
    };
    RemoteFilesEntityFetcherFactory.setBuilder(new FetcherBuilder() {

        @Override
        public FetcherBuilder setSourceFiles(String... filenames) {
            return this;
        }

        @Override
        public EntityFetcher build() {
            return fakeFetcher;
        }
    });
    sources = new ExtraInput().fetchAllDataSources();
    sources.putAll(new VendorDynamicInput(fakeFetcher).fetchAllDataSources());
}
Also used : InputStreamReader(java.io.InputStreamReader) InputStream(java.io.InputStream) IOException(java.io.IOException) FetcherBuilder(com.google.samples.apps.iosched.server.schedule.input.fetcher.RemoteFilesEntityFetcherFactory.FetcherBuilder) EntityFetcher(com.google.samples.apps.iosched.server.schedule.input.fetcher.EntityFetcher) JsonElement(com.google.gson.JsonElement) ExtraInput(com.google.samples.apps.iosched.server.schedule.server.input.ExtraInput) JsonReader(com.google.gson.stream.JsonReader) VendorDynamicInput(com.google.samples.apps.iosched.server.schedule.server.input.VendorDynamicInput) JsonParser(com.google.gson.JsonParser) Before(org.junit.Before)

Example 2 with EntityFetcher

use of com.google.samples.apps.iosched.server.schedule.input.fetcher.EntityFetcher in project iosched by google.

the class APIUpdater method run.

public void run(boolean force, boolean obfuscate, OutputStream optionalOutput) throws IOException {
    RemoteFilesEntityFetcherFactory.setBuilder(new RemoteFilesEntityFetcherFactory.FetcherBuilder() {

        String[] filenames;

        @Override
        public RemoteFilesEntityFetcherFactory.FetcherBuilder setSourceFiles(String... filenames) {
            this.filenames = filenames;
            return this;
        }

        @Override
        public EntityFetcher build() {
            return new CloudStorageRemoteFilesEntityFetcher(filenames);
        }
    });
    UpdateRunLogger logger = new UpdateRunLogger();
    CloudFileManager fileManager = new CloudFileManager();
    logger.startTimer();
    JsonDataSources sources = new ExtraInput().fetchAllDataSources();
    logger.stopTimer("fetchExtraAPI");
    logger.startTimer();
    sources.putAll(new VendorStaticInput().fetchAllDataSources());
    logger.stopTimer("fetchVendorStaticAPI");
    logger.startTimer();
    JsonObject newData = new DataExtractor(obfuscate).extractFromDataSources(sources);
    logger.stopTimer("extractOurData");
    logger.startTimer();
    byte[] newHash = CloudFileManager.calulateHash(newData);
    logger.stopTimer("calculateHash");
    // compare current Vendor API log with the one from previous run:
    logger.startTimer();
    if (!force && isUpToDate(newHash, logger)) {
        logger.logNoopRun();
        return;
    }
    logger.stopTimer("compareHash");
    logger.startTimer();
    ManifestData dataProduction = extractManifestData(fileManager.readProductionManifest(), null);
    //ManifestData dataStaging = extractManifestData(fileManager.readStagingManifest(), dataProduction);
    logger.stopTimer("readManifest");
    JsonWriter optionalOutputWriter = null;
    logger.startTimer();
    // Upload a new version of the sessions file
    if (optionalOutput != null) {
        // send data to the outputstream
        Writer writer = Channels.newWriter(Channels.newChannel(optionalOutput), "UTF-8");
        optionalOutputWriter = new JsonWriter(writer);
        optionalOutputWriter.setIndent("  ");
        new Gson().toJson(newData, optionalOutputWriter);
        optionalOutputWriter.flush();
    } else {
        // save data to the CloudStorage
        fileManager.createOrUpdate(dataProduction.sessionsFilename, newData, false);
    }
    logger.stopTimer("uploadNewSessionsFile");
    // Check data consistency
    logger.startTimer();
    DataCheck checker = new DataCheck(fileManager);
    CheckResult result = checker.check(sources, newData, dataProduction);
    if (!result.failures.isEmpty()) {
        reportDataCheckFailures(result, optionalOutput);
    }
    logger.stopTimer("runDataCheck");
    if (optionalOutput == null) {
        // Only update manifest and log if saving to persistent storage
        logger.startTimer();
        // Create new manifests
        JsonObject newProductionManifest = new JsonObject();
        newProductionManifest.add("format", new JsonPrimitive(Config.MANIFEST_FORMAT_VERSION));
        newProductionManifest.add("data_files", dataProduction.dataFiles);
        JsonObject newStagingManifest = new JsonObject();
        newStagingManifest.add("format", new JsonPrimitive(Config.MANIFEST_FORMAT_VERSION));
        // newStagingManifest.add("data_files", dataStaging.dataFiles);
        // save manifests to the CloudStorage
        fileManager.createOrUpdateProductionManifest(newProductionManifest);
        fileManager.createOrUpdateStagingManifest(newStagingManifest);
        try {
            // notify production GCM server:
            new GCMPing().notifyGCMServer(Config.GCM_URL, Config.GCM_API_KEY);
        } catch (Throwable t) {
            Logger.getLogger(APIUpdater.class.getName()).log(Level.SEVERE, "Error while pinging GCM server", t);
        }
        logger.stopTimer("uploadManifest");
        logger.logUpdateRun(dataProduction.majorVersion, dataProduction.minorVersion, dataProduction.sessionsFilename, newHash, newData, force);
    }
}
Also used : JsonPrimitive(com.google.gson.JsonPrimitive) CloudStorageRemoteFilesEntityFetcher(com.google.samples.apps.iosched.server.schedule.server.input.fetcher.CloudStorageRemoteFilesEntityFetcher) DataExtractor(com.google.samples.apps.iosched.server.schedule.model.DataExtractor) JsonObject(com.google.gson.JsonObject) Gson(com.google.gson.Gson) RemoteFilesEntityFetcherFactory(com.google.samples.apps.iosched.server.schedule.input.fetcher.RemoteFilesEntityFetcherFactory) EntityFetcher(com.google.samples.apps.iosched.server.schedule.input.fetcher.EntityFetcher) CloudStorageRemoteFilesEntityFetcher(com.google.samples.apps.iosched.server.schedule.server.input.fetcher.CloudStorageRemoteFilesEntityFetcher) JsonDataSources(com.google.samples.apps.iosched.server.schedule.model.JsonDataSources) ExtraInput(com.google.samples.apps.iosched.server.schedule.server.input.ExtraInput) CheckResult(com.google.samples.apps.iosched.server.schedule.model.DataCheck.CheckResult) VendorStaticInput(com.google.samples.apps.iosched.server.schedule.server.input.VendorStaticInput) DataCheck(com.google.samples.apps.iosched.server.schedule.model.DataCheck) JsonWriter(com.google.gson.stream.JsonWriter) CloudFileManager(com.google.samples.apps.iosched.server.schedule.server.cloudstorage.CloudFileManager) JsonWriter(com.google.gson.stream.JsonWriter) Writer(java.io.Writer)

Example 3 with EntityFetcher

use of com.google.samples.apps.iosched.server.schedule.input.fetcher.EntityFetcher in project iosched by google.

the class ExtraInputTest method setUp.

/**
   * @throws java.lang.Exception
   */
@Before
public void setUp() throws Exception {
    fakeFetcher = new EntityFetcher() {

        @Override
        public JsonElement fetch(Enum<?> entityType, Map<String, String> params) throws IOException {
            String filename = "sample_" + entityType.name() + ".json";
            InputStream stream = TestHelper.openTestDataFileStream(filename);
            assertNotNull("open file " + filename, stream);
            JsonReader reader = new JsonReader(new InputStreamReader(stream, Charset.forName("UTF-8")));
            return new JsonParser().parse(reader);
        }
    };
    RemoteFilesEntityFetcherFactory.setBuilder(new FetcherBuilder() {

        @Override
        public FetcherBuilder setSourceFiles(String... filenames) {
            return this;
        }

        @Override
        public EntityFetcher build() {
            return fakeFetcher;
        }
    });
}
Also used : EntityFetcher(com.google.samples.apps.iosched.server.schedule.input.fetcher.EntityFetcher) InputStreamReader(java.io.InputStreamReader) JsonElement(com.google.gson.JsonElement) InputStream(java.io.InputStream) JsonReader(com.google.gson.stream.JsonReader) IOException(java.io.IOException) FetcherBuilder(com.google.samples.apps.iosched.server.schedule.input.fetcher.RemoteFilesEntityFetcherFactory.FetcherBuilder) JsonParser(com.google.gson.JsonParser) Before(org.junit.Before)

Aggregations

EntityFetcher (com.google.samples.apps.iosched.server.schedule.input.fetcher.EntityFetcher)3 JsonElement (com.google.gson.JsonElement)2 JsonParser (com.google.gson.JsonParser)2 JsonReader (com.google.gson.stream.JsonReader)2 FetcherBuilder (com.google.samples.apps.iosched.server.schedule.input.fetcher.RemoteFilesEntityFetcherFactory.FetcherBuilder)2 ExtraInput (com.google.samples.apps.iosched.server.schedule.server.input.ExtraInput)2 IOException (java.io.IOException)2 InputStream (java.io.InputStream)2 InputStreamReader (java.io.InputStreamReader)2 Before (org.junit.Before)2 Gson (com.google.gson.Gson)1 JsonObject (com.google.gson.JsonObject)1 JsonPrimitive (com.google.gson.JsonPrimitive)1 JsonWriter (com.google.gson.stream.JsonWriter)1 RemoteFilesEntityFetcherFactory (com.google.samples.apps.iosched.server.schedule.input.fetcher.RemoteFilesEntityFetcherFactory)1 DataCheck (com.google.samples.apps.iosched.server.schedule.model.DataCheck)1 CheckResult (com.google.samples.apps.iosched.server.schedule.model.DataCheck.CheckResult)1 DataExtractor (com.google.samples.apps.iosched.server.schedule.model.DataExtractor)1 JsonDataSources (com.google.samples.apps.iosched.server.schedule.model.JsonDataSources)1 CloudFileManager (com.google.samples.apps.iosched.server.schedule.server.cloudstorage.CloudFileManager)1