use of com.google.samples.apps.iosched.server.schedule.input.fetcher.EntityFetcher in project iosched by google.
the class DataExtractorTest method setUp.
/**
* @throws java.lang.Exception
*/
@Before
public void setUp() throws Exception {
fakeFetcher = new EntityFetcher() {
@Override
public JsonElement fetch(Enum<?> entityType, Map<String, String> params) throws IOException {
String filename = "sample_" + entityType.name();
if (params != null && params.get("page") != null && Integer.parseInt(params.get("page")) > 1) {
filename += "_page" + params.get("page");
}
filename += ".json";
InputStream stream = TestHelper.openTestDataFileStream(filename);
JsonReader reader = new JsonReader(new InputStreamReader(stream, Charset.forName("UTF-8")));
return new JsonParser().parse(reader);
}
};
RemoteFilesEntityFetcherFactory.setBuilder(new FetcherBuilder() {
@Override
public FetcherBuilder setSourceFiles(String... filenames) {
return this;
}
@Override
public EntityFetcher build() {
return fakeFetcher;
}
});
sources = new ExtraInput().fetchAllDataSources();
sources.putAll(new VendorDynamicInput(fakeFetcher).fetchAllDataSources());
}
use of com.google.samples.apps.iosched.server.schedule.input.fetcher.EntityFetcher in project iosched by google.
the class APIUpdater method run.
public void run(boolean force, boolean obfuscate, OutputStream optionalOutput) throws IOException {
RemoteFilesEntityFetcherFactory.setBuilder(new RemoteFilesEntityFetcherFactory.FetcherBuilder() {
String[] filenames;
@Override
public RemoteFilesEntityFetcherFactory.FetcherBuilder setSourceFiles(String... filenames) {
this.filenames = filenames;
return this;
}
@Override
public EntityFetcher build() {
return new CloudStorageRemoteFilesEntityFetcher(filenames);
}
});
UpdateRunLogger logger = new UpdateRunLogger();
CloudFileManager fileManager = new CloudFileManager();
logger.startTimer();
JsonDataSources sources = new ExtraInput().fetchAllDataSources();
logger.stopTimer("fetchExtraAPI");
logger.startTimer();
sources.putAll(new VendorStaticInput().fetchAllDataSources());
logger.stopTimer("fetchVendorStaticAPI");
logger.startTimer();
JsonObject newData = new DataExtractor(obfuscate).extractFromDataSources(sources);
logger.stopTimer("extractOurData");
logger.startTimer();
byte[] newHash = CloudFileManager.calulateHash(newData);
logger.stopTimer("calculateHash");
// compare current Vendor API log with the one from previous run:
logger.startTimer();
if (!force && isUpToDate(newHash, logger)) {
logger.logNoopRun();
return;
}
logger.stopTimer("compareHash");
logger.startTimer();
ManifestData dataProduction = extractManifestData(fileManager.readProductionManifest(), null);
//ManifestData dataStaging = extractManifestData(fileManager.readStagingManifest(), dataProduction);
logger.stopTimer("readManifest");
JsonWriter optionalOutputWriter = null;
logger.startTimer();
// Upload a new version of the sessions file
if (optionalOutput != null) {
// send data to the outputstream
Writer writer = Channels.newWriter(Channels.newChannel(optionalOutput), "UTF-8");
optionalOutputWriter = new JsonWriter(writer);
optionalOutputWriter.setIndent(" ");
new Gson().toJson(newData, optionalOutputWriter);
optionalOutputWriter.flush();
} else {
// save data to the CloudStorage
fileManager.createOrUpdate(dataProduction.sessionsFilename, newData, false);
}
logger.stopTimer("uploadNewSessionsFile");
// Check data consistency
logger.startTimer();
DataCheck checker = new DataCheck(fileManager);
CheckResult result = checker.check(sources, newData, dataProduction);
if (!result.failures.isEmpty()) {
reportDataCheckFailures(result, optionalOutput);
}
logger.stopTimer("runDataCheck");
if (optionalOutput == null) {
// Only update manifest and log if saving to persistent storage
logger.startTimer();
// Create new manifests
JsonObject newProductionManifest = new JsonObject();
newProductionManifest.add("format", new JsonPrimitive(Config.MANIFEST_FORMAT_VERSION));
newProductionManifest.add("data_files", dataProduction.dataFiles);
JsonObject newStagingManifest = new JsonObject();
newStagingManifest.add("format", new JsonPrimitive(Config.MANIFEST_FORMAT_VERSION));
// newStagingManifest.add("data_files", dataStaging.dataFiles);
// save manifests to the CloudStorage
fileManager.createOrUpdateProductionManifest(newProductionManifest);
fileManager.createOrUpdateStagingManifest(newStagingManifest);
try {
// notify production GCM server:
new GCMPing().notifyGCMServer(Config.GCM_URL, Config.GCM_API_KEY);
} catch (Throwable t) {
Logger.getLogger(APIUpdater.class.getName()).log(Level.SEVERE, "Error while pinging GCM server", t);
}
logger.stopTimer("uploadManifest");
logger.logUpdateRun(dataProduction.majorVersion, dataProduction.minorVersion, dataProduction.sessionsFilename, newHash, newData, force);
}
}
use of com.google.samples.apps.iosched.server.schedule.input.fetcher.EntityFetcher in project iosched by google.
the class ExtraInputTest method setUp.
/**
* @throws java.lang.Exception
*/
@Before
public void setUp() throws Exception {
fakeFetcher = new EntityFetcher() {
@Override
public JsonElement fetch(Enum<?> entityType, Map<String, String> params) throws IOException {
String filename = "sample_" + entityType.name() + ".json";
InputStream stream = TestHelper.openTestDataFileStream(filename);
assertNotNull("open file " + filename, stream);
JsonReader reader = new JsonReader(new InputStreamReader(stream, Charset.forName("UTF-8")));
return new JsonParser().parse(reader);
}
};
RemoteFilesEntityFetcherFactory.setBuilder(new FetcherBuilder() {
@Override
public FetcherBuilder setSourceFiles(String... filenames) {
return this;
}
@Override
public EntityFetcher build() {
return fakeFetcher;
}
});
}
Aggregations