use of co.cask.cdap.test.ServiceManager in project cdap by caskdata.
the class LogAnalysisAppTest method test.
@Test
public void test() throws Exception {
// Deploy the App
ApplicationManager appManager = deployApplication(LogAnalysisApp.class);
// Send a stream events to the Stream
StreamManager streamManager = getStreamManager(LogAnalysisApp.LOG_STREAM);
streamManager.send(LOG_1);
streamManager.send(LOG_2);
streamManager.send(LOG_3);
// run the spark program
SparkManager sparkManager = appManager.getSparkManager(LogAnalysisApp.ResponseCounterSpark.class.getSimpleName()).start();
sparkManager.waitForRun(ProgramRunStatus.COMPLETED, 60, TimeUnit.SECONDS);
// run the mapreduce job
MapReduceManager mapReduceManager = appManager.getMapReduceManager(HitCounterProgram.class.getSimpleName()).start();
mapReduceManager.waitForRun(ProgramRunStatus.COMPLETED, 3, TimeUnit.MINUTES);
// start and wait for services
ServiceManager hitCounterServiceManager = getServiceManager(appManager, LogAnalysisApp.HIT_COUNTER_SERVICE);
ServiceManager responseCounterServiceManager = getServiceManager(appManager, LogAnalysisApp.RESPONSE_COUNTER_SERVICE);
ServiceManager requestCounterServiceManager = getServiceManager(appManager, LogAnalysisApp.REQUEST_COUNTER_SERVICE);
//Query for hit counts and verify it
URL totalHitsURL = new URL(hitCounterServiceManager.getServiceURL(15, TimeUnit.SECONDS), LogAnalysisApp.HitCounterServiceHandler.HIT_COUNTER_SERVICE_PATH);
HttpResponse response = HttpRequests.execute(HttpRequest.post(totalHitsURL).withBody("{\"url\":\"" + "/home.html" + "\"}").build());
Assert.assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
Assert.assertEquals(TOTAL_HITS_VALUE, response.getResponseBodyAsString());
// query for total responses for a response code and verify it
URL responseCodeURL = new URL(responseCounterServiceManager.getServiceURL(15, TimeUnit.SECONDS), LogAnalysisApp.ResponseCounterHandler.RESPONSE_COUNT_PATH + "/" + RESPONSE_CODE);
HttpRequest request = HttpRequest.get(responseCodeURL).build();
response = HttpRequests.execute(request);
Assert.assertEquals(TOTAL_RESPONSE_VALUE, response.getResponseBodyAsString());
// query to get partitions in the request count tpfs
URL requestCountFilsetsURL = new URL(requestCounterServiceManager.getServiceURL(15, TimeUnit.SECONDS), LogAnalysisApp.RequestCounterHandler.REQUEST_COUNTER_PARTITIONS_PATH);
request = HttpRequest.get(requestCountFilsetsURL).build();
response = HttpRequests.execute(request);
TreeSet<String> partitions = GSON.fromJson(response.getResponseBodyAsString(), new TypeToken<TreeSet<String>>() {
}.getType());
Assert.assertEquals(1, partitions.size());
String partition = partitions.iterator().next();
//Query for the contents of the files in this partition and verify
URL requestFilesetContentURL = new URL(requestCounterServiceManager.getServiceURL(15, TimeUnit.SECONDS), LogAnalysisApp.RequestCounterHandler.REQUEST_FILE_CONTENT_PATH);
response = HttpRequests.execute(HttpRequest.post(requestFilesetContentURL).withBody("{\"" + LogAnalysisApp.RequestCounterHandler.REQUEST_FILE_PATH_HANDLER_KEY + "\":\"" + partition + "\"}").build());
Assert.assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
Map<String, Integer> responseMap = GSON.fromJson(response.getResponseBodyAsString(), new TypeToken<Map<String, Integer>>() {
}.getType());
Assert.assertTrue(responseMap.equals(TPFS_RESULT));
}
use of co.cask.cdap.test.ServiceManager in project cdap by caskdata.
the class DecisionTreeRegressionAppTest method test.
@Test
public void test() throws Exception {
// Deploy the Application
ApplicationManager appManager = deployApplication(DecisionTreeRegressionApp.class);
// Start the Service
ServiceManager serviceManager = appManager.getServiceManager(ModelDataService.SERVICE_NAME).start();
serviceManager.waitForStatus(true, 30, 1);
URL serviceURL = serviceManager.getServiceURL(15, TimeUnit.SECONDS);
URL addDataURL = new URL(serviceURL, "labels");
HttpRequest request = HttpRequest.builder(HttpMethod.PUT, addDataURL).withBody(new InputSupplier<InputStream>() {
@Override
public InputStream getInput() throws IOException {
return getClass().getClassLoader().getResourceAsStream("sample_libsvm_data.txt");
}
}).build();
HttpResponse response = HttpRequests.execute(request);
Assert.assertEquals(200, response.getResponseCode());
// Start a Spark Program
SparkManager sparkManager = appManager.getSparkManager(ModelTrainer.NAME).start();
sparkManager.waitForRun(ProgramRunStatus.COMPLETED, 60, TimeUnit.SECONDS);
// Check that there is a new model
URL listModelsURL = new URL(serviceURL, "models");
request = HttpRequest.builder(HttpMethod.GET, listModelsURL).build();
response = HttpRequests.execute(request);
Assert.assertEquals(200, response.getResponseCode());
List<String> models = GSON.fromJson(response.getResponseBodyAsString(), new TypeToken<List<String>>() {
}.getType());
Assert.assertEquals(1, models.size());
// Check that there is some model metadata
String modelId = models.get(0);
URL modelMetaURL = new URL(serviceURL, "models/" + modelId);
request = HttpRequest.builder(HttpMethod.GET, modelMetaURL).build();
response = HttpRequests.execute(request);
Assert.assertEquals(200, response.getResponseCode());
ModelMeta meta = GSON.fromJson(response.getResponseBodyAsString(), ModelMeta.class);
Assert.assertNotNull(meta);
Assert.assertEquals(0.7, meta.getTrainingPercentage(), 0.000001);
Assert.assertEquals(692, meta.getNumFeatures());
// Check that the corresponding model file exists
DataSetManager<FileSet> modelFiles = getDataset(DecisionTreeRegressionApp.MODEL_DATASET);
Assert.assertTrue(modelFiles.get().getBaseLocation().append(modelId).exists());
}
use of co.cask.cdap.test.ServiceManager in project cdap by caskdata.
the class DataPipelineTest method testServiceUrl.
public void testServiceUrl(Engine engine) throws Exception {
// Deploy the ServiceApp application
ApplicationManager appManager = deployApplication(ServiceApp.class);
// Start Greeting service and use it
ServiceManager serviceManager = appManager.getServiceManager(ServiceApp.Name.SERVICE_NAME).start();
// Wait service startup
serviceManager.waitForStatus(true);
URL url = new URL(serviceManager.getServiceURL(), "name");
HttpRequest httpRequest = HttpRequest.post(url).withBody("bob").build();
HttpResponse httpResponse = HttpRequests.execute(httpRequest);
Assert.assertEquals(HttpURLConnection.HTTP_OK, httpResponse.getResponseCode());
url = new URL(serviceManager.getServiceURL(), "name/bob");
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
String response;
try {
response = new String(ByteStreams.toByteArray(connection.getInputStream()), Charsets.UTF_8);
} finally {
connection.disconnect();
}
Assert.assertEquals("bob", response);
String sourceName = "ServiceUrlInput-" + engine.name();
String sinkName = "ServiceUrlOutput-" + engine.name();
/*
* source --> filter --> sink
*/
ETLBatchConfig etlConfig = ETLBatchConfig.builder("* * * * *").setEngine(engine).addStage(new ETLStage("source", MockSource.getPlugin(sourceName))).addStage(new ETLStage("filter", FilterTransform.getPlugin("name"))).addStage(new ETLStage("sink", MockSink.getPlugin(sinkName))).addConnection("source", "filter").addConnection("filter", "sink").build();
AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(APP_ARTIFACT, etlConfig);
ApplicationId appId = NamespaceId.DEFAULT.app("ServiceUrl-" + engine);
appManager = deployApplication(appId.toId(), appRequest);
Schema schema = Schema.recordOf("testRecord", Schema.Field.of("name", Schema.of(Schema.Type.STRING)));
StructuredRecord recordSamuel = StructuredRecord.builder(schema).set("name", "samuel").build();
StructuredRecord recordBob = StructuredRecord.builder(schema).set("name", "bob").build();
StructuredRecord recordJane = StructuredRecord.builder(schema).set("name", "jane").build();
// write one record to each source
DataSetManager<Table> inputManager = getDataset(NamespaceId.DEFAULT.dataset(sourceName));
MockSource.writeInput(inputManager, ImmutableList.of(recordSamuel, recordBob, recordJane));
WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME);
workflowManager.start();
workflowManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES);
// check output
DataSetManager<Table> sinkManager = getDataset(sinkName);
Set<StructuredRecord> expected = ImmutableSet.of(recordBob);
Set<StructuredRecord> actual = Sets.newHashSet(MockSink.readOutput(sinkManager));
Assert.assertEquals(expected, actual);
serviceManager.stop();
serviceManager.waitForRun(ProgramRunStatus.KILLED, 180, TimeUnit.SECONDS);
}
use of co.cask.cdap.test.ServiceManager in project cdap by caskdata.
the class PurchaseAppTest method test.
@Test
public void test() throws Exception {
// Deploy the PurchaseApp application
ApplicationManager appManager = deployApplication(PurchaseApp.class);
// Start PurchaseFlow
FlowManager flowManager = appManager.getFlowManager("PurchaseFlow").start();
// Send stream events to the "purchaseStream" Stream
StreamManager streamManager = getStreamManager("purchaseStream");
streamManager.send("bob bought 3 apples for $30");
streamManager.send("joe bought 1 apple for $100");
streamManager.send("joe bought 10 pineapples for $20");
streamManager.send("cat bought 3 bottles for $12");
streamManager.send("cat bought 2 pops for $14");
try {
// Wait for the last Flowlet processing 5 events, or at most 15 seconds
RuntimeMetrics metrics = flowManager.getFlowletMetrics("collector");
metrics.waitForProcessed(5, 15, TimeUnit.SECONDS);
} finally {
flowManager.stop();
}
ServiceManager userProfileServiceManager = getUserProfileServiceManager(appManager);
// Add customer's profile information
URL userProfileUrl = new URL(userProfileServiceManager.getServiceURL(15, TimeUnit.SECONDS), UserProfileServiceHandler.USER_ENDPOINT);
HttpURLConnection userProfileConnection = (HttpURLConnection) userProfileUrl.openConnection();
String userProfileJson = "{'id' : 'joe', 'firstName': 'joe', 'lastName':'bernard', 'categories': ['fruits']}";
try {
userProfileConnection.setDoOutput(true);
userProfileConnection.setRequestMethod("POST");
userProfileConnection.getOutputStream().write(userProfileJson.getBytes(Charsets.UTF_8));
Assert.assertEquals(HttpURLConnection.HTTP_OK, userProfileConnection.getResponseCode());
} finally {
userProfileConnection.disconnect();
}
// Test service to retrieve customer's profile information
userProfileUrl = new URL(userProfileServiceManager.getServiceURL(15, TimeUnit.SECONDS), UserProfileServiceHandler.USER_ENDPOINT + "/joe");
userProfileConnection = (HttpURLConnection) userProfileUrl.openConnection();
Assert.assertEquals(HttpURLConnection.HTTP_OK, userProfileConnection.getResponseCode());
String customerJson;
try {
customerJson = new String(ByteStreams.toByteArray(userProfileConnection.getInputStream()), Charsets.UTF_8);
} finally {
userProfileConnection.disconnect();
}
UserProfile profileFromService = GSON.fromJson(customerJson, UserProfile.class);
Assert.assertEquals(profileFromService.getFirstName(), "joe");
Assert.assertEquals(profileFromService.getLastName(), "bernard");
// Run PurchaseHistoryWorkflow which will process the data
MapReduceManager mapReduceManager = appManager.getMapReduceManager(PurchaseHistoryBuilder.class.getSimpleName()).start();
mapReduceManager.waitForRun(ProgramRunStatus.COMPLETED, 3, TimeUnit.MINUTES);
// Start PurchaseHistoryService
ServiceManager purchaseHistoryServiceManager = appManager.getServiceManager(PurchaseHistoryService.SERVICE_NAME).start();
// Wait for service startup
purchaseHistoryServiceManager.waitForStatus(true);
// Test service to retrieve a customer's purchase history
URL url = new URL(purchaseHistoryServiceManager.getServiceURL(15, TimeUnit.SECONDS), "history/joe");
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
String historyJson;
try {
historyJson = new String(ByteStreams.toByteArray(conn.getInputStream()), Charsets.UTF_8);
} finally {
conn.disconnect();
}
PurchaseHistory history = GSON.fromJson(historyJson, PurchaseHistory.class);
Assert.assertEquals("joe", history.getCustomer());
Assert.assertEquals(2, history.getPurchases().size());
UserProfile profileFromPurchaseHistory = history.getUserProfile();
Assert.assertEquals(profileFromPurchaseHistory.getFirstName(), "joe");
Assert.assertEquals(profileFromPurchaseHistory.getLastName(), "bernard");
}
use of co.cask.cdap.test.ServiceManager in project cdap by caskdata.
the class PurchaseAppTest method getUserProfileServiceManager.
private ServiceManager getUserProfileServiceManager(ApplicationManager appManager) throws InterruptedException {
// Start UserProfileService
ServiceManager userProfileServiceManager = appManager.getServiceManager(UserProfileServiceHandler.SERVICE_NAME).start();
// Wait for service startup
userProfileServiceManager.waitForStatus(true);
return userProfileServiceManager;
}
Aggregations