use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.
the class FlowletProgramRunner method outputEmitterFactory.
private OutputEmitterFactory outputEmitterFactory(final BasicFlowletContext flowletContext, final String flowletName, final QueueClientFactory queueClientFactory, final ImmutableList.Builder<ProducerSupplier> producerBuilder, final Table<Node, String, Set<QueueSpecification>> queueSpecs) {
return new OutputEmitterFactory() {
@Override
public <T> OutputEmitter<T> create(String outputName, TypeToken<T> type) {
try {
// first iterate over all queue specifications to find the queue name and all consumer flowlet ids
QueueName queueName = null;
List<String> consumerFlowlets = Lists.newLinkedList();
Node flowlet = Node.flowlet(flowletName);
Schema schema = schemaGenerator.generate(type.getType());
for (Map.Entry<String, Set<QueueSpecification>> entry : queueSpecs.row(flowlet).entrySet()) {
for (QueueSpecification queueSpec : entry.getValue()) {
if (queueSpec.getQueueName().getSimpleName().equals(outputName) && queueSpec.getOutputSchema().equals(schema)) {
queueName = queueSpec.getQueueName();
consumerFlowlets.add(entry.getKey());
break;
}
}
}
if (queueName == null) {
throw new IllegalArgumentException(String.format("No queue specification found for %s, %s", flowletName, type));
}
// create a metric collector for this queue, and also one for each consumer flowlet
final MetricsContext metrics = flowletContext.getProgramMetrics().childContext(Constants.Metrics.Tag.FLOWLET_QUEUE, outputName);
final MetricsContext producerMetrics = metrics.childContext(Constants.Metrics.Tag.PRODUCER, flowletContext.getFlowletId());
final Iterable<MetricsContext> consumerMetrics = Iterables.transform(consumerFlowlets, new Function<String, MetricsContext>() {
@Override
public MetricsContext apply(String consumer) {
return producerMetrics.childContext(Constants.Metrics.Tag.CONSUMER, consumer);
}
});
// create a queue metrics emitter that emit to all of the above collectors
ProducerSupplier producerSupplier = new ProducerSupplier(queueName, queueClientFactory, new QueueMetrics() {
@Override
public void emitEnqueue(int count) {
metrics.increment("process.events.out", count);
for (MetricsContext collector : consumerMetrics) {
collector.increment("queue.pending", count);
}
}
@Override
public void emitEnqueueBytes(int bytes) {
// no-op
}
});
producerBuilder.add(producerSupplier);
return new DatumOutputEmitter<>(producerSupplier, schema, datumWriterFactory.create(type, schema));
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
};
}
use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.
the class FlowletProgramRunner method processSpecificationFactory.
private ProcessSpecificationFactory processSpecificationFactory(final BasicFlowletContext flowletContext, final DataFabricFacade dataFabricFacade, final QueueReaderFactory queueReaderFactory, final String flowletName, final Table<Node, String, Set<QueueSpecification>> queueSpecs, final ImmutableList.Builder<ConsumerSupplier<?>> queueConsumerSupplierBuilder, final SchemaCache schemaCache) {
return new ProcessSpecificationFactory() {
@Override
public <T> ProcessSpecification create(Set<String> inputNames, Schema schema, TypeToken<T> dataType, ProcessMethod<T> method, ConsumerConfig consumerConfig, int batchSize, Tick tickAnnotation) throws Exception {
List<QueueReader<T>> queueReaders = Lists.newLinkedList();
for (Map.Entry<Node, Set<QueueSpecification>> entry : queueSpecs.column(flowletName).entrySet()) {
for (QueueSpecification queueSpec : entry.getValue()) {
final QueueName queueName = queueSpec.getQueueName();
if (queueSpec.getInputSchema().equals(schema) && (inputNames.contains(queueName.getSimpleName()) || inputNames.contains(FlowletDefinition.ANY_INPUT))) {
Node sourceNode = entry.getKey();
if (sourceNode.getType() == FlowletConnection.Type.STREAM) {
ConsumerSupplier<StreamConsumer> consumerSupplier = ConsumerSupplier.create(flowletContext.getOwners(), runtimeUsageRegistry, dataFabricFacade, queueName, consumerConfig);
queueConsumerSupplierBuilder.add(consumerSupplier);
// No decoding is needed, as a process method can only have StreamEvent as type for consuming stream
Function<StreamEvent, T> decoder = wrapInputDecoder(flowletContext, null, queueName, new Function<StreamEvent, T>() {
@Override
@SuppressWarnings("unchecked")
public T apply(StreamEvent input) {
return (T) input;
}
});
queueReaders.add(queueReaderFactory.createStreamReader(queueName.toStreamId(), consumerSupplier, batchSize, decoder));
} else {
int numGroups = getNumGroups(Iterables.concat(queueSpecs.row(entry.getKey()).values()), queueName);
Function<ByteBuffer, T> decoder = wrapInputDecoder(// the producer flowlet,
flowletContext, // the producer flowlet,
entry.getKey().getName(), queueName, createInputDatumDecoder(dataType, schema, schemaCache));
ConsumerSupplier<QueueConsumer> consumerSupplier = ConsumerSupplier.create(flowletContext.getOwners(), runtimeUsageRegistry, dataFabricFacade, queueName, consumerConfig, numGroups);
queueConsumerSupplierBuilder.add(consumerSupplier);
queueReaders.add(queueReaderFactory.createQueueReader(consumerSupplier, batchSize, decoder));
}
}
}
}
// If inputs is needed but there is no available input queue, return null
if (!inputNames.isEmpty() && queueReaders.isEmpty()) {
return null;
}
return new ProcessSpecification<>(new RoundRobinQueueReader<>(queueReaders), method, tickAnnotation);
}
};
}
use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.
the class FlowProgramRunner method run.
@Override
public ProgramController run(Program program, ProgramOptions options) {
// Extract and verify parameters
ApplicationSpecification appSpec = program.getApplicationSpecification();
Preconditions.checkNotNull(appSpec, "Missing application specification.");
ProgramType processorType = program.getType();
Preconditions.checkNotNull(processorType, "Missing processor type.");
Preconditions.checkArgument(processorType == ProgramType.FLOW, "Only FLOW process type is supported.");
FlowSpecification flowSpec = appSpec.getFlows().get(program.getName());
Preconditions.checkNotNull(flowSpec, "Missing FlowSpecification for %s", program.getName());
try {
// Launch flowlet program runners
RunId runId = ProgramRunners.getRunId(options);
Multimap<String, QueueName> consumerQueues = FlowUtils.configureQueue(program, flowSpec, streamAdmin, queueAdmin, txExecutorFactory);
final Table<String, Integer, ProgramController> flowlets = createFlowlets(program, options, flowSpec);
return new FlowProgramController(flowlets, program, options, flowSpec, consumerQueues);
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.
the class OpenCloseDataSetTest method testDataSetsAreClosed.
@Test(timeout = 120000)
public void testDataSetsAreClosed() throws Exception {
final String tableName = "foo";
TrackingTable.resetTracker();
ApplicationWithPrograms app = AppFabricTestHelper.deployApplicationWithManager(DummyAppWithTrackingTable.class, TEMP_FOLDER_SUPPLIER);
List<ProgramController> controllers = Lists.newArrayList();
// start the programs
for (ProgramDescriptor programDescriptor : app.getPrograms()) {
if (programDescriptor.getProgramId().getType().equals(ProgramType.MAPREDUCE)) {
continue;
}
controllers.add(AppFabricTestHelper.submit(app, programDescriptor.getSpecification().getClassName(), new BasicArguments(), TEMP_FOLDER_SUPPLIER));
}
// write some data to queue
TransactionSystemClient txSystemClient = AppFabricTestHelper.getInjector().getInstance(TransactionSystemClient.class);
QueueName queueName = QueueName.fromStream(app.getApplicationId().getNamespace(), "xx");
QueueClientFactory queueClientFactory = AppFabricTestHelper.getInjector().getInstance(QueueClientFactory.class);
QueueProducer producer = queueClientFactory.createProducer(queueName);
// start tx to write in queue in tx
Transaction tx = txSystemClient.startShort();
((TransactionAware) producer).startTx(tx);
StreamEventCodec codec = new StreamEventCodec();
for (int i = 0; i < 4; i++) {
String msg = "x" + i;
StreamEvent event = new StreamEvent(ImmutableMap.<String, String>of(), ByteBuffer.wrap(msg.getBytes(Charsets.UTF_8)));
producer.enqueue(new QueueEntry(codec.encodePayload(event)));
}
// commit tx
((TransactionAware) producer).commitTx();
txSystemClient.commit(tx);
while (TrackingTable.getTracker(tableName, "write") < 4) {
TimeUnit.MILLISECONDS.sleep(50);
}
// get the number of writes to the foo table
Assert.assertEquals(4, TrackingTable.getTracker(tableName, "write"));
// only 2 "open" calls should be tracked:
// 1. the flow has started with single flowlet (service is loaded lazily on 1st request)
// 2. DatasetSystemMetadataWriter also instantiates the dataset because it needs to add some system tags
// for the dataset
Assert.assertEquals(2, TrackingTable.getTracker(tableName, "open"));
// now send a request to the service
Gson gson = new Gson();
DiscoveryServiceClient discoveryServiceClient = AppFabricTestHelper.getInjector().getInstance(DiscoveryServiceClient.class);
Discoverable discoverable = new RandomEndpointStrategy(discoveryServiceClient.discover(String.format("service.%s.%s.%s", DefaultId.NAMESPACE.getEntityName(), "dummy", "DummyService"))).pick(5, TimeUnit.SECONDS);
Assert.assertNotNull(discoverable);
HttpClient client = new DefaultHttpClient();
HttpGet get = new HttpGet(String.format("http://%s:%d/v3/namespaces/default/apps/%s/services/%s/methods/%s", discoverable.getSocketAddress().getHostName(), discoverable.getSocketAddress().getPort(), "dummy", "DummyService", "x1"));
HttpResponse response = client.execute(get);
String responseContent = gson.fromJson(new InputStreamReader(response.getEntity().getContent(), Charsets.UTF_8), String.class);
client.getConnectionManager().shutdown();
Assert.assertEquals("x1", responseContent);
// now the dataset must have a read and another open operation
Assert.assertEquals(1, TrackingTable.getTracker(tableName, "read"));
Assert.assertEquals(3, TrackingTable.getTracker(tableName, "open"));
// The dataset that was instantiated by the DatasetSystemMetadataWriter should have been closed
Assert.assertEquals(1, TrackingTable.getTracker(tableName, "close"));
// stop all programs, they should both close the data set foo
for (ProgramController controller : controllers) {
controller.stop().get();
}
int timesOpened = TrackingTable.getTracker(tableName, "open");
Assert.assertTrue(timesOpened >= 2);
Assert.assertEquals(timesOpened, TrackingTable.getTracker(tableName, "close"));
// now start the m/r job
ProgramController controller = null;
for (ProgramDescriptor programDescriptor : app.getPrograms()) {
if (programDescriptor.getProgramId().getType().equals(ProgramType.MAPREDUCE)) {
controller = AppFabricTestHelper.submit(app, programDescriptor.getSpecification().getClassName(), new BasicArguments(), TEMP_FOLDER_SUPPLIER);
}
}
Assert.assertNotNull(controller);
while (!controller.getState().equals(ProgramController.State.COMPLETED)) {
TimeUnit.MILLISECONDS.sleep(100);
}
// M/r job is done, one mapper and the m/r client should have opened and closed the data set foo
// we don't know the exact number of times opened, but it is at least once, and it must be closed the same number
// of times.
Assert.assertTrue(timesOpened < TrackingTable.getTracker(tableName, "open"));
Assert.assertEquals(TrackingTable.getTracker(tableName, "open"), TrackingTable.getTracker(tableName, "close"));
Assert.assertTrue(0 < TrackingTable.getTracker("bar", "open"));
Assert.assertEquals(TrackingTable.getTracker("bar", "open"), TrackingTable.getTracker("bar", "close"));
}
use of co.cask.cdap.common.queue.QueueName in project cdap by caskdata.
the class FlowTest method testFlow.
@Test
public void testFlow() throws Exception {
final ApplicationWithPrograms app = AppFabricTestHelper.deployApplicationWithManager(WordCountApp.class, TEMP_FOLDER_SUPPLIER);
List<ProgramController> controllers = Lists.newArrayList();
for (ProgramDescriptor programDescriptor : app.getPrograms()) {
// running mapreduce is out of scope of this tests (there's separate unit-test for that)
if (programDescriptor.getProgramId().getType() == ProgramType.MAPREDUCE) {
continue;
}
controllers.add(AppFabricTestHelper.submit(app, programDescriptor.getSpecification().getClassName(), new BasicArguments(), TEMP_FOLDER_SUPPLIER));
}
TimeUnit.SECONDS.sleep(1);
TransactionSystemClient txSystemClient = AppFabricTestHelper.getInjector().getInstance(TransactionSystemClient.class);
QueueName queueName = QueueName.fromStream(app.getApplicationId().getNamespace(), "text");
QueueClientFactory queueClientFactory = AppFabricTestHelper.getInjector().getInstance(QueueClientFactory.class);
QueueProducer producer = queueClientFactory.createProducer(queueName);
// start tx to write in queue in tx
Transaction tx = txSystemClient.startShort();
((TransactionAware) producer).startTx(tx);
StreamEventCodec codec = new StreamEventCodec();
for (int i = 0; i < 10; i++) {
String msg = "Testing message " + i;
StreamEvent event = new StreamEvent(ImmutableMap.<String, String>of(), ByteBuffer.wrap(msg.getBytes(Charsets.UTF_8)));
producer.enqueue(new QueueEntry(codec.encodePayload(event)));
}
// commit tx
((TransactionAware) producer).commitTx();
txSystemClient.commit(tx);
// Query the service for at most 10 seconds for the expected result
Gson gson = new Gson();
DiscoveryServiceClient discoveryServiceClient = AppFabricTestHelper.getInjector().getInstance(DiscoveryServiceClient.class);
ServiceDiscovered serviceDiscovered = discoveryServiceClient.discover(String.format("service.%s.%s.%s", DefaultId.NAMESPACE.getNamespace(), "WordCountApp", "WordFrequencyService"));
EndpointStrategy endpointStrategy = new RandomEndpointStrategy(serviceDiscovered);
int trials = 0;
while (trials++ < 10) {
Discoverable discoverable = endpointStrategy.pick(2, TimeUnit.SECONDS);
URL url = new URL(String.format("http://%s:%d/v3/namespaces/default/apps/%s/services/%s/methods/%s/%s", discoverable.getSocketAddress().getHostName(), discoverable.getSocketAddress().getPort(), "WordCountApp", "WordFrequencyService", "wordfreq", "text:Testing"));
try {
HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
Map<String, Long> responseContent = gson.fromJson(new InputStreamReader(urlConn.getInputStream(), Charsets.UTF_8), new TypeToken<Map<String, Long>>() {
}.getType());
LOG.info("Service response: " + responseContent);
if (ImmutableMap.of("text:Testing", 10L).equals(responseContent)) {
break;
}
} catch (Throwable t) {
LOG.info("Exception when trying to query service.", t);
}
TimeUnit.SECONDS.sleep(1);
}
Assert.assertTrue(trials < 10);
for (ProgramController controller : controllers) {
controller.stop().get();
}
}
Aggregations