Examples with GenericSystemDescriptor org.apache.samza.system.descriptors.GenericSystemDescriptor used on opensource projects

Example 31 with GenericSystemDescriptor

use of org.apache.samza.system.descriptors.GenericSystemDescriptor in project samza by apache.

the class TestStreamApplicationDescriptorImpl method testGetOutputStreamWithNullSerde.

@Test(expected = IllegalArgumentException.class)
public void testGetOutputStreamWithNullSerde() {
    String streamId = "test-stream-1";
    GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
    GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, null);
    new StreamApplicationDescriptorImpl(appDesc -> {
        appDesc.getOutputStream(osd);
    }, getConfig());
}

Example 32 with GenericSystemDescriptor

use of org.apache.samza.system.descriptors.GenericSystemDescriptor in project beam by apache.

the class ImpulseTranslator method translate.

@Override
public void translate(PTransform<PBegin, PCollection<byte[]>> transform, Node node, TranslationContext ctx) {
    final PCollection<byte[]> output = ctx.getOutput(transform);
    final String outputId = ctx.getIdForPValue(output);
    final GenericSystemDescriptor systemDescriptor = new GenericSystemDescriptor(outputId, SamzaImpulseSystemFactory.class.getName());
    // The KvCoder is needed here for Samza not to crop the key.
    final Serde<KV<?, OpMessage<byte[]>>> kvSerde = KVSerde.of(new NoOpSerde(), new NoOpSerde<>());
    final GenericInputDescriptor<KV<?, OpMessage<byte[]>>> inputDescriptor = systemDescriptor.getInputDescriptor(outputId, kvSerde);
    ctx.registerInputMessageStream(output, inputDescriptor);
}

Example 33 with GenericSystemDescriptor

use of org.apache.samza.system.descriptors.GenericSystemDescriptor in project beam by apache.

the class ImpulseTranslator method translatePortable.

@Override
public void translatePortable(PipelineNode.PTransformNode transform, QueryablePipeline pipeline, PortableTranslationContext ctx) {
    final String outputId = ctx.getOutputId(transform);
    final String escapedOutputId = SamzaPipelineTranslatorUtils.escape(outputId);
    final GenericSystemDescriptor systemDescriptor = new GenericSystemDescriptor(escapedOutputId, SamzaImpulseSystemFactory.class.getName());
    // The KvCoder is needed here for Samza not to crop the key.
    final Serde<KV<?, OpMessage<byte[]>>> kvSerde = KVSerde.of(new NoOpSerde(), new NoOpSerde<>());
    final GenericInputDescriptor<KV<?, OpMessage<byte[]>>> inputDescriptor = systemDescriptor.getInputDescriptor(escapedOutputId, kvSerde);
    ctx.registerInputMessageStream(outputId, inputDescriptor);
}

Example 34 with GenericSystemDescriptor

use of org.apache.samza.system.descriptors.GenericSystemDescriptor in project beam by apache.

the class ReadTranslator method translate.

@Override
public void translate(PTransform<PBegin, PCollection<T>> transform, TransformHierarchy.Node node, TranslationContext ctx) {
    final PCollection<T> output = ctx.getOutput(transform);
    final Coder<WindowedValue<T>> coder = SamzaCoders.of(output);
    final Source<?> source = transform instanceof SplittableParDo.PrimitiveBoundedRead ? ((SplittableParDo.PrimitiveBoundedRead) transform).getSource() : ((SplittableParDo.PrimitiveUnboundedRead) transform).getSource();
    final String id = ctx.getIdForPValue(output);
    // Create system descriptor
    final GenericSystemDescriptor systemDescriptor;
    if (source instanceof BoundedSource) {
        systemDescriptor = new GenericSystemDescriptor(id, BoundedSourceSystem.Factory.class.getName());
    } else {
        systemDescriptor = new GenericSystemDescriptor(id, UnboundedSourceSystem.Factory.class.getName());
    }
    final Map<String, String> systemConfig = ImmutableMap.of("source", Base64Serializer.serializeUnchecked(source), "coder", Base64Serializer.serializeUnchecked(coder), "stepName", node.getFullName());
    systemDescriptor.withSystemConfigs(systemConfig);
    // Create stream descriptor
    @SuppressWarnings("unchecked") final Serde<KV<?, OpMessage<T>>> kvSerde = (Serde) KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>());
    final GenericInputDescriptor<KV<?, OpMessage<T>>> inputDescriptor = systemDescriptor.getInputDescriptor(id, kvSerde);
    if (source instanceof BoundedSource) {
        inputDescriptor.isBounded();
    }
    ctx.registerInputMessageStream(output, inputDescriptor);
}

Example 35 with GenericSystemDescriptor

use of org.apache.samza.system.descriptors.GenericSystemDescriptor in project samza by apache.

the class TestJobGraphJsonGenerator method testRepartitionedWindowStreamApplication.

@Test
public void testRepartitionedWindowStreamApplication() throws Exception {
    Map<String, String> configMap = new HashMap<>();
    configMap.put(JobConfig.JOB_NAME, "test-app");
    configMap.put(JobConfig.JOB_DEFAULT_SYSTEM, "test-system");
    StreamTestUtils.addStreamConfigs(configMap, "PageView", "hdfs", "hdfs:/user/dummy/PageViewEvent");
    StreamTestUtils.addStreamConfigs(configMap, "PageViewCount", "kafka", "PageViewCount");
    Config config = new MapConfig(configMap);
    // set up external partition count
    Map<String, Integer> system1Map = new HashMap<>();
    system1Map.put("hdfs:/user/dummy/PageViewEvent", 512);
    Map<String, Integer> system2Map = new HashMap<>();
    system2Map.put("PageViewCount", 16);
    SystemAdmin systemAdmin1 = createSystemAdmin(system1Map);
    SystemAdmin systemAdmin2 = createSystemAdmin(system2Map);
    SystemAdmins systemAdmins = mock(SystemAdmins.class);
    when(systemAdmins.getSystemAdmin("hdfs")).thenReturn(systemAdmin1);
    when(systemAdmins.getSystemAdmin("kafka")).thenReturn(systemAdmin2);
    StreamManager streamManager = new StreamManager(systemAdmins);
    StreamApplicationDescriptorImpl graphSpec = new StreamApplicationDescriptorImpl(appDesc -> {
        KVSerde<String, PageViewEvent> pvSerde = KVSerde.of(new StringSerde(), new JsonSerdeV2<>(PageViewEvent.class));
        GenericSystemDescriptor isd = new GenericSystemDescriptor("hdfs", "mockSystemFactoryClass");
        GenericInputDescriptor<KV<String, PageViewEvent>> pageView = isd.getInputDescriptor("PageView", pvSerde);
        KVSerde<String, Long> pvcSerde = KVSerde.of(new StringSerde(), new LongSerde());
        GenericSystemDescriptor osd = new GenericSystemDescriptor("kafka", "mockSystemFactoryClass");
        GenericOutputDescriptor<KV<String, Long>> pageViewCount = osd.getOutputDescriptor("PageViewCount", pvcSerde);
        MessageStream<KV<String, PageViewEvent>> inputStream = appDesc.getInputStream(pageView);
        OutputStream<KV<String, Long>> outputStream = appDesc.getOutputStream(pageViewCount);
        inputStream.partitionBy(kv -> kv.getValue().getCountry(), kv -> kv.getValue(), pvSerde, "keyed-by-country").window(Windows.keyedTumblingWindow(kv -> kv.getValue().getCountry(), Duration.ofSeconds(10L), () -> 0L, (m, c) -> c + 1L, new StringSerde(), new LongSerde()), "count-by-country").map(pane -> new KV<>(pane.getKey().getKey(), pane.getMessage())).sendTo(outputStream);
    }, config);
    ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
    ExecutionPlan plan = planner.plan(graphSpec);
    String json = plan.getPlanAsJson();
    System.out.println(json);
    // deserialize
    ObjectMapper mapper = new ObjectMapper();
    JobGraphJsonGenerator.JobGraphJson nodes = mapper.readValue(json, JobGraphJsonGenerator.JobGraphJson.class);
    JobGraphJsonGenerator.OperatorGraphJson operatorGraphJson = nodes.jobs.get(0).operatorGraph;
    assertEquals(2, operatorGraphJson.inputStreams.size());
    assertEquals(4, operatorGraphJson.operators.size());
    assertEquals(1, nodes.sourceStreams.size());
    assertEquals(1, nodes.sinkStreams.size());
    assertEquals(1, nodes.intermediateStreams.size());
    // verify partitionBy op output to the intermdiate stream of the same id
    assertEquals(operatorGraphJson.operators.get("test-app-1-partition_by-keyed-by-country").get("outputStreamId"), "test-app-1-partition_by-keyed-by-country");
    assertEquals(operatorGraphJson.operators.get("test-app-1-send_to-5").get("outputStreamId"), "PageViewCount");
}