Search in sources :

Example 1 with BroadcastHashJoinTranslator

use of org.apache.beam.sdk.extensions.euphoria.core.translate.BroadcastHashJoinTranslator in project beam by apache.

the class DocumentationExamplesTest method testGenericTranslatorProvider.

@Test
public void testGenericTranslatorProvider() {
    GenericTranslatorProvider provider = GenericTranslatorProvider.newBuilder().register(FlatMap.class, // register by operator class
    new FlatMapTranslator<>()).register(Join.class, (Join op) -> {
        String name = ((Optional<String>) op.getName()).orElse("");
        return name.toLowerCase().startsWith("broadcast");
    }, // register by class and predicate
    new BroadcastHashJoinTranslator<>()).register(op -> op instanceof CompositeOperator, // register by predicate only
    new CompositeOperatorTranslator<>()).build();
    Assert.assertNotNull(provider);
}
Also used : Filter(org.apache.beam.sdk.extensions.euphoria.core.client.operator.Filter) Arrays(java.util.Arrays) Distinct(org.apache.beam.sdk.extensions.euphoria.core.client.operator.Distinct) TimestampCombiner(org.apache.beam.sdk.transforms.windowing.TimestampCombiner) ReduceWindow(org.apache.beam.sdk.extensions.euphoria.core.client.operator.ReduceWindow) SumByKey(org.apache.beam.sdk.extensions.euphoria.core.client.operator.SumByKey) Join(org.apache.beam.sdk.extensions.euphoria.core.client.operator.Join) Union(org.apache.beam.sdk.extensions.euphoria.core.client.operator.Union) GenericTranslatorProvider(org.apache.beam.sdk.extensions.euphoria.core.translate.provider.GenericTranslatorProvider) RightJoin(org.apache.beam.sdk.extensions.euphoria.core.client.operator.RightJoin) Create(org.apache.beam.sdk.transforms.Create) Arrays.asList(java.util.Arrays.asList) LeftJoin(org.apache.beam.sdk.extensions.euphoria.core.client.operator.LeftJoin) ReduceByKey(org.apache.beam.sdk.extensions.euphoria.core.client.operator.ReduceByKey) KryoOptions(org.apache.beam.sdk.extensions.kryo.KryoOptions) Fold(org.apache.beam.sdk.extensions.euphoria.core.client.util.Fold) Triple(org.apache.beam.sdk.extensions.euphoria.core.client.util.Triple) CompositeProvider(org.apache.beam.sdk.extensions.euphoria.core.translate.provider.CompositeProvider) CountByKey(org.apache.beam.sdk.extensions.euphoria.core.client.operator.CountByKey) Serializable(java.io.Serializable) List(java.util.List) Stream(java.util.stream.Stream) DefaultTrigger(org.apache.beam.sdk.transforms.windowing.DefaultTrigger) TypeDescriptors(org.apache.beam.sdk.values.TypeDescriptors) AssignEventTime(org.apache.beam.sdk.extensions.euphoria.core.client.operator.AssignEventTime) Optional(java.util.Optional) BroadcastHashJoinTranslator(org.apache.beam.sdk.extensions.euphoria.core.translate.BroadcastHashJoinTranslator) Collector(org.apache.beam.sdk.extensions.euphoria.core.client.io.Collector) KV(org.apache.beam.sdk.values.KV) TypeDescriptor(org.apache.beam.sdk.values.TypeDescriptor) FullJoin(org.apache.beam.sdk.extensions.euphoria.core.client.operator.FullJoin) Duration(org.joda.time.Duration) RunWith(org.junit.runner.RunWith) Splitter(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter) MapElements(org.apache.beam.sdk.extensions.euphoria.core.client.operator.MapElements) PipelineOptionsFactory(org.apache.beam.sdk.options.PipelineOptionsFactory) TestPipeline(org.apache.beam.sdk.testing.TestPipeline) Pipeline(org.apache.beam.sdk.Pipeline) PipelineOptions(org.apache.beam.sdk.options.PipelineOptions) Before(org.junit.Before) PAssert(org.apache.beam.sdk.testing.PAssert) FlatMapTranslator(org.apache.beam.sdk.extensions.euphoria.core.translate.FlatMapTranslator) FixedWindows(org.apache.beam.sdk.transforms.windowing.FixedWindows) Test(org.junit.Test) JUnit4(org.junit.runners.JUnit4) UnaryFunction(org.apache.beam.sdk.extensions.euphoria.core.client.functional.UnaryFunction) Operator(org.apache.beam.sdk.extensions.euphoria.core.client.operator.base.Operator) PCollection(org.apache.beam.sdk.values.PCollection) TopPerKey(org.apache.beam.sdk.extensions.euphoria.core.client.operator.TopPerKey) OperatorTranslator(org.apache.beam.sdk.extensions.euphoria.core.translate.OperatorTranslator) CompositeOperator(org.apache.beam.sdk.extensions.euphoria.core.client.operator.CompositeOperator) Rule(org.junit.Rule) Ignore(org.junit.Ignore) CompositeOperatorTranslator(org.apache.beam.sdk.extensions.euphoria.core.translate.CompositeOperatorTranslator) TranslatorProvider(org.apache.beam.sdk.extensions.euphoria.core.translate.TranslatorProvider) OnTimeBehavior(org.apache.beam.sdk.transforms.windowing.Window.OnTimeBehavior) FlatMap(org.apache.beam.sdk.extensions.euphoria.core.client.operator.FlatMap) KryoCoderProvider(org.apache.beam.sdk.extensions.kryo.KryoCoderProvider) Assert(org.junit.Assert) TextIO(org.apache.beam.sdk.io.TextIO) Optional(java.util.Optional) CompositeOperator(org.apache.beam.sdk.extensions.euphoria.core.client.operator.CompositeOperator) CompositeOperatorTranslator(org.apache.beam.sdk.extensions.euphoria.core.translate.CompositeOperatorTranslator) FlatMapTranslator(org.apache.beam.sdk.extensions.euphoria.core.translate.FlatMapTranslator) Join(org.apache.beam.sdk.extensions.euphoria.core.client.operator.Join) RightJoin(org.apache.beam.sdk.extensions.euphoria.core.client.operator.RightJoin) LeftJoin(org.apache.beam.sdk.extensions.euphoria.core.client.operator.LeftJoin) FullJoin(org.apache.beam.sdk.extensions.euphoria.core.client.operator.FullJoin) GenericTranslatorProvider(org.apache.beam.sdk.extensions.euphoria.core.translate.provider.GenericTranslatorProvider) FlatMap(org.apache.beam.sdk.extensions.euphoria.core.client.operator.FlatMap) Test(org.junit.Test)

Aggregations

Serializable (java.io.Serializable)1 Arrays (java.util.Arrays)1 Arrays.asList (java.util.Arrays.asList)1 List (java.util.List)1 Optional (java.util.Optional)1 Stream (java.util.stream.Stream)1 Pipeline (org.apache.beam.sdk.Pipeline)1 UnaryFunction (org.apache.beam.sdk.extensions.euphoria.core.client.functional.UnaryFunction)1 Collector (org.apache.beam.sdk.extensions.euphoria.core.client.io.Collector)1 AssignEventTime (org.apache.beam.sdk.extensions.euphoria.core.client.operator.AssignEventTime)1 CompositeOperator (org.apache.beam.sdk.extensions.euphoria.core.client.operator.CompositeOperator)1 CountByKey (org.apache.beam.sdk.extensions.euphoria.core.client.operator.CountByKey)1 Distinct (org.apache.beam.sdk.extensions.euphoria.core.client.operator.Distinct)1 Filter (org.apache.beam.sdk.extensions.euphoria.core.client.operator.Filter)1 FlatMap (org.apache.beam.sdk.extensions.euphoria.core.client.operator.FlatMap)1 FullJoin (org.apache.beam.sdk.extensions.euphoria.core.client.operator.FullJoin)1 Join (org.apache.beam.sdk.extensions.euphoria.core.client.operator.Join)1 LeftJoin (org.apache.beam.sdk.extensions.euphoria.core.client.operator.LeftJoin)1 MapElements (org.apache.beam.sdk.extensions.euphoria.core.client.operator.MapElements)1 ReduceByKey (org.apache.beam.sdk.extensions.euphoria.core.client.operator.ReduceByKey)1