Examples with LookupConfig io.cdap.cdap.etl.api.LookupConfig used on opensource projects

Example 1 with LookupConfig

use of io.cdap.cdap.etl.api.LookupConfig in project hydrator-plugins by cdapio.

the class JavaScriptTransformTest method testLookup.

@Test
public void testLookup() throws Exception {
    JavaScriptTransform.Config config = new JavaScriptTransform.Config("function transform(x, emitter, ctx) { " + "var single = ctx.getLookup('purchases').lookup('abc');" + "var batch = ctx.getLookup('purchases').lookup(['abc', 'sdf']);" + "x.stringField = '1_' + single + ' 2_' + batch['abc'] + batch['sdf'] + '::' + batch.abc;" + "emitter.emit(x);" + "}", null, new LookupConfig(ImmutableMap.of("purchases", new LookupTableConfig(LookupTableConfig.TableType.DATASET))));
    Transform<StructuredRecord, StructuredRecord> transform = new JavaScriptTransform(config);
    transform.initialize(new MockTransformContext("somestage", new HashMap<String, String>(), new MockLookupProvider(TEST_LOOKUP)));
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    transform.transform(STRING_RECORD, emitter);
    StructuredRecord output = emitter.getEmitted().get(0);
    // check record1
    Assert.assertEquals(STRING_SCHEMA, output.getSchema());
    Assert.assertEquals("1_abc 2_abcsdf::abc", output.get("stringField"));
}

Example 2 with LookupConfig

use of io.cdap.cdap.etl.api.LookupConfig in project hydrator-plugins by cdapio.

the class JavaScriptTransform method init.

private void init(@Nullable TransformContext context, FailureCollector collector) {
    ScriptEngineManager manager = new ScriptEngineManager();
    engine = manager.getEngineByName("JavaScript");
    try {
        engine.eval(ScriptConstants.HELPER_DEFINITION);
    } catch (ScriptException e) {
        // shouldn't happen
        collector.addFailure("Failed to define helper functions.", null);
        throw collector.getOrThrowException();
    }
    JavaTypeConverters js = ((Invocable) engine).getInterface(engine.get(ScriptConstants.HELPER_NAME), JavaTypeConverters.class);
    LookupConfig lookupConfig;
    try {
        lookupConfig = GSON.fromJson(config.lookup, LookupConfig.class);
    } catch (JsonSyntaxException e) {
        collector.addFailure("Invalid lookup config.", "Expected JSON map of string to string.").withConfigProperty(Config.LOOKUP);
        throw collector.getOrThrowException();
    }
    Arguments arguments = context == null ? null : context.getArguments();
    engine.put(CONTEXT_NAME, new ScriptContext(LOG, metrics, context, lookupConfig, js, arguments));
    try {
        // this is pretty ugly, but doing this so that we can pass the 'input' json into the transform function.
        // that is, we want people to implement
        // function transform(input) { ... }
        // rather than function transform() { ... } and have them access a global variable in the function
        String script = String.format("function %s() { return transform(%s, %s, %s); }\n%s", FUNCTION_NAME, VARIABLE_NAME, EMITTER_NAME, CONTEXT_NAME, config.script);
        engine.eval(script);
    } catch (ScriptException e) {
        collector.addFailure(String.format("Invalid script: %s.", e.getMessage()), null).withConfigProperty(Config.SCRIPT);
    }
    invocable = (Invocable) engine;
    if (config.schema != null) {
        try {
            schema = Schema.parseJson(config.schema);
        } catch (IOException e) {
            collector.addFailure(String.format("Invalid schema: %s.", e.getMessage()), "Output schema must be JSON parseable.").withConfigProperty(Config.SCHEMA);
        }
    }
    collector.getOrThrowException();
}