Examples with Schema co.cask.cdap.api.data.schema.Schema used on opensource projects

Example 76 with Schema

use of co.cask.cdap.api.data.schema.Schema in project cdap by caskdata.

the class PipelinePlannerTest method testGeneratePlan.

@Test
public void testGeneratePlan() {
    /*
             |--- n2(r) ----------|
             |                    |                                    |-- n10
        n1 --|--- n3(r) --- n5 ---|--- n6 --- n7(r) --- n8 --- n9(r) --|
             |                    |                                    |-- n11
             |--- n4(r) ----------|
     */
    // create the spec for this pipeline
    ArtifactId artifactId = new ArtifactId("dummy", new ArtifactVersion("1.0.0"), ArtifactScope.SYSTEM);
    Map<String, String> empty = ImmutableMap.of();
    PluginSpec nodePlugin = new PluginSpec(NODE, "mock", empty, artifactId);
    PluginSpec reducePlugin = new PluginSpec(AGGREGATOR, "mock", empty, artifactId);
    Schema schema = Schema.recordOf("stuff", Schema.Field.of("x", Schema.of(Schema.Type.INT)));
    Set<StageSpec> stageSpecs = ImmutableSet.of(StageSpec.builder("n1", nodePlugin).setOutputSchema(schema).addOutputs("n2", "n3", "n4").build(), StageSpec.builder("n2", reducePlugin).addInputSchema("n1", schema).setOutputSchema(schema).addInputs("n1").addOutputs("n6").build(), StageSpec.builder("n3", reducePlugin).addInputSchema("n1", schema).setOutputSchema(schema).addInputs("n1").addOutputs("n5").build(), StageSpec.builder("n4", reducePlugin).addInputSchema("n1", schema).setOutputSchema(schema).addInputs("n1").addOutputs("n6").build(), StageSpec.builder("n5", nodePlugin).addInputSchema("n3", schema).setOutputSchema(schema).addInputs("n3").addOutputs("n6").build(), StageSpec.builder("n6", nodePlugin).addInputSchemas(ImmutableMap.of("n2", schema, "n5", schema, "n4", schema)).setOutputSchema(schema).addInputs("n2", "n5", "n4").addOutputs("n7").build(), StageSpec.builder("n7", reducePlugin).addInputSchema("n6", schema).setOutputSchema(schema).addInputs("n6").addOutputs("n8").build(), StageSpec.builder("n8", nodePlugin).addInputSchema("n7", schema).setOutputSchema(schema).addInputs("n7").addOutputs("n9").build(), StageSpec.builder("n9", reducePlugin).addInputSchema("n8", schema).setOutputSchema(schema).addInputs("n8").addOutputs("n10", "n11").build(), StageSpec.builder("n10", nodePlugin).addInputSchema("n9", schema).addInputs("n9").build(), StageSpec.builder("n11", nodePlugin).addInputSchema("n9", schema).addInputs("n9").build());
    Set<Connection> connections = ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n2", "n6"), new Connection("n3", "n5"), new Connection("n4", "n6"), new Connection("n5", "n6"), new Connection("n6", "n7"), new Connection("n7", "n8"), new Connection("n8", "n9"), new Connection("n9", "n10"), new Connection("n9", "n11"));
    Set<String> pluginTypes = ImmutableSet.of(NODE, AGGREGATOR, Constants.CONNECTOR_TYPE);
    Set<String> reduceTypes = ImmutableSet.of(AGGREGATOR);
    Set<String> emptySet = ImmutableSet.of();
    PipelinePlanner planner = new PipelinePlanner(pluginTypes, reduceTypes, emptySet, emptySet);
    PipelineSpec pipelineSpec = PipelineSpec.builder().addStages(stageSpecs).addConnections(connections).build();
    Map<String, PipelinePhase> phases = new HashMap<>();
    /*
             |--- n2.connector
             |
        n1 --|--- n3.connector
             |
             |--- n4.connector
     */
    PipelinePhase phase1 = PipelinePhase.builder(pluginTypes).addStage(StageInfo.builder("n1", NODE).addOutputs("n2", "n3", "n4").setOutputSchema(schema).build()).addStage(StageInfo.builder("n2.connector", Constants.CONNECTOR_TYPE).build()).addStage(StageInfo.builder("n3.connector", Constants.CONNECTOR_TYPE).build()).addStage(StageInfo.builder("n4.connector", Constants.CONNECTOR_TYPE).build()).addConnections("n1", ImmutableSet.of("n2.connector", "n3.connector", "n4.connector")).build();
    String phase1Name = getPhaseName("n1", "n2.connector", "n3.connector", "n4.connector");
    phases.put(phase1Name, phase1);
    /*
        phase2:
        n2.connector --- n2(r) --- n6 --- n7.connector
     */
    PipelinePhase phase2 = PipelinePhase.builder(pluginTypes).addStage(StageInfo.builder("n2", AGGREGATOR).addInputs("n1").addInputSchema("n1", schema).addOutputs("n6").setOutputSchema(schema).build()).addStage(StageInfo.builder("n6", NODE).addInputs("n2", "n4", "n5").addInputSchema("n2", schema).addInputSchema("n4", schema).addInputSchema("n5", schema).addOutputs("n7").setOutputSchema(schema).build()).addStage(StageInfo.builder("n2.connector", Constants.CONNECTOR_TYPE).build()).addStage(StageInfo.builder("n7.connector", Constants.CONNECTOR_TYPE).build()).addConnection("n2.connector", "n2").addConnection("n2", "n6").addConnection("n6", "n7.connector").build();
    String phase2Name = getPhaseName("n2.connector", "n7.connector");
    phases.put(phase2Name, phase2);
    /*
        phase3:
        n3.connector --- n3(r) --- n5 --- n6 --- n7.connector
     */
    PipelinePhase phase3 = PipelinePhase.builder(pluginTypes).addStage(StageInfo.builder("n5", NODE).addInputs("n3").addInputSchema("n3", schema).addOutputs("n6").setOutputSchema(schema).build()).addStage(StageInfo.builder("n6", NODE).addInputs("n2", "n4", "n5").addInputSchema("n2", schema).addInputSchema("n4", schema).addInputSchema("n5", schema).addOutputs("n7").setOutputSchema(schema).build()).addStage(StageInfo.builder("n3", AGGREGATOR).addInputs("n1").addInputSchema("n1", schema).addOutputs("n5").setOutputSchema(schema).build()).addStage(StageInfo.builder("n3.connector", Constants.CONNECTOR_TYPE).build()).addStage(StageInfo.builder("n7.connector", Constants.CONNECTOR_TYPE).build()).addConnection("n3.connector", "n3").addConnection("n3", "n5").addConnection("n5", "n6").addConnection("n6", "n7.connector").build();
    String phase3Name = getPhaseName("n3.connector", "n7.connector");
    phases.put(phase3Name, phase3);
    /*
        phase4:
        n4.connector --- n4(r) --- n6 --- n7.connector
     */
    PipelinePhase phase4 = PipelinePhase.builder(pluginTypes).addStage(StageInfo.builder("n4", AGGREGATOR).addInputs("n1").addInputSchema("n1", schema).addOutputs("n6").setOutputSchema(schema).build()).addStage(StageInfo.builder("n6", NODE).addInputs("n2", "n4", "n5").addInputSchema("n2", schema).addInputSchema("n4", schema).addInputSchema("n5", schema).addOutputs("n7").setOutputSchema(schema).build()).addStage(StageInfo.builder("n4.connector", Constants.CONNECTOR_TYPE).build()).addStage(StageInfo.builder("n7.connector", Constants.CONNECTOR_TYPE).build()).addConnection("n4.connector", "n4").addConnection("n4", "n6").addConnection("n6", "n7.connector").build();
    String phase4Name = getPhaseName("n4.connector", "n7.connector");
    phases.put(phase4Name, phase4);
    /*
        phase5:
        n7.connector --- n7(r) --- n8 --- n9.connector
     */
    PipelinePhase phase5 = PipelinePhase.builder(pluginTypes).addStage(StageInfo.builder("n8", NODE).addInputs("n7").addInputSchema("n7", schema).addOutputs("n9").setOutputSchema(schema).build()).addStage(StageInfo.builder("n7", AGGREGATOR).addInputs("n6").addInputSchema("n6", schema).addOutputs("n8").setOutputSchema(schema).build()).addStage(StageInfo.builder("n7.connector", Constants.CONNECTOR_TYPE).build()).addStage(StageInfo.builder("n9.connector", Constants.CONNECTOR_TYPE).build()).addConnection("n7.connector", "n7").addConnection("n7", "n8").addConnection("n8", "n9.connector").build();
    String phase5Name = getPhaseName("n7.connector", "n9.connector");
    phases.put(phase5Name, phase5);
    /*
        phase6:
                                 |-- n10
        n9.connector --- n9(r) --|
                                 |-- n11
     */
    PipelinePhase phase6 = PipelinePhase.builder(pluginTypes).addStage(StageInfo.builder("n10", NODE).addInputs("n9").addInputSchema("n9", schema).build()).addStage(StageInfo.builder("n11", NODE).addInputs("n9").addInputSchema("n9", schema).build()).addStage(StageInfo.builder("n9", AGGREGATOR).addInputs("n8").addInputSchema("n8", schema).addOutputs("n10", "n11").setOutputSchema(schema).build()).addStage(StageInfo.builder("n9.connector", Constants.CONNECTOR_TYPE).build()).addConnection("n9.connector", "n9").addConnection("n9", "n10").addConnection("n9", "n11").build();
    String phase6Name = getPhaseName("n9.connector", "n10", "n11");
    phases.put(phase6Name, phase6);
    Set<Connection> phaseConnections = new HashSet<>();
    phaseConnections.add(new Connection(phase1Name, phase2Name));
    phaseConnections.add(new Connection(phase1Name, phase3Name));
    phaseConnections.add(new Connection(phase1Name, phase4Name));
    phaseConnections.add(new Connection(phase2Name, phase5Name));
    phaseConnections.add(new Connection(phase3Name, phase5Name));
    phaseConnections.add(new Connection(phase4Name, phase5Name));
    phaseConnections.add(new Connection(phase5Name, phase6Name));
    PipelinePlan expected = new PipelinePlan(phases, phaseConnections);
    PipelinePlan actual = planner.plan(pipelineSpec);
    Assert.assertEquals(expected, actual);
}

Example 77 with Schema

use of co.cask.cdap.api.data.schema.Schema in project cdap by caskdata.

the class AbstractSchemaGenerator method doGenerate.

/**
   * Actual schema generation. It recursively resolves container types.
   *
   * @param typeToken    Encapsulate the Java type for generating a {@link Schema}.
   * @param knownRecords Set of record names that has the schema already generated. It is used for
   *                     recursive class field references.
   * @param acceptRecursion Whether to tolerate type recursion. If false, will throw UnsupportedTypeException if
   *                        a recursive type is encountered.
   * @return A {@link Schema} representing the given java {@link Type}.
   * @throws UnsupportedTypeException Indicates schema generation is not support for the given java {@link Type}.
   */
@SuppressWarnings("unchecked")
protected final Schema doGenerate(TypeToken<?> typeToken, Set<String> knownRecords, boolean acceptRecursion) throws UnsupportedTypeException {
    Type type = typeToken.getType();
    Class<?> rawType = typeToken.getRawType();
    if (SIMPLE_SCHEMAS.containsKey(rawType)) {
        return SIMPLE_SCHEMAS.get(rawType);
    }
    // Enum type, simply use all the enum constants for ENUM schema.
    if (rawType.isEnum()) {
        return Schema.enumWith((Class<Enum<?>>) rawType);
    }
    // Java array, use ARRAY schema.
    if (rawType.isArray()) {
        Schema componentSchema = doGenerate(TypeToken.of(rawType.getComponentType()), knownRecords, acceptRecursion);
        if (rawType.getComponentType().isPrimitive()) {
            return Schema.arrayOf(componentSchema);
        }
        return Schema.arrayOf(Schema.unionOf(componentSchema, Schema.of(Schema.Type.NULL)));
    }
    if (!(type instanceof Class || type instanceof ParameterizedType)) {
        throw new UnsupportedTypeException("Type " + type + " is not supported. " + "Only Class or ParameterizedType are supported.");
    }
    // Any parameterized Collection class would be represented by ARRAY schema.
    if (Collection.class.isAssignableFrom(rawType)) {
        if (!(type instanceof ParameterizedType)) {
            throw new UnsupportedTypeException("Only supports parameterized Collection type.");
        }
        TypeToken<?> componentType = typeToken.resolveType(((ParameterizedType) type).getActualTypeArguments()[0]);
        Schema componentSchema = doGenerate(componentType, knownRecords, acceptRecursion);
        return Schema.arrayOf(Schema.unionOf(componentSchema, Schema.of(Schema.Type.NULL)));
    }
    // Java Map, use MAP schema.
    if (Map.class.isAssignableFrom(rawType)) {
        if (!(type instanceof ParameterizedType)) {
            throw new UnsupportedTypeException("Only supports parameterized Map type.");
        }
        Type[] typeArgs = ((ParameterizedType) type).getActualTypeArguments();
        TypeToken<?> keyType = typeToken.resolveType(typeArgs[0]);
        TypeToken<?> valueType = typeToken.resolveType(typeArgs[1]);
        Schema valueSchema = doGenerate(valueType, knownRecords, acceptRecursion);
        return Schema.mapOf(doGenerate(keyType, knownRecords, acceptRecursion), Schema.unionOf(valueSchema, Schema.of(Schema.Type.NULL)));
    }
    // Any Java class, class name as the record name.
    String recordName = typeToken.getRawType().getName();
    if (knownRecords.contains(recordName)) {
        // Record already seen before
        if (acceptRecursion) {
            // simply create a reference RECORD schema by the name.
            return Schema.recordOf(recordName);
        } else {
            throw new UnsupportedTypeException("Recursive type not supported for class " + recordName);
        }
    }
    // Delegate to child class to generate RECORD schema.
    return generateRecord(typeToken, knownRecords, acceptRecursion);
}

Example 78 with Schema

use of co.cask.cdap.api.data.schema.Schema in project cdap by caskdata.

the class ReflectionSchemaGenerator method generateRecord.

@Override
protected Schema generateRecord(TypeToken<?> typeToken, Set<String> knowRecords, boolean acceptRecursion) throws UnsupportedTypeException {
    String recordName = typeToken.getRawType().getName();
    Map<String, TypeToken<?>> recordFieldTypes = typeToken.getRawType().isInterface() ? collectByMethods(typeToken, new TreeMap<String, TypeToken<?>>()) : collectByFields(typeToken, new TreeMap<String, TypeToken<?>>());
    // Recursively generate field type schema.
    List<Schema.Field> fields = new ArrayList<>();
    for (Map.Entry<String, TypeToken<?>> fieldType : recordFieldTypes.entrySet()) {
        Set<String> records = new HashSet<>(knowRecords);
        records.add(recordName);
        Schema fieldSchema = doGenerate(fieldType.getValue(), records, acceptRecursion);
        if (!fieldType.getValue().getRawType().isPrimitive()) {
            boolean isNotNull = typeToken.getRawType().isAnnotationPresent(Nonnull.class);
            boolean isNull = typeToken.getRawType().isAnnotationPresent(Nullable.class);
            // ii) if it is not nullable by default and nullable annotation is present
            if ((isNullableByDefault && !isNotNull) || (!isNullableByDefault && isNull)) {
                fieldSchema = Schema.unionOf(fieldSchema, Schema.of(Schema.Type.NULL));
            }
        }
        fields.add(Schema.Field.of(fieldType.getKey(), fieldSchema));
    }
    return Schema.recordOf(recordName, Collections.unmodifiableList(fields));
}

Example 79 with Schema

use of co.cask.cdap.api.data.schema.Schema in project cdap by caskdata.

the class ArtifactInspector method inspectApplications.

private ArtifactClasses.Builder inspectApplications(Id.Artifact artifactId, ArtifactClasses.Builder builder, Location artifactLocation, ClassLoader artifactClassLoader) throws IOException, InvalidArtifactException {
    // right now we force users to include the application main class as an attribute in their manifest,
    // which forces them to have a single application class.
    // in the future, we may want to let users do this or maybe specify a list of classes or
    // a package that will be searched for applications, to allow multiple applications in a single artifact.
    String mainClassName;
    try {
        Manifest manifest = BundleJarUtil.getManifest(artifactLocation);
        if (manifest == null) {
            return builder;
        }
        Attributes manifestAttributes = manifest.getMainAttributes();
        if (manifestAttributes == null) {
            return builder;
        }
        mainClassName = manifestAttributes.getValue(ManifestFields.MAIN_CLASS);
    } catch (ZipException e) {
        throw new InvalidArtifactException(String.format("Couldn't unzip artifact %s, please check it is a valid jar file.", artifactId), e);
    }
    if (mainClassName == null) {
        return builder;
    }
    try {
        Object appMain = artifactClassLoader.loadClass(mainClassName).newInstance();
        if (!(appMain instanceof Application)) {
            // possible for 3rd party plugin artifacts to have the main class set
            return builder;
        }
        Application app = (Application) appMain;
        java.lang.reflect.Type configType;
        // we can deserialize the config into that object. Otherwise it'll just be a Config
        try {
            configType = Artifacts.getConfigType(app.getClass());
        } catch (Exception e) {
            throw new InvalidArtifactException(String.format("Could not resolve config type for Application class %s in artifact %s. " + "The type must extend Config and cannot be parameterized.", mainClassName, artifactId));
        }
        Schema configSchema = configType == Config.class ? null : schemaGenerator.generate(configType);
        builder.addApp(new ApplicationClass(mainClassName, "", configSchema));
    } catch (ClassNotFoundException e) {
        throw new InvalidArtifactException(String.format("Could not find Application main class %s in artifact %s.", mainClassName, artifactId));
    } catch (UnsupportedTypeException e) {
        throw new InvalidArtifactException(String.format("Config for Application %s in artifact %s has an unsupported schema. " + "The type must extend Config and cannot be parameterized.", mainClassName, artifactId));
    } catch (InstantiationException | IllegalAccessException e) {
        throw new InvalidArtifactException(String.format("Could not instantiate Application class %s in artifact %s.", mainClassName, artifactId), e);
    }
    return builder;
}

Example 80 with Schema

use of co.cask.cdap.api.data.schema.Schema in project cdap by caskdata.

the class DataPipelineTest method testTableLookup.

@Test
public void testTableLookup() throws Exception {
    addDatasetInstance(Table.class.getName(), "personTable");
    DataSetManager<Table> lookupTableManager = getDataset("personTable");
    Table lookupTable = lookupTableManager.get();
    lookupTable.put("samuel".getBytes(Charsets.UTF_8), "age".getBytes(Charsets.UTF_8), "12".getBytes(Charsets.UTF_8));
    lookupTable.put("samuel".getBytes(Charsets.UTF_8), "gender".getBytes(Charsets.UTF_8), "m".getBytes(Charsets.UTF_8));
    lookupTable.put("bob".getBytes(Charsets.UTF_8), "age".getBytes(Charsets.UTF_8), "36".getBytes(Charsets.UTF_8));
    lookupTable.put("bob".getBytes(Charsets.UTF_8), "gender".getBytes(Charsets.UTF_8), "m".getBytes(Charsets.UTF_8));
    lookupTable.put("jane".getBytes(Charsets.UTF_8), "age".getBytes(Charsets.UTF_8), "25".getBytes(Charsets.UTF_8));
    lookupTable.put("jane".getBytes(Charsets.UTF_8), "gender".getBytes(Charsets.UTF_8), "f".getBytes(Charsets.UTF_8));
    lookupTableManager.flush();
    ETLBatchConfig etlConfig = ETLBatchConfig.builder("* * * * *").addStage(new ETLStage("source", MockSource.getPlugin("inputTable"))).addStage(new ETLStage("transform", LookupTransform.getPlugin("person", "age", "personTable"))).addStage(new ETLStage("sink", MockSink.getPlugin("outputTable"))).addConnection("source", "transform").addConnection("transform", "sink").build();
    ApplicationId appId = NamespaceId.DEFAULT.app("testTableLookup");
    AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(APP_ARTIFACT, etlConfig);
    ApplicationManager appManager = deployApplication(appId, appRequest);
    // set up input data
    Schema inputSchema = Schema.recordOf("person", Schema.Field.of("person", Schema.of(Schema.Type.STRING)));
    StructuredRecord recordSamuel = StructuredRecord.builder(inputSchema).set("person", "samuel").build();
    StructuredRecord recordBob = StructuredRecord.builder(inputSchema).set("person", "bob").build();
    StructuredRecord recordJane = StructuredRecord.builder(inputSchema).set("person", "jane").build();
    DataSetManager<Table> inputTable = getDataset("inputTable");
    MockSource.writeInput(inputTable, ImmutableList.of(recordSamuel, recordBob, recordJane));
    WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME).start();
    workflowManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES);
    Schema schema = Schema.recordOf("person", Schema.Field.of("person", Schema.of(Schema.Type.STRING)), Schema.Field.of("age", Schema.of(Schema.Type.STRING)), Schema.Field.of("gender", Schema.of(Schema.Type.STRING)));
    Set<StructuredRecord> expected = new HashSet<>();
    expected.add(StructuredRecord.builder(schema).set("person", "samuel").set("age", "12").set("gender", "m").build());
    expected.add(StructuredRecord.builder(schema).set("person", "bob").set("age", "36").set("gender", "m").build());
    expected.add(StructuredRecord.builder(schema).set("person", "jane").set("age", "25").set("gender", "f").build());
    DataSetManager<Table> outputTable = getDataset("outputTable");
    Set<StructuredRecord> actual = new HashSet<>(MockSink.readOutput(outputTable));
    Assert.assertEquals(expected, actual);
    validateMetric(3, appId, "source.records.out");
    validateMetric(3, appId, "sink.records.in");
    deleteDatasetInstance(NamespaceId.DEFAULT.dataset("inputTable"));
    deleteDatasetInstance(NamespaceId.DEFAULT.dataset("outputTable"));
}