Examples with DataSource com.google.cloud.bigquery.datatransfer.v1.DataSource used on opensource projects

Example 21 with DataSource

use of com.google.cloud.bigquery.datatransfer.v1.DataSource in project atlasmap by atlasmap.

the class XmlFieldReaderTest method mockDataSources.

private void mockDataSources(String docId, AtlasInternalSession session) {
    AtlasMapping atlasMapping = mock(AtlasMapping.class);
    List<DataSource> dataSources = new ArrayList<>();
    XmlDataSource xmlDataSource = new XmlDataSource();
    xmlDataSource.setId(docId);
    xmlDataSource.setDataSourceType(DataSourceType.SOURCE);
    XmlNamespaces atlasNamespaces = new XmlNamespaces();
    XmlNamespace xmlNamespaceQ = new XmlNamespace();
    xmlNamespaceQ.setAlias("q");
    xmlNamespaceQ.setUri("http://www.example.com/q/");
    XmlNamespace xmlNamespaceX = new XmlNamespace();
    xmlNamespaceX.setAlias("");
    xmlNamespaceX.setUri("http://www.example.com/x/");
    XmlNamespace xmlNamespaceY = new XmlNamespace();
    xmlNamespaceY.setAlias("y");
    xmlNamespaceY.setUri("http://www.example.com/y/");
    atlasNamespaces.getXmlNamespace().add(xmlNamespaceQ);
    atlasNamespaces.getXmlNamespace().add(xmlNamespaceX);
    atlasNamespaces.getXmlNamespace().add(xmlNamespaceY);
    xmlDataSource.setXmlNamespaces(atlasNamespaces);
    dataSources.add(xmlDataSource);
    when(atlasMapping.getDataSource()).thenReturn(dataSources);
    when(session.getMapping()).thenReturn(atlasMapping);
}

Example 22 with DataSource

use of com.google.cloud.bigquery.datatransfer.v1.DataSource in project syndesis-qe by syndesisio.

the class AtlasMapperGenerator method processSources.

/**
 * Gets list of output data shapes for preceding steps.
 *
 * @param precedingSteps
 * @return
 */
private List<DataSource> processSources(List<StepDefinition> precedingSteps) {
    List<DataSource> sources = new ArrayList<>();
    for (StepDefinition s : precedingSteps) {
        DataShape outDataShape = s.getConnectorDescriptor().get().getOutputDataShape().get();
        sources.add(createDataSource(outDataShape, s, DataSourceType.SOURCE));
    }
    return sources;
}

Example 23 with DataSource

use of com.google.cloud.bigquery.datatransfer.v1.DataSource in project syndesis-qe by syndesisio.

the class AtlasMapperGenerator method createDataSource.

/**
 * Used to generate data source elements for atlasMapping. There are three types of them: Java, Json, XML.
 *
 * TODO(tplevko): update also for XML
 *
 * @param dataShape
 * @param step
 * @param dataSourceType
 * @return
 */
private DataSource createDataSource(DataShape dataShape, StepDefinition step, DataSourceType dataSourceType) {
    DataShapeKinds dataShapeKind = dataShape.getKind();
    DataSource source = null;
    if (dataShapeKind.toString().contains("json")) {
        source = new JsonDataSource();
        source.setUri("atlas:" + "json:" + step.getStep().getId().get());
    } else if (dataShapeKind.toString().contains("java")) {
        source = new DataSource();
        source.setUri("atlas:" + "java:" + step.getStep().getId().get() + "?className=" + dataShape.getType());
    } else if (dataShapeKind.toString().contains("xml")) {
        source = new XmlDataSource();
    // TODO(tplevko): find out how should look the XML datasource definition
    }
    source.setId(step.getStep().getId().get());
    source.setDataSourceType(dataSourceType);
    return source;
}

Example 24 with DataSource

use of com.google.cloud.bigquery.datatransfer.v1.DataSource in project atlasmap by atlasmap.

the class ConcurrencyChaosMonkeyTest method generateMappingURI.

protected URI generateMappingURI() throws Exception {
    AtlasMapping mapping = AtlasModelFactory.createAtlasMapping();
    mapping.setName("mockMapping");
    DataSource src = new DataSource();
    src.setDataSourceType(DataSourceType.SOURCE);
    src.setUri("atlas:java?className=twitter4j.Status");
    DataSource tgt = new DataSource();
    tgt.setDataSourceType(DataSourceType.TARGET);
    tgt.setUri("atlas:java?className=org.apache.camel.salesforce.dto.Contact");
    mapping.getDataSource().add(src);
    mapping.getDataSource().add(tgt);
    Mapping sepMapping = AtlasModelFactory.createMapping(MappingType.SEPARATE);
    JavaField jNameField = new JavaField();
    jNameField.setName("Name");
    jNameField.setPath("User.name");
    jNameField.setGetMethod("getName");
    jNameField.setFieldType(FieldType.STRING);
    JavaField jFirstNameField = new JavaField();
    jFirstNameField.setName("FirstName");
    jFirstNameField.setPath("FirstName");
    jFirstNameField.setSetMethod("setFirstName");
    jFirstNameField.setFieldType(FieldType.STRING);
    jFirstNameField.setIndex(0);
    JavaField jLastNameField = new JavaField();
    jLastNameField.setName("LastName");
    jLastNameField.setPath("LastName");
    jLastNameField.setSetMethod("setLastName");
    jLastNameField.setFieldType(FieldType.STRING);
    jLastNameField.setIndex(1);
    sepMapping.getInputField().add(jNameField);
    sepMapping.getOutputField().add(jFirstNameField);
    sepMapping.getOutputField().add(jLastNameField);
    mapping.getMappings().getMapping().add(sepMapping);
    Mapping textDescMapping = AtlasModelFactory.createMapping(MappingType.MAP);
    JavaField jTextField = new JavaField();
    jTextField.setName("Text");
    jTextField.setPath("Text");
    jTextField.setGetMethod("getText");
    jTextField.setFieldType(FieldType.STRING);
    JavaField jDescField = new JavaField();
    jDescField.setName("Description");
    jDescField.setPath("description");
    jDescField.setSetMethod("setDescription");
    jDescField.setFieldType(FieldType.STRING);
    textDescMapping.getInputField().add(jTextField);
    textDescMapping.getOutputField().add(jDescField);
    mapping.getMappings().getMapping().add(textDescMapping);
    Mapping screenTitleMapping = AtlasModelFactory.createMapping(MappingType.MAP);
    JavaField jScreenField = new JavaField();
    jScreenField.setName("ScreenName");
    jScreenField.setPath("User.screenName");
    jScreenField.setGetMethod("getScreenName");
    jScreenField.setFieldType(FieldType.STRING);
    JavaField jTitleField = new JavaField();
    jTitleField.setName("Title");
    jTitleField.setPath("Title");
    jTitleField.setSetMethod("setTitle");
    jTitleField.setFieldType(FieldType.STRING);
    screenTitleMapping.getInputField().add(jScreenField);
    screenTitleMapping.getOutputField().add(jTitleField);
    mapping.getMappings().getMapping().add(screenTitleMapping);
    File mappingFile = new File("target/junit-atlasmapping.xml");
    atlasContextFactory.getMappingService().saveMappingAsFile(mapping, mappingFile);
    return mappingFile.toURI();
}

Example 25 with DataSource

use of com.google.cloud.bigquery.datatransfer.v1.DataSource in project atlasmap by atlasmap.

the class BaseModuleValidationService method validateMapping.

@Override
public List<Validation> validateMapping(AtlasMapping mapping) {
    List<Validation> validations = new ArrayList<>();
    if (getMode() == AtlasModuleMode.UNSET) {
        Validation validation = new Validation();
        validation.setMessage(String.format("No mode specified for %s/%s, skipping module validations", this.getModuleDetail().name(), this.getClass().getSimpleName()));
    }
    if (mapping != null && mapping.getMappings() != null && mapping.getMappings().getMapping() != null && !mapping.getMappings().getMapping().isEmpty()) {
        validateMappingEntries(mapping.getMappings().getMapping(), validations);
    }
    boolean found = false;
    for (DataSource ds : mapping.getDataSource()) {
        if (ds.getUri() != null && ds.getUri().startsWith(getModuleDetail().uri())) {
            found = true;
            break;
        }
    }
    if (!found) {
        Validation validation = new Validation();
        validation.setScope(ValidationScope.DATA_SOURCE);
        validation.setMessage(String.format("No DataSource with '%s' uri specified", getModuleDetail().uri()));
        validation.setStatus(ValidationStatus.ERROR);
        validations.add(validation);
    }
    return validations;
}