Examples with DataSource com.google.cloud.bigquery.datatransfer.v1.DataSource used on opensource projects

Example 6 with DataSource

use of com.google.cloud.bigquery.datatransfer.v1.DataSource in project java-docs-samples by GoogleCloudPlatform.

the class QuickstartSample method main.

/**
 * List available data sources for the BigQuery Data Transfer service.
 */
public static void main(String... args) throws Exception {
    // Sets your Google Cloud Platform project ID.
    // String projectId = "YOUR_PROJECT_ID";
    String projectId = args[0];
    // GOOGLE_APPLICATION_CREDENTIALS environment variable.
    try (DataTransferServiceClient client = DataTransferServiceClient.create()) {
        // Request the list of available data sources.
        String parent = String.format("projects/%s", projectId);
        ListDataSourcesRequest request = ListDataSourcesRequest.newBuilder().setParent(parent).build();
        ListDataSourcesPagedResponse response = client.listDataSources(request);
        // Print the results.
        System.out.println("Supported Data Sources:");
        for (DataSource dataSource : response.iterateAll()) {
            System.out.println(dataSource.getDisplayName());
            System.out.printf("\tID: %s%n", dataSource.getDataSourceId());
            System.out.printf("\tFull path: %s%n", dataSource.getName());
            System.out.printf("\tDescription: %s%n", dataSource.getDescription());
        }
    }
}

Example 7 with DataSource

use of com.google.cloud.bigquery.datatransfer.v1.DataSource in project syndesis-qe by syndesisio.

the class AtlasMapperGenerator method processSources.

/**
 * Gets list of output data shapes for preceding steps.
 *
 * @return list of datasources from preceding steps
 */
private List<DataSource> processSources() {
    List<DataSource> sources = new ArrayList<>();
    precedingSteps.stream().filter(s -> s.getStep().getAction().isPresent()).forEach(s -> {
        DataShape outDataShape = s.getStep().getAction().get().getOutputDataShape().get();
        // Steps with "ANY" or "NONE" are ignored for sources and only those that have proper datashape are used
        if (outDataShape.getKind() != DataShapeKinds.ANY && outDataShape.getKind() != DataShapeKinds.NONE) {
            sources.add(createDataSource(outDataShape, s, DataSourceType.SOURCE));
        }
    });
    return sources;
}

Example 8 with DataSource

use of com.google.cloud.bigquery.datatransfer.v1.DataSource in project atlasmap by atlasmap.

the class DefaultAtlasContextTest method testInit.

// (expected = AtlasException.class)
@Test
public void testInit() throws AtlasException {
    File file = Paths.get("src" + File.separator + "test" + File.separator + "resources" + File.separator + "atlasmapping.json").toFile();
    DefaultAtlasContext ctx = new DefaultAtlasContext(DefaultAtlasContextFactory.getInstance(), file.toURI());
    ctx.init();
    DataSource dataSource = new DataSource();
    dataSource.setUri("URI");
    mapping.getDataSource().add(dataSource);
    dataSource = new DataSource();
    dataSource.setUri(null);
    mapping.getDataSource().add(dataSource);
    dataSource = new DataSource();
    dataSource.setUri("java:source");
    dataSource.setDataSourceType(DataSourceType.SOURCE);
    dataSource.setId("io.atlasmap.core.DefaultAtlasContext.constants.docId");
    mapping.getDataSource().add(dataSource);
    dataSource = new DataSource();
    dataSource.setUri("java:target");
    dataSource.setDataSourceType(DataSourceType.TARGET);
    dataSource.setId("io.atlasmap.core.DefaultAtlasContext.constants.docId");
    mapping.getDataSource().add(dataSource);
    dataSource = new DataSource();
    dataSource.setUri("java:target");
    dataSource.setDataSourceType(DataSourceType.TARGET);
    dataSource.setId("io.atlasmap.core.DefaultAtlasContext.constants.docId");
    mapping.getDataSource().add(dataSource);
    ctx = new DefaultAtlasContext(DefaultAtlasContextFactory.getInstance(), mapping);
    ctx.getTargetModules().put("io.atlasmap.core.DefaultAtlasContext.constants.docId", new ConstantModule());
    ctx.init();
    @SuppressWarnings("unchecked") Map<String, AtlasModule> targetModules = spy(Map.class);
    when(targetModules.put(any(String.class), any(AtlasModule.class))).thenThrow(new RuntimeException("mockException"));
    ctx.setTargetModules(targetModules);
    ctx.init();
}

Example 9 with DataSource

use of com.google.cloud.bigquery.datatransfer.v1.DataSource in project atlasmap by atlasmap.

the class BaseDefaultAtlasContextTest method init.

@BeforeEach
public void init() throws AtlasException {
    mapping = AtlasTestData.generateAtlasMapping();
    context = new DefaultAtlasContext(DefaultAtlasContextFactory.getInstance(), mapping) {

        protected void init() {
        // hijack initialization
        }
    };
    previewContext = new DefaultAtlasPreviewContext(DefaultAtlasContextFactory.getInstance());
    sourceModule = mockAtlasModule();
    DataSource sds = new DataSource();
    sds.setDataSourceType(DataSourceType.SOURCE);
    sourceModule.setDataSource(sds);
    targetModule = mockAtlasModule();
    DataSource tds = new DataSource();
    tds.setDataSourceType(DataSourceType.TARGET);
    targetModule.setDataSource(tds);
    context.getSourceModules().put(AtlasConstants.DEFAULT_SOURCE_DOCUMENT_ID, sourceModule);
    context.getTargetModules().put(AtlasConstants.DEFAULT_TARGET_DOCUMENT_ID, targetModule);
    ConstantModule constantModule = new ConstantModule();
    constantModule.setConversionService(DefaultAtlasConversionService.getInstance());
    context.getSourceModules().put(AtlasConstants.CONSTANTS_DOCUMENT_ID, constantModule);
    PropertyModule sourcePropertyModule = new PropertyModule(new DefaultAtlasPropertyStrategy());
    sourcePropertyModule.setConversionService(DefaultAtlasConversionService.getInstance());
    context.getSourceModules().put(AtlasConstants.PROPERTIES_SOURCE_DOCUMENT_ID, sourcePropertyModule);
    PropertyModule targetPropertyModule = new PropertyModule(new DefaultAtlasPropertyStrategy());
    targetPropertyModule.setConversionService(DefaultAtlasConversionService.getInstance());
    context.getTargetModules().put(AtlasConstants.PROPERTIES_TARGET_DOCUMENT_ID, targetPropertyModule);
    recreateSession();
}

Example 10 with DataSource

use of com.google.cloud.bigquery.datatransfer.v1.DataSource in project atlasmap by atlasmap.

the class BaseMarshallerTest method generateMultiSourceMapping.

protected AtlasMapping generateMultiSourceMapping() {
    AtlasMapping mapping = generateSeparateAtlasMapping();
    DataSource source1 = new DataSource();
    source1.setUri("xml:foo1");
    source1.setDataSourceType(DataSourceType.SOURCE);
    source1.setId("xml1");
    DataSource source2 = new DataSource();
    source2.setUri("xml:foo2");
    source2.setDataSourceType(DataSourceType.SOURCE);
    source2.setId("xml2");
    DataSource target = new DataSource();
    target.setUri("xml:bar");
    target.setDataSourceType(DataSourceType.TARGET);
    target.setId("target1");
    mapping.getDataSource().add(source1);
    mapping.getDataSource().add(source2);
    mapping.getDataSource().add(target);
    Mapping fm = (Mapping) mapping.getMappings().getMapping().get(0);
    fm.getInputField().get(0).setDocId("xml1");
    fm.getOutputField().get(0).setDocId("target1");
    fm.getOutputField().get(1).setDocId("target1");
    return mapping;
}