Examples with ReflectionSchemaGenerator co.cask.cdap.internal.io.ReflectionSchemaGenerator used on opensource projects

Example 21 with ReflectionSchemaGenerator

use of co.cask.cdap.internal.io.ReflectionSchemaGenerator in project cdap by caskdata.

the class ExploreTableManager method hiveSchemaFor.

// TODO: replace with SchemaConverter.toHiveSchema when we tackle queries on Tables.
// but unfortunately, SchemaConverter is not compatible with this, for example:
// - a byte becomes a tinyint here, but an int there
// - SchemaConverter sort fields alphabetically, whereas this preserves the order
// - ExploreExtensiveSchemaTableTestRun will fail because of this
private String hiveSchemaFor(Type type) throws UnsupportedTypeException {
    // This call will make sure that the type is not recursive
    try {
        new ReflectionSchemaGenerator().generate(type, false);
    } catch (Exception e) {
        throw new UnsupportedTypeException("Unable to derive schema from " + type, e);
    }
    ObjectInspector objectInspector = ObjectInspectorFactory.getReflectionObjectInspector(type);
    if (!(objectInspector instanceof StructObjectInspector)) {
        throw new UnsupportedTypeException(String.format("Type must be a RECORD, but is %s", type.getClass().getName()));
    }
    StructObjectInspector structObjectInspector = (StructObjectInspector) objectInspector;
    StringBuilder sb = new StringBuilder();
    boolean first = true;
    for (StructField structField : structObjectInspector.getAllStructFieldRefs()) {
        if (first) {
            first = false;
        } else {
            sb.append(", ");
        }
        ObjectInspector oi = structField.getFieldObjectInspector();
        String typeName;
        typeName = oi.getTypeName();
        if (shouldEscapeColumns) {
            // a literal backtick(`) is represented as a double backtick(``)
            sb.append('`').append(structField.getFieldName().replace("`", "``")).append('`');
        } else {
            sb.append(structField.getFieldName());
        }
        sb.append(" ").append(typeName);
    }
    return sb.toString();
}

Example 22 with ReflectionSchemaGenerator

use of co.cask.cdap.internal.io.ReflectionSchemaGenerator in project cdap by caskdata.

the class ObjectStoreDatasetTest method testWithCustomClassLoader.

@Test
public void testWithCustomClassLoader() throws Exception {
    DatasetId kv = DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("kv");
    // create a dummy class loader that records the name of the class it loaded
    final AtomicReference<String> lastClassLoaded = new AtomicReference<>(null);
    ClassLoader loader = new ClassLoader() {

        @Override
        public Class<?> loadClass(String name) throws ClassNotFoundException {
            lastClassLoaded.set(name);
            return super.loadClass(name);
        }
    };
    dsFrameworkUtil.createInstance("keyValueTable", kv, DatasetProperties.EMPTY);
    KeyValueTable kvTable = dsFrameworkUtil.getInstance(kv);
    Type type = Custom.class;
    TypeRepresentation typeRep = new TypeRepresentation(type);
    Schema schema = new ReflectionSchemaGenerator().generate(type);
    final ObjectStoreDataset<Custom> objectStore = new ObjectStoreDataset<>("kv", kvTable, typeRep, schema, loader);
    TransactionExecutor txnl = dsFrameworkUtil.newInMemoryTransactionExecutor(objectStore);
    // need to call this to actually load the Custom class, because the Custom class is no longer used in the
    // ObjectStoreDataset's constructor, but rather lazily when its actually needed.
    objectStore.getRecordType();
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            objectStore.write("dummy", new Custom(382, Lists.newArrayList("blah")));
        }
    });
    // verify the class name was recorded (the dummy class loader was used).
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals(Custom.class.getName(), lastClassLoaded.get());
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            deleteAndVerify(objectStore, Bytes.toBytes("dummy"));
        }
    });
    dsFrameworkUtil.deleteInstance(kv);
}

Example 23 with ReflectionSchemaGenerator

use of co.cask.cdap.internal.io.ReflectionSchemaGenerator in project cdap by caskdata.

the class ReflectionTableTest method testStructuredRecordRepresentation.

@Test
public void testStructuredRecordRepresentation() throws Exception {
    dsFrameworkUtil.createInstance("table", users, DatasetProperties.builder().build());
    try {
        final Table usersTable = dsFrameworkUtil.getInstance(users);
        final byte[] rowKey = Bytes.toBytes(123);
        final Schema schema = new ReflectionSchemaGenerator().generate(User.class);
        // TableDataset is not accessible here, but we know that's the underlying implementation...
        TransactionExecutor tx = dsFrameworkUtil.newTransactionExecutor((TransactionAware) usersTable);
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                Put put = new Put(rowKey);
                ReflectionPutWriter<User> putWriter = new ReflectionPutWriter<>(schema);
                putWriter.write(SAMUEL, put);
                usersTable.put(put);
                Row row = usersTable.get(rowKey);
                ReflectionRowRecordReader rowReader = new ReflectionRowRecordReader(schema, null);
                StructuredRecord actual = rowReader.read(row, schema);
                assertRecordEqualsUser(SAMUEL, actual);
            }
        });
    } finally {
        dsFrameworkUtil.deleteInstance(users);
    }
}

Example 24 with ReflectionSchemaGenerator

use of co.cask.cdap.internal.io.ReflectionSchemaGenerator in project cdap by caskdata.

the class MetricsTestBase method init.

@Before
public void init() throws IOException, UnsupportedTypeException {
    cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    injector = Guice.createInjector(getModules());
    messagingService = injector.getInstance(MessagingService.class);
    if (messagingService instanceof Service) {
        ((Service) messagingService).startAndWait();
    }
    metricValueType = TypeToken.of(MetricValues.class);
    schema = new ReflectionSchemaGenerator().generate(metricValueType.getType());
    recordWriter = new ASMDatumWriterFactory(new ASMFieldAccessorFactory()).create(metricValueType, schema);
}

Example 25 with ReflectionSchemaGenerator

use of co.cask.cdap.internal.io.ReflectionSchemaGenerator in project cdap by caskdata.

the class ArtifactClientTestRun method testArtifacts.

@Test
public void testArtifacts() throws Exception {
    // add 2 versions of an artifact with an application
    ArtifactId myapp1Id = NamespaceId.DEFAULT.artifact("myapp", "1.0.0");
    ArtifactId myapp2Id = NamespaceId.DEFAULT.artifact("myapp", "2.0.0");
    LocalLocationFactory locationFactory = new LocalLocationFactory(TMP_FOLDER.newFolder());
    Manifest manifest = new Manifest();
    manifest.getMainAttributes().put(ManifestFields.BUNDLE_VERSION, "2.0.0");
    final Location appJarLoc = AppJarHelper.createDeploymentJar(locationFactory, MyApp.class, manifest);
    InputSupplier<InputStream> inputSupplier = new InputSupplier<InputStream>() {

        @Override
        public InputStream getInput() throws IOException {
            return appJarLoc.getInputStream();
        }
    };
    artifactClient.add(myapp1Id.getParent(), myapp1Id.getArtifact(), inputSupplier, myapp1Id.getVersion());
    // add some properties
    Map<String, String> myapp1Properties = ImmutableMap.of("k1", "v1");
    artifactClient.writeProperties(myapp1Id, myapp1Properties);
    // let it derive version from jar manifest, which has bundle-version at 2.0.0
    artifactClient.add(myapp2Id.getParent(), myapp2Id.getArtifact(), inputSupplier, null, null);
    // add some properties
    Map<String, String> myapp2Properties = ImmutableMap.of("k1", "v1", "k2", "v2");
    artifactClient.writeProperties(myapp2Id, myapp2Properties);
    // add an artifact that contains a plugin, but only extends myapp-2.0.0
    ArtifactId pluginId = NamespaceId.DEFAULT.artifact("myapp-plugins", "2.0.0");
    manifest = new Manifest();
    manifest.getMainAttributes().put(ManifestFields.EXPORT_PACKAGE, Plugin1.class.getPackage().getName());
    final Location pluginJarLoc = PluginJarHelper.createPluginJar(locationFactory, manifest, Plugin1.class);
    inputSupplier = new InputSupplier<InputStream>() {

        @Override
        public InputStream getInput() throws IOException {
            return pluginJarLoc.getInputStream();
        }
    };
    Set<ArtifactRange> parents = Sets.newHashSet(new ArtifactRange(myapp2Id.getParent().getNamespace(), myapp2Id.getArtifact(), new ArtifactVersion(myapp2Id.getVersion()), new ArtifactVersion("3.0.0")));
    Set<PluginClass> additionalPlugins = Sets.newHashSet(new PluginClass("jdbc", "mysql", "", "com.mysql.jdbc.Driver", null, Collections.<String, PluginPropertyField>emptyMap()));
    artifactClient.add(pluginId.getParent(), pluginId.getArtifact(), inputSupplier, pluginId.getVersion(), parents, additionalPlugins);
    ArtifactSummary myapp1Summary = new ArtifactSummary(myapp1Id.getArtifact(), myapp1Id.getVersion());
    ArtifactSummary myapp2Summary = new ArtifactSummary(myapp2Id.getArtifact(), myapp2Id.getVersion());
    ArtifactSummary pluginArtifactSummary = new ArtifactSummary(pluginId.getArtifact(), pluginId.getVersion());
    Set<ArtifactSummary> artifacts = Sets.newHashSet(artifactClient.list(NamespaceId.DEFAULT));
    Assert.assertEquals(Sets.newHashSet(myapp1Summary, myapp2Summary, pluginArtifactSummary), artifacts);
    // list all artifacts named 'myapp'
    Assert.assertEquals(Sets.newHashSet(myapp1Summary, myapp2Summary), Sets.newHashSet(artifactClient.listVersions(NamespaceId.DEFAULT, myapp1Id.getArtifact())));
    // list all artifacts named 'myapp-plugins'
    Assert.assertEquals(Sets.newHashSet(pluginArtifactSummary), Sets.newHashSet(artifactClient.listVersions(NamespaceId.DEFAULT, pluginId.getArtifact())));
    // artifacts should be in user scope
    try {
        artifactClient.listVersions(NamespaceId.DEFAULT, pluginId.getArtifact(), ArtifactScope.SYSTEM);
        Assert.fail();
    } catch (ArtifactNotFoundException e) {
    // expected
    }
    // get info about specific artifacts
    Schema myAppConfigSchema = new ReflectionSchemaGenerator(false).generate(MyApp.Conf.class);
    ArtifactClasses myAppClasses = ArtifactClasses.builder().addApp(new ApplicationClass(MyApp.class.getName(), "", myAppConfigSchema)).build();
    // test get myapp-1.0.0
    ArtifactInfo myapp1Info = new ArtifactInfo(myapp1Id.getArtifact(), myapp1Id.getVersion(), ArtifactScope.USER, myAppClasses, myapp1Properties);
    Assert.assertEquals(myapp1Info, artifactClient.getArtifactInfo(myapp1Id));
    // test get myapp-2.0.0
    ArtifactInfo myapp2Info = new ArtifactInfo(myapp2Id.getArtifact(), myapp2Id.getVersion(), ArtifactScope.USER, myAppClasses, myapp2Properties);
    Assert.assertEquals(myapp2Info, artifactClient.getArtifactInfo(myapp2Id));
    // test overwriting properties
    myapp2Properties = ImmutableMap.of("k1", "v3", "k5", "v5");
    artifactClient.writeProperties(myapp2Id, myapp2Properties);
    Assert.assertEquals(myapp2Properties, artifactClient.getArtifactInfo(myapp2Id).getProperties());
    // test deleting property
    artifactClient.deleteProperty(myapp2Id, "k1");
    Assert.assertEquals(ImmutableMap.of("k5", "v5"), artifactClient.getArtifactInfo(myapp2Id).getProperties());
    // test writing property
    artifactClient.writeProperty(myapp2Id, "k5", "v4");
    Assert.assertEquals(ImmutableMap.of("k5", "v4"), artifactClient.getArtifactInfo(myapp2Id).getProperties());
    // test deleting properties
    artifactClient.deleteProperties(myapp2Id);
    Assert.assertTrue(artifactClient.getArtifactInfo(myapp2Id).getProperties().isEmpty());
    // test get myapp-plugins-2.0.0
    Map<String, PluginPropertyField> props = ImmutableMap.of("x", new PluginPropertyField("x", "", "int", true, false));
    ArtifactClasses pluginClasses = ArtifactClasses.builder().addPlugin(new PluginClass("callable", "plugin1", "p1 description", Plugin1.class.getName(), "conf", props)).addPlugins(additionalPlugins).build();
    ArtifactInfo pluginArtifactInfo = new ArtifactInfo(pluginId.getArtifact(), pluginId.getVersion(), ArtifactScope.USER, pluginClasses, ImmutableMap.<String, String>of());
    Assert.assertEquals(pluginArtifactInfo, artifactClient.getArtifactInfo(pluginId));
    // test get all app classes in namespace
    Set<ApplicationClassSummary> expectedSummaries = ImmutableSet.of(new ApplicationClassSummary(myapp1Summary, MyApp.class.getName()), new ApplicationClassSummary(myapp2Summary, MyApp.class.getName()));
    Set<ApplicationClassSummary> appClassSummaries = Sets.newHashSet(artifactClient.getApplicationClasses(NamespaceId.DEFAULT));
    Assert.assertEquals(expectedSummaries, appClassSummaries);
    // test get all app classes in namespace with name MyApp.class.getName()
    Set<ApplicationClassInfo> appClassInfos = Sets.newHashSet(artifactClient.getApplicationClasses(NamespaceId.DEFAULT, MyApp.class.getName()));
    Set<ApplicationClassInfo> expectedInfos = ImmutableSet.of(new ApplicationClassInfo(myapp1Summary, MyApp.class.getName(), myAppConfigSchema), new ApplicationClassInfo(myapp2Summary, MyApp.class.getName(), myAppConfigSchema));
    Assert.assertEquals(expectedInfos, appClassInfos);
    // test get plugin types for myapp-1.0.0. should be empty, since plugins only extends versions [2.0.0 - 3.0.0)
    Assert.assertTrue(artifactClient.getPluginTypes(myapp1Id).isEmpty());
    // test get plugin types for myapp-2.0.0
    Assert.assertEquals(Lists.newArrayList("callable", "jdbc"), artifactClient.getPluginTypes(myapp2Id));
    // test get plugins of type callable for myapp-2.0.0
    PluginSummary pluginSummary = new PluginSummary("plugin1", "callable", "p1 description", Plugin1.class.getName(), pluginArtifactSummary);
    Assert.assertEquals(Sets.newHashSet(pluginSummary), Sets.newHashSet(artifactClient.getPluginSummaries(myapp2Id, "callable")));
    // no plugins of type "runnable"
    Assert.assertTrue(artifactClient.getPluginSummaries(myapp2Id, "runnable").isEmpty());
    // test get plugin details for plugin1 for myapp-2.0.0
    PluginInfo pluginInfo = new PluginInfo("plugin1", "callable", "p1 description", Plugin1.class.getName(), "conf", pluginArtifactSummary, props, new HashSet<String>());
    Assert.assertEquals(Sets.newHashSet(pluginInfo), Sets.newHashSet(artifactClient.getPluginInfo(myapp2Id, "callable", "plugin1")));
}