Search in sources :

Example 21 with HBaseTableUtilFactory

use of co.cask.cdap.data2.util.hbase.HBaseTableUtilFactory in project cdap by caskdata.

the class HBasePayloadTableTestRun method setupBeforeClass.

@BeforeClass
public static void setupBeforeClass() throws Exception {
    hConf = HBASE_TEST_BASE.getConfiguration();
    hConf.set(HBaseTableUtil.CFG_HBASE_TABLE_COMPRESSION, HBaseTableUtil.CompressionType.NONE.name());
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    cConf.set(Constants.CFG_HDFS_NAMESPACE, cConf.get(Constants.CFG_LOCAL_DATA_DIR));
    cConf.set(Constants.CFG_HDFS_USER, System.getProperty("user.name"));
    tableUtil = new HBaseTableUtilFactory(cConf).get();
    ddlExecutor = new HBaseDDLExecutorFactory(cConf, hConf).get();
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(NamespaceId.SYSTEM));
    LocationFactory locationFactory = getInjector().getInstance(LocationFactory.class);
    tableFactory = new HBaseTableFactory(cConf, hConf, tableUtil, locationFactory);
    ConfigurationTable configTable = new ConfigurationTable(hConf);
    configTable.write(ConfigurationTable.Type.DEFAULT, cConf);
}
Also used : ConfigurationTable(co.cask.cdap.data2.util.hbase.ConfigurationTable) HBaseDDLExecutorFactory(co.cask.cdap.data2.util.hbase.HBaseDDLExecutorFactory) HBaseTableUtilFactory(co.cask.cdap.data2.util.hbase.HBaseTableUtilFactory) LocationFactory(org.apache.twill.filesystem.LocationFactory) BeforeClass(org.junit.BeforeClass)

Example 22 with HBaseTableUtilFactory

use of co.cask.cdap.data2.util.hbase.HBaseTableUtilFactory in project cdap by caskdata.

the class HBaseTableCoprocessorTestRun method setupBeforeClass.

@BeforeClass
public static void setupBeforeClass() throws Exception {
    hConf = HBASE_TEST_BASE.getConfiguration();
    hConf.set(HBaseTableUtil.CFG_HBASE_TABLE_COMPRESSION, HBaseTableUtil.CompressionType.NONE.name());
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    cConf.set(Constants.CFG_HDFS_NAMESPACE, cConf.get(Constants.CFG_LOCAL_DATA_DIR));
    cConf.set(Constants.CFG_HDFS_USER, System.getProperty("user.name"));
    // Reduce the metadata cache refresh frequency for unit tests
    cConf.set(Constants.MessagingSystem.COPROCESSOR_METADATA_CACHE_UPDATE_FREQUENCY_SECONDS, Integer.toString(METADATA_CACHE_EXPIRY));
    hBaseAdmin = HBASE_TEST_BASE.getHBaseAdmin();
    hBaseAdmin.getConfiguration().set(HBaseTableUtil.CFG_HBASE_TABLE_COMPRESSION, HBaseTableUtil.CompressionType.NONE.name());
    tableUtil = new HBaseTableUtilFactory(cConf).get();
    ddlExecutor = new HBaseDDLExecutorFactory(cConf, hConf).get();
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(NamespaceId.SYSTEM));
    LocationFactory locationFactory = getInjector().getInstance(LocationFactory.class);
    tableFactory = new HBaseTableFactory(cConf, hBaseAdmin.getConfiguration(), tableUtil, locationFactory);
    ConfigurationTable configTable = new ConfigurationTable(hConf);
    configTable.write(ConfigurationTable.Type.DEFAULT, cConf);
    // write an initial transaction snapshot
    TransactionSnapshot txSnapshot = TransactionSnapshot.copyFrom(System.currentTimeMillis(), V[6] - 1, V[7], invalidSet, // this will set visibility upper bound to V[6]
    Maps.newTreeMap(ImmutableSortedMap.of(V[6], new TransactionManager.InProgressTx(V[6] - 1, Long.MAX_VALUE, TransactionManager.InProgressType.SHORT))), new HashMap<Long, Set<ChangeId>>(), new TreeMap<Long, Set<ChangeId>>());
    HDFSTransactionStateStorage tmpStorage = new HDFSTransactionStateStorage(hConf, new SnapshotCodecProvider(hConf), new TxMetricsCollector());
    tmpStorage.startAndWait();
    tmpStorage.writeSnapshot(txSnapshot);
    tmpStorage.stopAndWait();
}
Also used : Set(java.util.Set) TxMetricsCollector(org.apache.tephra.metrics.TxMetricsCollector) SnapshotCodecProvider(org.apache.tephra.snapshot.SnapshotCodecProvider) LocationFactory(org.apache.twill.filesystem.LocationFactory) HDFSTransactionStateStorage(org.apache.tephra.persist.HDFSTransactionStateStorage) TransactionSnapshot(org.apache.tephra.persist.TransactionSnapshot) ConfigurationTable(co.cask.cdap.data2.util.hbase.ConfigurationTable) TransactionManager(org.apache.tephra.TransactionManager) HBaseDDLExecutorFactory(co.cask.cdap.data2.util.hbase.HBaseDDLExecutorFactory) HBaseTableUtilFactory(co.cask.cdap.data2.util.hbase.HBaseTableUtilFactory) BeforeClass(org.junit.BeforeClass)

Example 23 with HBaseTableUtilFactory

use of co.cask.cdap.data2.util.hbase.HBaseTableUtilFactory in project cdap by caskdata.

the class HBaseMessageTableTestRun method setupBeforeClass.

@BeforeClass
public static void setupBeforeClass() throws Exception {
    hConf = HBASE_TEST_BASE.getConfiguration();
    hConf.set(HBaseTableUtil.CFG_HBASE_TABLE_COMPRESSION, HBaseTableUtil.CompressionType.NONE.name());
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    cConf.set(Constants.CFG_HDFS_NAMESPACE, cConf.get(Constants.CFG_LOCAL_DATA_DIR));
    cConf.set(Constants.CFG_HDFS_USER, System.getProperty("user.name"));
    tableUtil = new HBaseTableUtilFactory(cConf).get();
    ddlExecutor = new HBaseDDLExecutorFactory(cConf, hConf).get();
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(NamespaceId.SYSTEM));
    LocationFactory locationFactory = getInjector().getInstance(LocationFactory.class);
    tableFactory = new HBaseTableFactory(cConf, hConf, tableUtil, locationFactory);
    ConfigurationTable configTable = new ConfigurationTable(hConf);
    configTable.write(ConfigurationTable.Type.DEFAULT, cConf);
}
Also used : ConfigurationTable(co.cask.cdap.data2.util.hbase.ConfigurationTable) HBaseDDLExecutorFactory(co.cask.cdap.data2.util.hbase.HBaseDDLExecutorFactory) HBaseTableUtilFactory(co.cask.cdap.data2.util.hbase.HBaseTableUtilFactory) LocationFactory(org.apache.twill.filesystem.LocationFactory) BeforeClass(org.junit.BeforeClass)

Example 24 with HBaseTableUtilFactory

use of co.cask.cdap.data2.util.hbase.HBaseTableUtilFactory in project cdap by caskdata.

the class CoprocessorBuildTool method main.

public static void main(final String[] args) throws ParseException {
    Options options = new Options().addOption(new Option("h", "help", false, "Print this usage message.")).addOption(new Option("f", "force", false, "Overwrites any coprocessors that already exist."));
    CommandLineParser parser = new BasicParser();
    CommandLine commandLine = parser.parse(options, args);
    String[] commandArgs = commandLine.getArgs();
    // if help is an option, or if there isn't a single 'ensure' command, print usage and exit.
    if (commandLine.hasOption("h") || commandArgs.length != 1 || !"check".equalsIgnoreCase(commandArgs[0])) {
        HelpFormatter helpFormatter = new HelpFormatter();
        helpFormatter.printHelp(CoprocessorBuildTool.class.getName() + " check", "Checks that HBase coprocessors required by CDAP are loaded onto HDFS. " + "If not, the coprocessors are built and placed on HDFS.", options, "");
        System.exit(0);
    }
    boolean overwrite = commandLine.hasOption("f");
    CConfiguration cConf = CConfiguration.create();
    Configuration hConf = HBaseConfiguration.create();
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), // for LocationFactory
    new PrivateModule() {

        @Override
        protected void configure() {
            bind(FileContext.class).toProvider(FileContextProvider.class).in(Scopes.SINGLETON);
            expose(LocationFactory.class);
        }

        @Provides
        @Singleton
        private LocationFactory providesLocationFactory(Configuration hConf, CConfiguration cConf, FileContext fc) {
            final String namespace = cConf.get(Constants.CFG_HDFS_NAMESPACE);
            if (UserGroupInformation.isSecurityEnabled()) {
                return new FileContextLocationFactory(hConf, namespace);
            }
            return new InsecureFileContextLocationFactory(hConf, namespace, fc);
        }
    });
    try {
        SecurityUtil.loginForMasterService(cConf);
    } catch (Exception e) {
        LOG.error("Failed to login as CDAP user", e);
        System.exit(1);
    }
    LocationFactory locationFactory = injector.getInstance(LocationFactory.class);
    HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get();
    CoprocessorManager coprocessorManager = new CoprocessorManager(cConf, locationFactory, tableUtil);
    try {
        Location location = coprocessorManager.ensureCoprocessorExists(overwrite);
        LOG.info("coprocessor exists at {}.", location);
    } catch (IOException e) {
        LOG.error("Unable to build and upload coprocessor jars.", e);
        System.exit(1);
    }
}
Also used : Options(org.apache.commons.cli.Options) CConfiguration(co.cask.cdap.common.conf.CConfiguration) Configuration(org.apache.hadoop.conf.Configuration) HBaseConfiguration(org.apache.hadoop.hbase.HBaseConfiguration) ConfigModule(co.cask.cdap.common.guice.ConfigModule) InsecureFileContextLocationFactory(co.cask.cdap.common.guice.InsecureFileContextLocationFactory) FileContextLocationFactory(org.apache.twill.filesystem.FileContextLocationFactory) HelpFormatter(org.apache.commons.cli.HelpFormatter) Injector(com.google.inject.Injector) InsecureFileContextLocationFactory(co.cask.cdap.common.guice.InsecureFileContextLocationFactory) HBaseTableUtilFactory(co.cask.cdap.data2.util.hbase.HBaseTableUtilFactory) CommandLineParser(org.apache.commons.cli.CommandLineParser) PrivateModule(com.google.inject.PrivateModule) IOException(java.io.IOException) Provides(com.google.inject.Provides) CConfiguration(co.cask.cdap.common.conf.CConfiguration) HBaseTableUtil(co.cask.cdap.data2.util.hbase.HBaseTableUtil) IOException(java.io.IOException) ParseException(org.apache.commons.cli.ParseException) InsecureFileContextLocationFactory(co.cask.cdap.common.guice.InsecureFileContextLocationFactory) LocationFactory(org.apache.twill.filesystem.LocationFactory) FileContextLocationFactory(org.apache.twill.filesystem.FileContextLocationFactory) BasicParser(org.apache.commons.cli.BasicParser) CommandLine(org.apache.commons.cli.CommandLine) CoprocessorManager(co.cask.cdap.data2.util.hbase.CoprocessorManager) Singleton(com.google.inject.Singleton) Option(org.apache.commons.cli.Option) FileContext(org.apache.hadoop.fs.FileContext) Location(org.apache.twill.filesystem.Location)

Example 25 with HBaseTableUtilFactory

use of co.cask.cdap.data2.util.hbase.HBaseTableUtilFactory in project cdap by caskdata.

the class IncrementHandlerTest method createTable.

@Override
public HTable createTable(TableId tableId) throws Exception {
    HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get();
    HTableDescriptorBuilder tableDesc = tableUtil.buildHTableDescriptor(tableId);
    HColumnDescriptor columnDesc = new HColumnDescriptor(FAMILY);
    columnDesc.setMaxVersions(Integer.MAX_VALUE);
    columnDesc.setValue(IncrementHandlerState.PROPERTY_TRANSACTIONAL, "false");
    tableDesc.addFamily(columnDesc);
    tableDesc.addCoprocessor(IncrementHandler.class.getName());
    HTableDescriptor htd = tableDesc.build();
    TEST_HBASE.getHBaseAdmin().createTable(htd);
    TEST_HBASE.waitUntilTableAvailable(htd.getName(), 5000);
    return tableUtil.createHTable(conf, tableId);
}
Also used : HTableDescriptorBuilder(co.cask.cdap.data2.util.hbase.HTableDescriptorBuilder) HColumnDescriptor(org.apache.hadoop.hbase.HColumnDescriptor) HBaseTableUtilFactory(co.cask.cdap.data2.util.hbase.HBaseTableUtilFactory) HBaseTableUtil(co.cask.cdap.data2.util.hbase.HBaseTableUtil) HTableDescriptor(org.apache.hadoop.hbase.HTableDescriptor)

Aggregations

HBaseTableUtilFactory (co.cask.cdap.data2.util.hbase.HBaseTableUtilFactory)25 HBaseTableUtil (co.cask.cdap.data2.util.hbase.HBaseTableUtil)18 HTableDescriptorBuilder (co.cask.cdap.data2.util.hbase.HTableDescriptorBuilder)14 HTableDescriptor (org.apache.hadoop.hbase.HTableDescriptor)14 FileSystem (org.apache.hadoop.fs.FileSystem)7 Path (org.apache.hadoop.fs.Path)7 HColumnDescriptor (org.apache.hadoop.hbase.HColumnDescriptor)7 HRegionInfo (org.apache.hadoop.hbase.HRegionInfo)7 HRegion (org.apache.hadoop.hbase.regionserver.HRegion)7 HRegionFileSystem (org.apache.hadoop.hbase.regionserver.HRegionFileSystem)7 HBaseDDLExecutorFactory (co.cask.cdap.data2.util.hbase.HBaseDDLExecutorFactory)6 BeforeClass (org.junit.BeforeClass)6 WAL (org.apache.hadoop.hbase.wal.WAL)5 WALFactory (org.apache.hadoop.hbase.wal.WALFactory)5 LocationFactory (org.apache.twill.filesystem.LocationFactory)5 IOException (java.io.IOException)4 HTable (org.apache.hadoop.hbase.client.HTable)4 ConfigurationTable (co.cask.cdap.data2.util.hbase.ConfigurationTable)3 SimpleNamespaceQueryAdmin (co.cask.cdap.common.namespace.SimpleNamespaceQueryAdmin)2 TableId (co.cask.cdap.data2.util.TableId)2