Examples with TableConfiguration org.apache.accumulo.server.conf.TableConfiguration used on opensource projects

Example 6 with TableConfiguration

use of org.apache.accumulo.server.conf.TableConfiguration in project accumulo by apache.

the class HostRegexTableLoadBalancerTest method testSplitCurrentByRegexUsingOverlappingPools.

@Test
public void testSplitCurrentByRegexUsingOverlappingPools() {
    init(new AccumuloServerContext(instance, new TestServerConfigurationFactory(instance) {

        @Override
        public TableConfiguration getTableConfiguration(Table.ID tableId) {
            NamespaceConfiguration defaultConf = new NamespaceConfiguration(Namespace.ID.DEFAULT, this.instance, DefaultConfiguration.getInstance());
            return new TableConfiguration(instance, tableId, defaultConf) {

                HashMap<String, String> tableProperties = new HashMap<>();

                {
                    tableProperties.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + FOO.getTableName(), "r.*");
                    tableProperties.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "r01.*|r02.*");
                }

                @Override
                public String get(Property property) {
                    return tableProperties.get(property.name());
                }

                @Override
                public void getProperties(Map<String, String> props, Predicate<String> filter) {
                    for (Entry<String, String> e : tableProperties.entrySet()) {
                        if (filter.test(e.getKey())) {
                            props.put(e.getKey(), e.getValue());
                        }
                    }
                }

                @Override
                public long getUpdateCount() {
                    return 0;
                }
            };
        }
    }));
    Map<String, SortedMap<TServerInstance, TabletServerStatus>> groups = this.splitCurrentByRegex(createCurrent(15));
    Assert.assertEquals(2, groups.size());
    Assert.assertTrue(groups.containsKey(FOO.getTableName()));
    SortedMap<TServerInstance, TabletServerStatus> fooHosts = groups.get(FOO.getTableName());
    Assert.assertEquals(15, fooHosts.size());
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.11:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.12:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.13:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.14:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.15:9997", 1)));
    Assert.assertTrue(groups.containsKey(BAR.getTableName()));
    SortedMap<TServerInstance, TabletServerStatus> barHosts = groups.get(BAR.getTableName());
    Assert.assertEquals(10, barHosts.size());
    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1)));
    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1)));
    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1)));
    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1)));
    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1)));
    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1)));
    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1)));
    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1)));
    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1)));
    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1)));
}

Example 7 with TableConfiguration

use of org.apache.accumulo.server.conf.TableConfiguration in project accumulo by apache.

the class HostRegexTableLoadBalancerTest method testSplitCurrentByRegexUsingIP.

@Test
public void testSplitCurrentByRegexUsingIP() {
    init(new AccumuloServerContext(instance, new TestServerConfigurationFactory(instance) {

        @Override
        public synchronized AccumuloConfiguration getSystemConfiguration() {
            HashMap<String, String> props = new HashMap<>();
            props.put(HostRegexTableLoadBalancer.HOST_BALANCER_OOB_CHECK_KEY, "30s");
            props.put(HostRegexTableLoadBalancer.HOST_BALANCER_REGEX_USING_IPS_KEY, "true");
            return new ConfigurationCopy(props);
        }

        @Override
        public TableConfiguration getTableConfiguration(Table.ID tableId) {
            NamespaceConfiguration defaultConf = new NamespaceConfiguration(Namespace.ID.DEFAULT, this.instance, DefaultConfiguration.getInstance());
            return new TableConfiguration(instance, tableId, defaultConf) {

                HashMap<String, String> tableProperties = new HashMap<>();

                {
                    tableProperties.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + FOO.getTableName(), "192\\.168\\.0\\.[1-5]");
                    tableProperties.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "192\\.168\\.0\\.[6-9]|192\\.168\\.0\\.10");
                }

                @Override
                public String get(Property property) {
                    return tableProperties.get(property.name());
                }

                @Override
                public void getProperties(Map<String, String> props, Predicate<String> filter) {
                    for (Entry<String, String> e : tableProperties.entrySet()) {
                        if (filter.test(e.getKey())) {
                            props.put(e.getKey(), e.getValue());
                        }
                    }
                }

                @Override
                public long getUpdateCount() {
                    return 0;
                }
            };
        }
    }));
    Assert.assertTrue(isIpBasedRegex());
    Map<String, SortedMap<TServerInstance, TabletServerStatus>> groups = this.splitCurrentByRegex(createCurrent(15));
    Assert.assertEquals(3, groups.size());
    Assert.assertTrue(groups.containsKey(FOO.getTableName()));
    SortedMap<TServerInstance, TabletServerStatus> fooHosts = groups.get(FOO.getTableName());
    Assert.assertEquals(5, fooHosts.size());
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1)));
    Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1)));
    Assert.assertTrue(groups.containsKey(BAR.getTableName()));
    SortedMap<TServerInstance, TabletServerStatus> barHosts = groups.get(BAR.getTableName());
    Assert.assertEquals(5, barHosts.size());
    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1)));
    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1)));
    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1)));
    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1)));
    Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1)));
    Assert.assertTrue(groups.containsKey(DEFAULT_POOL));
    SortedMap<TServerInstance, TabletServerStatus> defHosts = groups.get(DEFAULT_POOL);
    Assert.assertEquals(5, defHosts.size());
    Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.11:9997", 1)));
    Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.12:9997", 1)));
    Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.13:9997", 1)));
    Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.14:9997", 1)));
    Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.15:9997", 1)));
}

Example 8 with TableConfiguration

use of org.apache.accumulo.server.conf.TableConfiguration in project accumulo by apache.

the class TableLoadBalancerTest method test.

@Test
public void test() throws Exception {
    final Instance inst = EasyMock.createMock(Instance.class);
    EasyMock.expect(inst.getInstanceID()).andReturn(UUID.nameUUIDFromBytes(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0 }).toString()).anyTimes();
    EasyMock.expect(inst.getZooKeepers()).andReturn("10.0.0.1:1234").anyTimes();
    EasyMock.expect(inst.getZooKeepersSessionTimeOut()).andReturn(30_000).anyTimes();
    EasyMock.replay(inst);
    ServerConfigurationFactory confFactory = new ServerConfigurationFactory(inst) {

        @Override
        public TableConfiguration getTableConfiguration(Table.ID tableId) {
            // create a dummy namespaceConfiguration to satisfy requireNonNull in TableConfiguration constructor
            NamespaceConfiguration dummyConf = new NamespaceConfiguration(null, inst, null);
            return new TableConfiguration(inst, tableId, dummyConf) {

                @Override
                public String get(Property property) {
                    // fake the get table configuration so the test doesn't try to look in zookeeper for per-table classpath stuff
                    return DefaultConfiguration.getInstance().get(property);
                }
            };
        }
    };
    String t1Id = TABLE_ID_MAP.get("t1"), t2Id = TABLE_ID_MAP.get("t2"), t3Id = TABLE_ID_MAP.get("t3");
    state = new TreeMap<>();
    TServerInstance svr = mkts("10.0.0.1", "0x01020304");
    state.put(svr, status(t1Id, 10, t2Id, 10, t3Id, 10));
    Set<KeyExtent> migrations = Collections.emptySet();
    List<TabletMigration> migrationsOut = new ArrayList<>();
    TableLoadBalancer tls = new TableLoadBalancer();
    tls.init(new AccumuloServerContext(inst, confFactory));
    tls.balance(state, migrations, migrationsOut);
    Assert.assertEquals(0, migrationsOut.size());
    state.put(mkts("10.0.0.2", "0x02030405"), status());
    tls = new TableLoadBalancer();
    tls.init(new AccumuloServerContext(inst, confFactory));
    tls.balance(state, migrations, migrationsOut);
    int count = 0;
    Map<Table.ID, Integer> movedByTable = new HashMap<>();
    movedByTable.put(Table.ID.of(t1Id), 0);
    movedByTable.put(Table.ID.of(t2Id), 0);
    movedByTable.put(Table.ID.of(t3Id), 0);
    for (TabletMigration migration : migrationsOut) {
        if (migration.oldServer.equals(svr))
            count++;
        Table.ID key = migration.tablet.getTableId();
        movedByTable.put(key, movedByTable.get(key) + 1);
    }
    Assert.assertEquals(15, count);
    for (Integer moved : movedByTable.values()) {
        Assert.assertEquals(5, moved.intValue());
    }
}

Example 9 with TableConfiguration

use of org.apache.accumulo.server.conf.TableConfiguration in project accumulo by apache.

the class LookupTask method run.

@Override
public void run() {
    MultiScanSession session = (MultiScanSession) server.getSession(scanID);
    String oldThreadName = Thread.currentThread().getName();
    try {
        if (isCancelled() || session == null)
            return;
        TableConfiguration acuTableConf = server.getTableConfiguration(session.threadPoolExtent);
        long maxResultsSize = acuTableConf.getAsBytes(Property.TABLE_SCAN_MAXMEM);
        runState.set(ScanRunState.RUNNING);
        Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Table: ");
        long bytesAdded = 0;
        long maxScanTime = 4000;
        long startTime = System.currentTimeMillis();
        List<KVEntry> results = new ArrayList<>();
        Map<KeyExtent, List<Range>> failures = new HashMap<>();
        List<KeyExtent> fullScans = new ArrayList<>();
        KeyExtent partScan = null;
        Key partNextKey = null;
        boolean partNextKeyInclusive = false;
        Iterator<Entry<KeyExtent, List<Range>>> iter = session.queries.entrySet().iterator();
        // check the time so that the read ahead thread is not monopolized
        while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
            Entry<KeyExtent, List<Range>> entry = iter.next();
            iter.remove();
            // check that tablet server is serving requested tablet
            Tablet tablet = server.getOnlineTablet(entry.getKey());
            if (tablet == null) {
                failures.put(entry.getKey(), entry.getValue());
                continue;
            }
            Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Tablet: " + entry.getKey().toString());
            LookupResult lookupResult;
            try {
                // canceled
                if (isCancelled())
                    interruptFlag.set(true);
                lookupResult = tablet.lookup(entry.getValue(), session.columnSet, session.auths, results, maxResultsSize - bytesAdded, session.ssiList, session.ssio, interruptFlag, session.samplerConfig, session.batchTimeOut, session.context);
                // if the tablet was closed it it possible that the
                // interrupt flag was set.... do not want it set for
                // the next
                // lookup
                interruptFlag.set(false);
            } catch (IOException e) {
                log.warn("lookup failed for tablet " + entry.getKey(), e);
                throw new RuntimeException(e);
            }
            bytesAdded += lookupResult.bytesAdded;
            if (lookupResult.unfinishedRanges.size() > 0) {
                if (lookupResult.closed) {
                    failures.put(entry.getKey(), lookupResult.unfinishedRanges);
                } else {
                    session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
                    partScan = entry.getKey();
                    partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
                    partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
                }
            } else {
                fullScans.add(entry.getKey());
            }
        }
        long finishTime = System.currentTimeMillis();
        session.totalLookupTime += (finishTime - startTime);
        session.numEntries += results.size();
        // convert everything to thrift before adding result
        List<TKeyValue> retResults = new ArrayList<>();
        for (KVEntry entry : results) retResults.add(new TKeyValue(entry.getKey().toThrift(), ByteBuffer.wrap(entry.getValue().get())));
        Map<TKeyExtent, List<TRange>> retFailures = Translator.translate(failures, Translators.KET, new Translator.ListTranslator<>(Translators.RT));
        List<TKeyExtent> retFullScans = Translator.translate(fullScans, Translators.KET);
        TKeyExtent retPartScan = null;
        TKey retPartNextKey = null;
        if (partScan != null) {
            retPartScan = partScan.toThrift();
            retPartNextKey = partNextKey.toThrift();
        }
        // add results to queue
        addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, session.queries.size() != 0));
    } catch (IterationInterruptedException iie) {
        if (!isCancelled()) {
            log.warn("Iteration interrupted, when scan not cancelled", iie);
            addResult(iie);
        }
    } catch (SampleNotPresentException e) {
        addResult(e);
    } catch (Throwable e) {
        log.warn("exception while doing multi-scan ", e);
        addResult(e);
    } finally {
        Thread.currentThread().setName(oldThreadName);
        runState.set(ScanRunState.FINISHED);
    }
}

Example 10 with TableConfiguration

use of org.apache.accumulo.server.conf.TableConfiguration in project accumulo by apache.

the class TabletTest method correctValuesSetForProperties.

@Test
public void correctValuesSetForProperties() {
    TableConfiguration tableConf = EasyMock.createMock(TableConfiguration.class);
    CompactionPlan plan = EasyMock.createMock(CompactionPlan.class);
    WriteParameters writeParams = EasyMock.createMock(WriteParameters.class);
    plan.writeParameters = writeParams;
    DatafileManager dfm = EasyMock.createMock(DatafileManager.class);
    TabletTime time = EasyMock.createMock(TabletTime.class);
    TabletServer tserver = EasyMock.createMock(TabletServer.class);
    TabletResourceManager tserverResourceManager = EasyMock.createMock(TabletResourceManager.class);
    TabletMemory tabletMemory = EasyMock.createMock(TabletMemory.class);
    KeyExtent extent = EasyMock.createMock(KeyExtent.class);
    ConfigurationObserver obs = EasyMock.createMock(ConfigurationObserver.class);
    Tablet tablet = new Tablet(time, "", 0, new Path("/foo"), dfm, tserver, tserverResourceManager, tabletMemory, tableConf, extent, obs);
    long hdfsBlockSize = 10000l, blockSize = 5000l, indexBlockSize = 500l;
    int replication = 5;
    String compressType = "snappy";
    EasyMock.expect(tableConf.iterator()).andReturn(Collections.emptyIterator());
    EasyMock.expect(writeParams.getHdfsBlockSize()).andReturn(hdfsBlockSize).times(2);
    EasyMock.expect(writeParams.getBlockSize()).andReturn(blockSize).times(2);
    EasyMock.expect(writeParams.getIndexBlockSize()).andReturn(indexBlockSize).times(2);
    EasyMock.expect(writeParams.getCompressType()).andReturn(compressType).times(2);
    EasyMock.expect(writeParams.getReplication()).andReturn(replication).times(2);
    EasyMock.replay(tableConf, plan, writeParams);
    AccumuloConfiguration aConf = tablet.createTableConfiguration(tableConf, plan);
    EasyMock.verify(tableConf, plan, writeParams);
    Assert.assertEquals(hdfsBlockSize, Long.parseLong(aConf.get(Property.TABLE_FILE_BLOCK_SIZE)));
    Assert.assertEquals(blockSize, Long.parseLong(aConf.get(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE)));
    Assert.assertEquals(indexBlockSize, Long.parseLong(aConf.get(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX)));
    Assert.assertEquals(compressType, aConf.get(Property.TABLE_FILE_COMPRESSION_TYPE));
    Assert.assertEquals(replication, Integer.parseInt(aConf.get(Property.TABLE_FILE_REPLICATION)));
}