use of org.apache.accumulo.server.conf.TableConfiguration in project accumulo by apache.
the class HostRegexTableLoadBalancerTest method testSplitCurrentByRegexUsingOverlappingPools.
@Test
public void testSplitCurrentByRegexUsingOverlappingPools() {
init(new AccumuloServerContext(instance, new TestServerConfigurationFactory(instance) {
@Override
public TableConfiguration getTableConfiguration(Table.ID tableId) {
NamespaceConfiguration defaultConf = new NamespaceConfiguration(Namespace.ID.DEFAULT, this.instance, DefaultConfiguration.getInstance());
return new TableConfiguration(instance, tableId, defaultConf) {
HashMap<String, String> tableProperties = new HashMap<>();
{
tableProperties.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + FOO.getTableName(), "r.*");
tableProperties.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "r01.*|r02.*");
}
@Override
public String get(Property property) {
return tableProperties.get(property.name());
}
@Override
public void getProperties(Map<String, String> props, Predicate<String> filter) {
for (Entry<String, String> e : tableProperties.entrySet()) {
if (filter.test(e.getKey())) {
props.put(e.getKey(), e.getValue());
}
}
}
@Override
public long getUpdateCount() {
return 0;
}
};
}
}));
Map<String, SortedMap<TServerInstance, TabletServerStatus>> groups = this.splitCurrentByRegex(createCurrent(15));
Assert.assertEquals(2, groups.size());
Assert.assertTrue(groups.containsKey(FOO.getTableName()));
SortedMap<TServerInstance, TabletServerStatus> fooHosts = groups.get(FOO.getTableName());
Assert.assertEquals(15, fooHosts.size());
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.11:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.12:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.13:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.14:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.15:9997", 1)));
Assert.assertTrue(groups.containsKey(BAR.getTableName()));
SortedMap<TServerInstance, TabletServerStatus> barHosts = groups.get(BAR.getTableName());
Assert.assertEquals(10, barHosts.size());
Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1)));
Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1)));
Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1)));
Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1)));
Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1)));
Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1)));
Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1)));
Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1)));
Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1)));
Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1)));
}
use of org.apache.accumulo.server.conf.TableConfiguration in project accumulo by apache.
the class HostRegexTableLoadBalancerTest method testSplitCurrentByRegexUsingIP.
@Test
public void testSplitCurrentByRegexUsingIP() {
init(new AccumuloServerContext(instance, new TestServerConfigurationFactory(instance) {
@Override
public synchronized AccumuloConfiguration getSystemConfiguration() {
HashMap<String, String> props = new HashMap<>();
props.put(HostRegexTableLoadBalancer.HOST_BALANCER_OOB_CHECK_KEY, "30s");
props.put(HostRegexTableLoadBalancer.HOST_BALANCER_REGEX_USING_IPS_KEY, "true");
return new ConfigurationCopy(props);
}
@Override
public TableConfiguration getTableConfiguration(Table.ID tableId) {
NamespaceConfiguration defaultConf = new NamespaceConfiguration(Namespace.ID.DEFAULT, this.instance, DefaultConfiguration.getInstance());
return new TableConfiguration(instance, tableId, defaultConf) {
HashMap<String, String> tableProperties = new HashMap<>();
{
tableProperties.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + FOO.getTableName(), "192\\.168\\.0\\.[1-5]");
tableProperties.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "192\\.168\\.0\\.[6-9]|192\\.168\\.0\\.10");
}
@Override
public String get(Property property) {
return tableProperties.get(property.name());
}
@Override
public void getProperties(Map<String, String> props, Predicate<String> filter) {
for (Entry<String, String> e : tableProperties.entrySet()) {
if (filter.test(e.getKey())) {
props.put(e.getKey(), e.getValue());
}
}
}
@Override
public long getUpdateCount() {
return 0;
}
};
}
}));
Assert.assertTrue(isIpBasedRegex());
Map<String, SortedMap<TServerInstance, TabletServerStatus>> groups = this.splitCurrentByRegex(createCurrent(15));
Assert.assertEquals(3, groups.size());
Assert.assertTrue(groups.containsKey(FOO.getTableName()));
SortedMap<TServerInstance, TabletServerStatus> fooHosts = groups.get(FOO.getTableName());
Assert.assertEquals(5, fooHosts.size());
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.1:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.2:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.3:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.4:9997", 1)));
Assert.assertTrue(fooHosts.containsKey(new TServerInstance("192.168.0.5:9997", 1)));
Assert.assertTrue(groups.containsKey(BAR.getTableName()));
SortedMap<TServerInstance, TabletServerStatus> barHosts = groups.get(BAR.getTableName());
Assert.assertEquals(5, barHosts.size());
Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.6:9997", 1)));
Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.7:9997", 1)));
Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.8:9997", 1)));
Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.9:9997", 1)));
Assert.assertTrue(barHosts.containsKey(new TServerInstance("192.168.0.10:9997", 1)));
Assert.assertTrue(groups.containsKey(DEFAULT_POOL));
SortedMap<TServerInstance, TabletServerStatus> defHosts = groups.get(DEFAULT_POOL);
Assert.assertEquals(5, defHosts.size());
Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.11:9997", 1)));
Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.12:9997", 1)));
Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.13:9997", 1)));
Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.14:9997", 1)));
Assert.assertTrue(defHosts.containsKey(new TServerInstance("192.168.0.15:9997", 1)));
}
use of org.apache.accumulo.server.conf.TableConfiguration in project accumulo by apache.
the class TableLoadBalancerTest method test.
@Test
public void test() throws Exception {
final Instance inst = EasyMock.createMock(Instance.class);
EasyMock.expect(inst.getInstanceID()).andReturn(UUID.nameUUIDFromBytes(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0 }).toString()).anyTimes();
EasyMock.expect(inst.getZooKeepers()).andReturn("10.0.0.1:1234").anyTimes();
EasyMock.expect(inst.getZooKeepersSessionTimeOut()).andReturn(30_000).anyTimes();
EasyMock.replay(inst);
ServerConfigurationFactory confFactory = new ServerConfigurationFactory(inst) {
@Override
public TableConfiguration getTableConfiguration(Table.ID tableId) {
// create a dummy namespaceConfiguration to satisfy requireNonNull in TableConfiguration constructor
NamespaceConfiguration dummyConf = new NamespaceConfiguration(null, inst, null);
return new TableConfiguration(inst, tableId, dummyConf) {
@Override
public String get(Property property) {
// fake the get table configuration so the test doesn't try to look in zookeeper for per-table classpath stuff
return DefaultConfiguration.getInstance().get(property);
}
};
}
};
String t1Id = TABLE_ID_MAP.get("t1"), t2Id = TABLE_ID_MAP.get("t2"), t3Id = TABLE_ID_MAP.get("t3");
state = new TreeMap<>();
TServerInstance svr = mkts("10.0.0.1", "0x01020304");
state.put(svr, status(t1Id, 10, t2Id, 10, t3Id, 10));
Set<KeyExtent> migrations = Collections.emptySet();
List<TabletMigration> migrationsOut = new ArrayList<>();
TableLoadBalancer tls = new TableLoadBalancer();
tls.init(new AccumuloServerContext(inst, confFactory));
tls.balance(state, migrations, migrationsOut);
Assert.assertEquals(0, migrationsOut.size());
state.put(mkts("10.0.0.2", "0x02030405"), status());
tls = new TableLoadBalancer();
tls.init(new AccumuloServerContext(inst, confFactory));
tls.balance(state, migrations, migrationsOut);
int count = 0;
Map<Table.ID, Integer> movedByTable = new HashMap<>();
movedByTable.put(Table.ID.of(t1Id), 0);
movedByTable.put(Table.ID.of(t2Id), 0);
movedByTable.put(Table.ID.of(t3Id), 0);
for (TabletMigration migration : migrationsOut) {
if (migration.oldServer.equals(svr))
count++;
Table.ID key = migration.tablet.getTableId();
movedByTable.put(key, movedByTable.get(key) + 1);
}
Assert.assertEquals(15, count);
for (Integer moved : movedByTable.values()) {
Assert.assertEquals(5, moved.intValue());
}
}
use of org.apache.accumulo.server.conf.TableConfiguration in project accumulo by apache.
the class LookupTask method run.
@Override
public void run() {
MultiScanSession session = (MultiScanSession) server.getSession(scanID);
String oldThreadName = Thread.currentThread().getName();
try {
if (isCancelled() || session == null)
return;
TableConfiguration acuTableConf = server.getTableConfiguration(session.threadPoolExtent);
long maxResultsSize = acuTableConf.getAsBytes(Property.TABLE_SCAN_MAXMEM);
runState.set(ScanRunState.RUNNING);
Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Table: ");
long bytesAdded = 0;
long maxScanTime = 4000;
long startTime = System.currentTimeMillis();
List<KVEntry> results = new ArrayList<>();
Map<KeyExtent, List<Range>> failures = new HashMap<>();
List<KeyExtent> fullScans = new ArrayList<>();
KeyExtent partScan = null;
Key partNextKey = null;
boolean partNextKeyInclusive = false;
Iterator<Entry<KeyExtent, List<Range>>> iter = session.queries.entrySet().iterator();
// check the time so that the read ahead thread is not monopolized
while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
Entry<KeyExtent, List<Range>> entry = iter.next();
iter.remove();
// check that tablet server is serving requested tablet
Tablet tablet = server.getOnlineTablet(entry.getKey());
if (tablet == null) {
failures.put(entry.getKey(), entry.getValue());
continue;
}
Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Tablet: " + entry.getKey().toString());
LookupResult lookupResult;
try {
// canceled
if (isCancelled())
interruptFlag.set(true);
lookupResult = tablet.lookup(entry.getValue(), session.columnSet, session.auths, results, maxResultsSize - bytesAdded, session.ssiList, session.ssio, interruptFlag, session.samplerConfig, session.batchTimeOut, session.context);
// if the tablet was closed it it possible that the
// interrupt flag was set.... do not want it set for
// the next
// lookup
interruptFlag.set(false);
} catch (IOException e) {
log.warn("lookup failed for tablet " + entry.getKey(), e);
throw new RuntimeException(e);
}
bytesAdded += lookupResult.bytesAdded;
if (lookupResult.unfinishedRanges.size() > 0) {
if (lookupResult.closed) {
failures.put(entry.getKey(), lookupResult.unfinishedRanges);
} else {
session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
partScan = entry.getKey();
partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
}
} else {
fullScans.add(entry.getKey());
}
}
long finishTime = System.currentTimeMillis();
session.totalLookupTime += (finishTime - startTime);
session.numEntries += results.size();
// convert everything to thrift before adding result
List<TKeyValue> retResults = new ArrayList<>();
for (KVEntry entry : results) retResults.add(new TKeyValue(entry.getKey().toThrift(), ByteBuffer.wrap(entry.getValue().get())));
Map<TKeyExtent, List<TRange>> retFailures = Translator.translate(failures, Translators.KET, new Translator.ListTranslator<>(Translators.RT));
List<TKeyExtent> retFullScans = Translator.translate(fullScans, Translators.KET);
TKeyExtent retPartScan = null;
TKey retPartNextKey = null;
if (partScan != null) {
retPartScan = partScan.toThrift();
retPartNextKey = partNextKey.toThrift();
}
// add results to queue
addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, session.queries.size() != 0));
} catch (IterationInterruptedException iie) {
if (!isCancelled()) {
log.warn("Iteration interrupted, when scan not cancelled", iie);
addResult(iie);
}
} catch (SampleNotPresentException e) {
addResult(e);
} catch (Throwable e) {
log.warn("exception while doing multi-scan ", e);
addResult(e);
} finally {
Thread.currentThread().setName(oldThreadName);
runState.set(ScanRunState.FINISHED);
}
}
use of org.apache.accumulo.server.conf.TableConfiguration in project accumulo by apache.
the class TabletTest method correctValuesSetForProperties.
@Test
public void correctValuesSetForProperties() {
TableConfiguration tableConf = EasyMock.createMock(TableConfiguration.class);
CompactionPlan plan = EasyMock.createMock(CompactionPlan.class);
WriteParameters writeParams = EasyMock.createMock(WriteParameters.class);
plan.writeParameters = writeParams;
DatafileManager dfm = EasyMock.createMock(DatafileManager.class);
TabletTime time = EasyMock.createMock(TabletTime.class);
TabletServer tserver = EasyMock.createMock(TabletServer.class);
TabletResourceManager tserverResourceManager = EasyMock.createMock(TabletResourceManager.class);
TabletMemory tabletMemory = EasyMock.createMock(TabletMemory.class);
KeyExtent extent = EasyMock.createMock(KeyExtent.class);
ConfigurationObserver obs = EasyMock.createMock(ConfigurationObserver.class);
Tablet tablet = new Tablet(time, "", 0, new Path("/foo"), dfm, tserver, tserverResourceManager, tabletMemory, tableConf, extent, obs);
long hdfsBlockSize = 10000l, blockSize = 5000l, indexBlockSize = 500l;
int replication = 5;
String compressType = "snappy";
EasyMock.expect(tableConf.iterator()).andReturn(Collections.emptyIterator());
EasyMock.expect(writeParams.getHdfsBlockSize()).andReturn(hdfsBlockSize).times(2);
EasyMock.expect(writeParams.getBlockSize()).andReturn(blockSize).times(2);
EasyMock.expect(writeParams.getIndexBlockSize()).andReturn(indexBlockSize).times(2);
EasyMock.expect(writeParams.getCompressType()).andReturn(compressType).times(2);
EasyMock.expect(writeParams.getReplication()).andReturn(replication).times(2);
EasyMock.replay(tableConf, plan, writeParams);
AccumuloConfiguration aConf = tablet.createTableConfiguration(tableConf, plan);
EasyMock.verify(tableConf, plan, writeParams);
Assert.assertEquals(hdfsBlockSize, Long.parseLong(aConf.get(Property.TABLE_FILE_BLOCK_SIZE)));
Assert.assertEquals(blockSize, Long.parseLong(aConf.get(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE)));
Assert.assertEquals(indexBlockSize, Long.parseLong(aConf.get(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX)));
Assert.assertEquals(compressType, aConf.get(Property.TABLE_FILE_COMPRESSION_TYPE));
Assert.assertEquals(replication, Integer.parseInt(aConf.get(Property.TABLE_FILE_REPLICATION)));
}
Aggregations