Examples with ServerConfigurationFactory org.apache.accumulo.server.conf.ServerConfigurationFactory used on opensource projects

Example 16 with ServerConfigurationFactory

use of org.apache.accumulo.server.conf.ServerConfigurationFactory in project accumulo by apache.

the class LargestFirstMemoryManagerTest method test.

@Test
public void test() throws Exception {
    LargestFirstMemoryManagerUnderTest mgr = new LargestFirstMemoryManagerUnderTest();
    ServerConfiguration config = new ServerConfiguration() {

        ServerConfigurationFactory delegate = new ServerConfigurationFactory(inst);

        @Override
        public AccumuloConfiguration getSystemConfiguration() {
            SiteConfiguration conf = SiteConfiguration.getInstance();
            conf.set(Property.TSERV_MAXMEM, "1g");
            return conf;
        }

        @Override
        public TableConfiguration getTableConfiguration(Table.ID tableId) {
            return delegate.getTableConfiguration(tableId);
        }

        @Override
        public NamespaceConfiguration getNamespaceConfiguration(Namespace.ID namespaceId) {
            return delegate.getNamespaceConfiguration(namespaceId);
        }
    };
    mgr.init(config);
    MemoryManagementActions result;
    // nothing to do
    result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1000, 0), t(k("y"), ZERO, 2000, 0)));
    assertEquals(0, result.tabletsToMinorCompact.size());
    // one tablet is really big
    result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, ONE_GIG, 0), t(k("y"), ZERO, 2000, 0)));
    assertEquals(1, result.tabletsToMinorCompact.size());
    assertEquals(k("x"), result.tabletsToMinorCompact.get(0));
    // one tablet is idle
    mgr.currentTime = LATER;
    result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER, 2000, 0)));
    assertEquals(1, result.tabletsToMinorCompact.size());
    assertEquals(k("x"), result.tabletsToMinorCompact.get(0));
    // one tablet is idle, but one is really big
    result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER, ONE_GIG, 0)));
    assertEquals(1, result.tabletsToMinorCompact.size());
    assertEquals(k("y"), result.tabletsToMinorCompact.get(0));
    // lots of work to do
    mgr = new LargestFirstMemoryManagerUnderTest();
    mgr.init(config);
    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0), t(k("d"), ZERO, HALF_GIG + 3, 0), t(k("e"), ZERO, HALF_GIG + 4, 0), t(k("f"), ZERO, HALF_GIG + 5, 0), t(k("g"), ZERO, HALF_GIG + 6, 0), t(k("h"), ZERO, HALF_GIG + 7, 0), t(k("i"), ZERO, HALF_GIG + 8, 0)));
    assertEquals(2, result.tabletsToMinorCompact.size());
    assertEquals(k("i"), result.tabletsToMinorCompact.get(0));
    assertEquals(k("h"), result.tabletsToMinorCompact.get(1));
    // one finished, one in progress, one filled up
    mgr = new LargestFirstMemoryManagerUnderTest();
    mgr.init(config);
    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0), t(k("d"), ZERO, HALF_GIG + 3, 0), t(k("e"), ZERO, HALF_GIG + 4, 0), t(k("f"), ZERO, HALF_GIG + 5, 0), t(k("g"), ZERO, ONE_GIG, 0), t(k("h"), ZERO, 0, HALF_GIG + 7), t(k("i"), ZERO, 0, 0)));
    assertEquals(1, result.tabletsToMinorCompact.size());
    assertEquals(k("g"), result.tabletsToMinorCompact.get(0));
    // memory is very full, lots of candidates
    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0), t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO, ONE_GIG + 5, 0), t(k("g"), ZERO, ONE_GIG + 6, 0), t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0)));
    assertEquals(2, result.tabletsToMinorCompact.size());
    assertEquals(k("g"), result.tabletsToMinorCompact.get(0));
    assertEquals(k("f"), result.tabletsToMinorCompact.get(1));
    // only have two compactors, still busy
    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0), t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO, ONE_GIG, ONE_GIG + 5), t(k("g"), ZERO, ONE_GIG, ONE_GIG + 6), t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0)));
    assertEquals(0, result.tabletsToMinorCompact.size());
    // finished one
    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0), t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO, ONE_GIG, ONE_GIG + 5), t(k("g"), ZERO, ONE_GIG, 0), t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0)));
    assertEquals(1, result.tabletsToMinorCompact.size());
    assertEquals(k("e"), result.tabletsToMinorCompact.get(0));
    // many are running: do nothing
    mgr = new LargestFirstMemoryManagerUnderTest();
    mgr.init(config);
    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0), t(k("d"), ZERO, 0, HALF_GIG), t(k("e"), ZERO, 0, HALF_GIG), t(k("f"), ZERO, 0, HALF_GIG), t(k("g"), ZERO, 0, HALF_GIG), t(k("i"), ZERO, 0, HALF_GIG), t(k("j"), ZERO, 0, HALF_GIG), t(k("k"), ZERO, 0, HALF_GIG), t(k("l"), ZERO, 0, HALF_GIG), t(k("m"), ZERO, 0, HALF_GIG)));
    assertEquals(0, result.tabletsToMinorCompact.size());
    // observe adjustment:
    mgr = new LargestFirstMemoryManagerUnderTest();
    mgr.init(config);
    // compact the largest
    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, QGIG + 2, 0)));
    assertEquals(1, result.tabletsToMinorCompact.size());
    assertEquals(k("c"), result.tabletsToMinorCompact.get(0));
    // show that it is compacting... do nothing
    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, 0, QGIG + 2)));
    assertEquals(0, result.tabletsToMinorCompact.size());
    // not going to bother compacting any more
    mgr.currentTime += ONE_MINUTE;
    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, 0, QGIG + 2)));
    assertEquals(0, result.tabletsToMinorCompact.size());
    // now do nothing
    mgr.currentTime += ONE_MINUTE;
    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, 0, 0), t(k("c"), ZERO, 0, 0)));
    assertEquals(0, result.tabletsToMinorCompact.size());
    // on no! more data, this time we compact because we've adjusted
    mgr.currentTime += ONE_MINUTE;
    result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, 0, 0)));
    assertEquals(1, result.tabletsToMinorCompact.size());
    assertEquals(k("b"), result.tabletsToMinorCompact.get(0));
}

Example 17 with ServerConfigurationFactory

use of org.apache.accumulo.server.conf.ServerConfigurationFactory in project accumulo by apache.

the class LargestFirstMemoryManagerTest method testDeletedTable.

@Test
public void testDeletedTable() throws Exception {
    final String deletedTableId = "1";
    Function<Table.ID, Boolean> existenceCheck = tableId -> !deletedTableId.contentEquals(tableId.canonicalID());
    LargestFirstMemoryManagerWithExistenceCheck mgr = new LargestFirstMemoryManagerWithExistenceCheck(existenceCheck);
    ServerConfiguration config = new ServerConfiguration() {

        ServerConfigurationFactory delegate = new ServerConfigurationFactory(inst);

        @Override
        public AccumuloConfiguration getSystemConfiguration() {
            return DefaultConfiguration.getInstance();
        }

        @Override
        public TableConfiguration getTableConfiguration(Table.ID tableId) {
            return delegate.getTableConfiguration(tableId);
        }

        @Override
        public NamespaceConfiguration getNamespaceConfiguration(Namespace.ID namespaceId) {
            return delegate.getNamespaceConfiguration(namespaceId);
        }
    };
    mgr.init(config);
    MemoryManagementActions result;
    // one tablet is really big and the other is for a nonexistent table
    KeyExtent extent = new KeyExtent(Table.ID.of("2"), new Text("j"), null);
    result = mgr.getMemoryManagementActions(tablets(t(extent, ZERO, ONE_GIG, 0), t(k("j"), ZERO, ONE_GIG, 0)));
    assertEquals(1, result.tabletsToMinorCompact.size());
    assertEquals(extent, result.tabletsToMinorCompact.get(0));
}

Example 18 with ServerConfigurationFactory

use of org.apache.accumulo.server.conf.ServerConfigurationFactory in project accumulo by apache.

the class WrongTabletTest method main.

public static void main(String[] args) {
    final Opts opts = new Opts();
    opts.parseArgs(WrongTabletTest.class.getName(), args);
    final HostAndPort location = HostAndPort.fromString(opts.location);
    final Instance inst = opts.getInstance();
    final ServerConfigurationFactory conf = new ServerConfigurationFactory(inst);
    final ClientContext context = new AccumuloServerContext(inst, conf) {

        @Override
        public synchronized Credentials getCredentials() {
            try {
                return new Credentials(opts.getPrincipal(), opts.getToken());
            } catch (AccumuloSecurityException e) {
                throw new RuntimeException(e);
            }
        }
    };
    try {
        TabletClientService.Iface client = ThriftUtil.getTServerClient(location, context);
        Mutation mutation = new Mutation(new Text("row_0003750001"));
        mutation.putDelete(new Text("colf"), new Text("colq"));
        client.update(Tracer.traceInfo(), context.rpcCreds(), new KeyExtent(Table.ID.of("!!"), null, new Text("row_0003750000")).toThrift(), mutation.toThrift(), TDurability.DEFAULT);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 19 with ServerConfigurationFactory

use of org.apache.accumulo.server.conf.ServerConfigurationFactory in project accumulo by apache.

the class BulkFileIT method testBulkFile.

@Test
public void testBulkFile() throws Exception {
    Connector c = getConnector();
    String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    SortedSet<Text> splits = new TreeSet<>();
    for (String split : "0333 0666 0999 1333 1666".split(" ")) splits.add(new Text(split));
    c.tableOperations().addSplits(tableName, splits);
    Configuration conf = new Configuration();
    AccumuloConfiguration aconf = new ServerConfigurationFactory(c.getInstance()).getSystemConfiguration();
    FileSystem fs = getCluster().getFileSystem();
    String rootPath = cluster.getTemporaryPath().toString();
    String dir = rootPath + "/bulk_test_diff_files_89723987592_" + getUniqueNames(1)[0];
    fs.delete(new Path(dir), true);
    FileSKVWriter writer1 = FileOperations.getInstance().newWriterBuilder().forFile(dir + "/f1." + RFile.EXTENSION, fs, conf).withTableConfiguration(aconf).build();
    writer1.startDefaultLocalityGroup();
    writeData(writer1, 0, 333);
    writer1.close();
    FileSKVWriter writer2 = FileOperations.getInstance().newWriterBuilder().forFile(dir + "/f2." + RFile.EXTENSION, fs, conf).withTableConfiguration(aconf).build();
    writer2.startDefaultLocalityGroup();
    writeData(writer2, 334, 999);
    writer2.close();
    FileSKVWriter writer3 = FileOperations.getInstance().newWriterBuilder().forFile(dir + "/f3." + RFile.EXTENSION, fs, conf).withTableConfiguration(aconf).build();
    writer3.startDefaultLocalityGroup();
    writeData(writer3, 1000, 1999);
    writer3.close();
    FunctionalTestUtils.bulkImport(c, fs, tableName, dir);
    FunctionalTestUtils.checkRFiles(c, tableName, 6, 6, 1, 1);
    verifyData(tableName, 0, 1999);
}

Example 20 with ServerConfigurationFactory

use of org.apache.accumulo.server.conf.ServerConfigurationFactory in project accumulo by apache.

the class TotalQueuedIT method getSyncs.

private long getSyncs() throws Exception {
    Connector c = getConnector();
    ServerConfigurationFactory confFactory = new ServerConfigurationFactory(c.getInstance());
    AccumuloServerContext context = new AccumuloServerContext(c.getInstance(), confFactory);
    for (String address : c.instanceOperations().getTabletServers()) {
        TabletClientService.Client client = ThriftUtil.getTServerClient(HostAndPort.fromString(address), context);
        TabletServerStatus status = client.getTabletServerStatus(null, context.rpcCreds());
        return status.syncs;
    }
    return 0;
}