Examples with ServerContext org.apache.accumulo.server.ServerContext used on opensource projects

Example 36 with ServerContext

use of org.apache.accumulo.server.ServerContext in project accumulo by apache.

the class ManyWriteAheadLogsIT method testMany.

/**
 * This creates a situation where many tablets reference many different write ahead logs. However
 * not single tablet references a lot of write ahead logs. Want to ensure the tablet server forces
 * minor compactions for this situation.
 */
@Test
public void testMany() throws Exception {
    SortedSet<Text> splits = new TreeSet<>();
    for (int i = 1; i < 100; i++) {
        splits.add(new Text(String.format("%05x", i * 100)));
    }
    ServerContext context = getServerContext();
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        String[] tableNames = getUniqueNames(2);
        String manyWALsTable = tableNames[0];
        String rollWALsTable = tableNames[1];
        NewTableConfiguration ntc = new NewTableConfiguration().withSplits(splits);
        c.tableOperations().create(manyWALsTable, ntc);
        c.tableOperations().create(rollWALsTable);
        Set<String> allWalsSeen = new HashSet<>();
        addOpenWals(context, allWalsSeen);
        try (BatchWriter manyWALsWriter = c.createBatchWriter(manyWALsTable);
            BatchWriter rollWALsWriter = c.createBatchWriter(rollWALsTable)) {
            byte[] val = new byte[768];
            for (int i = 0; i < 100; i++) {
                int startRow = i * 100;
                // write a small amount of data to each tablet in the table
                for (int j = 0; j < 10; j++) {
                    int row = startRow + j;
                    Mutation m = new Mutation(String.format("%05x", row));
                    random.nextBytes(val);
                    m.put("f", "q", "v");
                    manyWALsWriter.addMutation(m);
                }
                manyWALsWriter.flush();
                // write a lot of data to second table to forces the logs to roll
                for (int j = 0; j < 1000; j++) {
                    Mutation m = new Mutation(String.format("%03d", j));
                    random.nextBytes(val);
                    m.put("f", "q", Base64.getEncoder().encodeToString(val));
                    rollWALsWriter.addMutation(m);
                }
                rollWALsWriter.flush();
                // keep track of the open WALs as the test runs. Should see a lot of open WALs over the
                // lifetime of the test, but never a lot at any one time.
                addOpenWals(context, allWalsSeen);
            }
        }
        assertTrue("Number of WALs seen was less than expected " + allWalsSeen.size(), allWalsSeen.size() >= 50);
        // the total number of closed write ahead logs should get small
        int closedLogs = countClosedWals(context);
        while (closedLogs > 3) {
            log.debug("Waiting for wals to shrink " + closedLogs);
            Thread.sleep(250);
            closedLogs = countClosedWals(context);
        }
    }
}

Example 37 with ServerContext

use of org.apache.accumulo.server.ServerContext in project accumulo by apache.

the class CleanZookeeper method main.

/**
 * @param args
 *          must contain one element: the address of a zookeeper node a second parameter provides
 *          an additional authentication value
 */
public static void main(String[] args) {
    Opts opts = new Opts();
    opts.parseArgs(CleanZookeeper.class.getName(), args);
    try (var context = new ServerContext(SiteConfiguration.auto())) {
        String root = Constants.ZROOT;
        ZooReaderWriter zk = context.getZooReaderWriter();
        if (opts.auth != null) {
            ZooUtil.digestAuth(zk.getZooKeeper(), opts.auth);
        }
        for (String child : zk.getChildren(root)) {
            if (Constants.ZINSTANCES.equals("/" + child)) {
                for (String instanceName : zk.getChildren(root + Constants.ZINSTANCES)) {
                    String instanceNamePath = root + Constants.ZINSTANCES + "/" + instanceName;
                    byte[] id = zk.getData(instanceNamePath);
                    if (id != null && !new String(id, UTF_8).equals(context.getInstanceID().canonical())) {
                        try {
                            zk.recursiveDelete(instanceNamePath, NodeMissingPolicy.SKIP);
                        } catch (KeeperException.NoAuthException ex) {
                            log.warn("Unable to delete {}", instanceNamePath);
                        }
                    }
                }
            } else if (!child.equals(context.getInstanceID().canonical())) {
                String path = root + "/" + child;
                try {
                    zk.recursiveDelete(path, NodeMissingPolicy.SKIP);
                } catch (KeeperException.NoAuthException ex) {
                    log.warn("Unable to delete {}", path);
                }
            }
        }
    } catch (Exception ex) {
        System.out.println("Error Occurred: " + ex);
    }
}

Example 38 with ServerContext

use of org.apache.accumulo.server.ServerContext in project accumulo by apache.

the class WebViewsIT method createMocks.

@BeforeClass
public static void createMocks() throws TableNotFoundException {
    ServerContext contextMock = createMock(ServerContext.class);
    expect(contextMock.getConfiguration()).andReturn(DefaultConfiguration.getInstance()).anyTimes();
    expect(contextMock.getInstanceID()).andReturn(InstanceId.of("foo")).atLeastOnce();
    expect(contextMock.getInstanceName()).andReturn("foo").anyTimes();
    expect(contextMock.getZooKeepers()).andReturn("foo:2181").anyTimes();
    expect(contextMock.getTableName(TableId.of("foo"))).andReturn("bar").anyTimes();
    Monitor monitorMock = createMock(Monitor.class);
    expect(monitorMock.getContext()).andReturn(contextMock).anyTimes();
    replay(contextMock, monitorMock);
    monitor.set(monitorMock);
}

Example 39 with ServerContext

use of org.apache.accumulo.server.ServerContext in project accumulo by apache.

the class GetManagerStats method main.

public static void main(String[] args) throws Exception {
    ManagerClientService.Iface client = null;
    ManagerMonitorInfo stats = null;
    var context = new ServerContext(SiteConfiguration.auto());
    while (true) {
        try {
            client = ManagerClient.getConnectionWithRetry(context);
            stats = client.getManagerStats(TraceUtil.traceInfo(), context.rpcCreds());
            break;
        } catch (ThriftNotActiveServiceException e) {
            // Let it loop, fetching a new location
            sleepUninterruptibly(100, MILLISECONDS);
        } finally {
            if (client != null) {
                ManagerClient.close(client, context);
            }
        }
    }
    out(0, "State: " + stats.state.name());
    out(0, "Goal State: " + stats.goalState.name());
    if (stats.serversShuttingDown != null && !stats.serversShuttingDown.isEmpty()) {
        out(0, "Servers to shutdown");
        for (String server : stats.serversShuttingDown) {
            out(1, "%s", server);
        }
    }
    out(0, "Unassigned tablets: %d", stats.unassignedTablets);
    if (stats.badTServers != null && !stats.badTServers.isEmpty()) {
        out(0, "Bad servers");
        for (Entry<String, Byte> entry : stats.badTServers.entrySet()) {
            out(1, "%s: %d", entry.getKey(), (int) entry.getValue());
        }
    }
    out(0, "Dead tablet servers count: %s", stats.deadTabletServers.size());
    for (DeadServer dead : stats.deadTabletServers) {
        out(1, "Dead tablet server: %s", dead.server);
        out(2, "Last report: %s", new SimpleDateFormat().format(new Date(dead.lastStatus)));
        out(2, "Cause: %s", dead.status);
    }
    out(0, "Bulk imports: %s", stats.bulkImports.size());
    for (BulkImportStatus bulk : stats.bulkImports) {
        out(1, "Import directory: %s", bulk.filename);
        out(2, "Bulk state %s", bulk.state);
        out(2, "Bulk start %s", bulk.startTime);
    }
    if (stats.tableMap != null && !stats.tableMap.isEmpty()) {
        out(0, "Tables");
        for (Entry<String, TableInfo> entry : stats.tableMap.entrySet()) {
            TableInfo v = entry.getValue();
            out(1, "%s", entry.getKey());
            out(2, "Records: %d", v.recs);
            out(2, "Records in Memory: %d", v.recsInMemory);
            out(2, "Tablets: %d", v.tablets);
            out(2, "Online Tablets: %d", v.onlineTablets);
            out(2, "Ingest Rate: %.2f", v.ingestRate);
            out(2, "Query Rate: %.2f", v.queryRate);
        }
    }
    if (stats.tServerInfo != null && !stats.tServerInfo.isEmpty()) {
        out(0, "Tablet Servers");
        long now = System.currentTimeMillis();
        for (TabletServerStatus server : stats.tServerInfo) {
            TableInfo summary = TableInfoUtil.summarizeTableStats(server);
            out(1, "Name: %s", server.name);
            out(2, "Ingest: %.2f", summary.ingestRate);
            out(2, "Last Contact: %s", server.lastContact);
            out(2, "OS Load Average: %.2f", server.osLoad);
            out(2, "Queries: %.2f", summary.queryRate);
            out(2, "Time Difference: %.1f", ((now - server.lastContact) / 1000.));
            out(2, "Total Records: %d", summary.recs);
            out(2, "Lookups: %d", server.lookups);
            if (server.holdTime > 0) {
                out(2, "Hold Time: %d", server.holdTime);
            }
            if (server.tableMap != null && !server.tableMap.isEmpty()) {
                out(2, "Tables");
                for (Entry<String, TableInfo> status : server.tableMap.entrySet()) {
                    TableInfo info = status.getValue();
                    out(3, "Table: %s", status.getKey());
                    out(4, "Tablets: %d", info.onlineTablets);
                    out(4, "Records: %d", info.recs);
                    out(4, "Records in Memory: %d", info.recsInMemory);
                    out(4, "Ingest: %.2f", info.ingestRate);
                    out(4, "Queries: %.2f", info.queryRate);
                    out(4, "Major Compacting: %d", info.majors == null ? 0 : info.majors.running);
                    out(4, "Queued for Major Compaction: %d", info.majors == null ? 0 : info.majors.queued);
                    out(4, "Minor Compacting: %d", info.minors == null ? 0 : info.minors.running);
                    out(4, "Queued for Minor Compaction: %d", info.minors == null ? 0 : info.minors.queued);
                }
            }
            out(2, "Recoveries: %d", server.logSorts.size());
            for (RecoveryStatus sort : server.logSorts) {
                out(3, "File: %s", sort.name);
                out(3, "Progress: %.2f%%", sort.progress * 100);
                out(3, "Time running: %s", sort.runtime / 1000.);
            }
            out(3, "Bulk imports: %s", stats.bulkImports.size());
            for (BulkImportStatus bulk : stats.bulkImports) {
                out(4, "Import file: %s", bulk.filename);
                out(5, "Bulk state %s", bulk.state);
                out(5, "Bulk start %s", bulk.startTime);
            }
        }
    }
}

Example 40 with ServerContext

use of org.apache.accumulo.server.ServerContext in project accumulo by apache.

the class MetaConstraintRetryIT method test.

// a test for ACCUMULO-3096
@Test
public void test() throws Exception {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
        client.securityOperations().grantTablePermission(getAdminPrincipal(), MetadataTable.NAME, TablePermission.WRITE);
        ServerContext context = getServerContext();
        Writer w = new Writer(context, MetadataTable.ID);
        KeyExtent extent = new KeyExtent(TableId.of("5"), null, null);
        Mutation m = new Mutation(extent.toMetaRow());
        // unknown columns should cause constraint violation
        m.put("badcolfam", "badcolqual", "3");
        var e = assertThrows(RuntimeException.class, () -> MetadataTableUtil.update(context, w, null, m, extent));
        assertEquals(ConstraintViolationException.class, e.getCause().getClass());
    }
}