Examples with ServerContext org.apache.accumulo.server.ServerContext used on opensource projects

Example 16 with ServerContext

use of org.apache.accumulo.server.ServerContext in project accumulo by apache.

the class HostRegexTableLoadBalancerReconfigurationTest method testConfigurationChanges.

@Test
public void testConfigurationChanges() {
    ServerContext context1 = createMockContext();
    replay(context1);
    final TestServerConfigurationFactory factory = new TestServerConfigurationFactory(context1);
    ServerContext context2 = createMockContext();
    expect(context2.getConfiguration()).andReturn(factory.getSystemConfiguration()).anyTimes();
    expect(context2.getTableConfiguration(FOO.getId())).andReturn(factory.getTableConfiguration(FOO.getId())).anyTimes();
    expect(context2.getTableConfiguration(BAR.getId())).andReturn(factory.getTableConfiguration(BAR.getId())).anyTimes();
    expect(context2.getTableConfiguration(BAZ.getId())).andReturn(factory.getTableConfiguration(BAZ.getId())).anyTimes();
    replay(context2);
    init(context2);
    Map<KeyExtent, TServerInstance> unassigned = new HashMap<>();
    for (List<KeyExtent> extents : tableExtents.values()) {
        for (KeyExtent ke : extents) {
            unassigned.put(ke, null);
        }
    }
    this.getAssignments(Collections.unmodifiableSortedMap(allTabletServers), Collections.unmodifiableMap(unassigned), assignments);
    assertEquals(15, assignments.size());
    // Ensure unique tservers
    for (Entry<KeyExtent, TServerInstance> e : assignments.entrySet()) {
        for (Entry<KeyExtent, TServerInstance> e2 : assignments.entrySet()) {
            if (e.getKey().equals(e2.getKey())) {
                continue;
            }
            if (e.getValue().equals(e2.getValue())) {
                fail("Assignment failure. " + e.getKey() + " and " + e2.getKey() + " are assigned to the same host: " + e.getValue());
            }
        }
    }
    // Ensure assignments are correct
    for (Entry<KeyExtent, TServerInstance> e : assignments.entrySet()) {
        if (!tabletInBounds(e.getKey(), e.getValue())) {
            fail("tablet not in bounds: " + e.getKey() + " -> " + e.getValue().getHost());
        }
    }
    Set<KeyExtent> migrations = new HashSet<>();
    List<TabletMigration> migrationsOut = new ArrayList<>();
    // Wait to trigger the out of bounds check which will call our version of
    // getOnlineTabletsForTable
    UtilWaitThread.sleep(3000);
    this.balance(Collections.unmodifiableSortedMap(allTabletServers), migrations, migrationsOut);
    assertEquals(0, migrationsOut.size());
    // Change property, simulate call by TableConfWatcher
    ((ConfigurationCopy) factory.getSystemConfiguration()).set(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "r01.*");
    // Wait to trigger the out of bounds check and the repool check
    UtilWaitThread.sleep(10000);
    this.balance(Collections.unmodifiableSortedMap(allTabletServers), migrations, migrationsOut);
    assertEquals(5, migrationsOut.size());
    for (TabletMigration migration : migrationsOut) {
        assertTrue(migration.newServer.getHost().startsWith("192.168.0.1") || migration.newServer.getHost().startsWith("192.168.0.2") || migration.newServer.getHost().startsWith("192.168.0.3") || migration.newServer.getHost().startsWith("192.168.0.4") || migration.newServer.getHost().startsWith("192.168.0.5"));
    }
}

Example 17 with ServerContext

use of org.apache.accumulo.server.ServerContext in project accumulo by apache.

the class BulkImporterTest method testFindOverlappingTablets.

@Test
public void testFindOverlappingTablets() throws Exception {
    MockTabletLocator locator = new MockTabletLocator();
    FileSystem fs = FileSystem.getLocal(new Configuration());
    ServerContext context = MockServerContext.get();
    EasyMock.replay(context);
    String file = "target/testFile.rf";
    fs.delete(new Path(file), true);
    FileSKVWriter writer = FileOperations.getInstance().newWriterBuilder().forFile(file, fs, fs.getConf(), CryptoServiceFactory.newDefaultInstance()).withTableConfiguration(context.getConfiguration()).build();
    writer.startDefaultLocalityGroup();
    Value empty = new Value();
    writer.append(new Key("a", "cf", "cq"), empty);
    writer.append(new Key("a", "cf", "cq1"), empty);
    writer.append(new Key("a", "cf", "cq2"), empty);
    writer.append(new Key("a", "cf", "cq3"), empty);
    writer.append(new Key("a", "cf", "cq4"), empty);
    writer.append(new Key("a", "cf", "cq5"), empty);
    writer.append(new Key("d", "cf", "cq"), empty);
    writer.append(new Key("d", "cf", "cq1"), empty);
    writer.append(new Key("d", "cf", "cq2"), empty);
    writer.append(new Key("d", "cf", "cq3"), empty);
    writer.append(new Key("d", "cf", "cq4"), empty);
    writer.append(new Key("d", "cf", "cq5"), empty);
    writer.append(new Key("dd", "cf", "cq1"), empty);
    writer.append(new Key("ichabod", "cf", "cq"), empty);
    writer.append(new Key("icky", "cf", "cq1"), empty);
    writer.append(new Key("iffy", "cf", "cq2"), empty);
    writer.append(new Key("internal", "cf", "cq3"), empty);
    writer.append(new Key("is", "cf", "cq4"), empty);
    writer.append(new Key("iterator", "cf", "cq5"), empty);
    writer.append(new Key("xyzzy", "cf", "cq"), empty);
    writer.close();
    try (var vm = VolumeManagerImpl.getLocalForTesting("file:///")) {
        List<TabletLocation> overlaps = BulkImporter.findOverlappingTablets(context, vm, locator, new Path(file));
        assertEquals(5, overlaps.size());
        Collections.sort(overlaps);
        assertEquals(new KeyExtent(tableId, new Text("a"), null), overlaps.get(0).tablet_extent);
        assertEquals(new KeyExtent(tableId, new Text("d"), new Text("cm")), overlaps.get(1).tablet_extent);
        assertEquals(new KeyExtent(tableId, new Text("dm"), new Text("d")), overlaps.get(2).tablet_extent);
        assertEquals(new KeyExtent(tableId, new Text("j"), new Text("i")), overlaps.get(3).tablet_extent);
        assertEquals(new KeyExtent(tableId, null, new Text("l")), overlaps.get(4).tablet_extent);
        List<TabletLocation> overlaps2 = BulkImporter.findOverlappingTablets(context, vm, locator, new Path(file), new KeyExtent(tableId, new Text("h"), new Text("b")));
        assertEquals(3, overlaps2.size());
        assertEquals(new KeyExtent(tableId, new Text("d"), new Text("cm")), overlaps2.get(0).tablet_extent);
        assertEquals(new KeyExtent(tableId, new Text("dm"), new Text("d")), overlaps2.get(1).tablet_extent);
        assertEquals(new KeyExtent(tableId, new Text("j"), new Text("i")), overlaps2.get(2).tablet_extent);
        assertEquals(locator.invalidated, 1);
    }
}

Example 18 with ServerContext

use of org.apache.accumulo.server.ServerContext in project accumulo by apache.

the class Upgrader9to10 method getOldCandidates.

/**
 * Return path of the file from old delete markers
 */
private Iterator<String> getOldCandidates(ServerContext context, String tableName) throws TableNotFoundException {
    Range range = DeletesSection.getRange();
    Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY);
    scanner.setRange(range);
    return StreamSupport.stream(scanner.spliterator(), false).filter(entry -> !entry.getValue().equals(UPGRADED)).map(entry -> entry.getKey().getRow().toString().substring(OLD_DELETE_PREFIX.length())).iterator();
}

Example 19 with ServerContext

use of org.apache.accumulo.server.ServerContext in project accumulo by apache.

the class Monitor method fetchData.

public void fetchData() {
    ServerContext context = getContext();
    double totalIngestRate = 0.;
    double totalIngestByteRate = 0.;
    double totalQueryRate = 0.;
    double totalQueryByteRate = 0.;
    double totalScanRate = 0.;
    long totalEntries = 0;
    int totalTabletCount = 0;
    long totalHoldTime = 0;
    long totalLookups = 0;
    boolean retry = true;
    // only recalc every so often
    long currentTime = System.currentTimeMillis();
    if (currentTime - lastRecalc.get() < REFRESH_TIME * 1000) {
        return;
    }
    // try to begin fetching; return if unsuccessful (because another thread is already fetching)
    if (!fetching.compareAndSet(false, true)) {
        return;
    }
    // Otherwise, we'll never release the lock by unsetting 'fetching' in the the finally block
    try {
        while (retry) {
            ManagerClientService.Iface client = null;
            try {
                client = ManagerClient.getConnection(context);
                if (client != null) {
                    mmi = client.getManagerStats(TraceUtil.traceInfo(), context.rpcCreds());
                    retry = false;
                } else {
                    mmi = null;
                    log.error("Unable to get info from Manager");
                }
                gcStatus = fetchGcStatus();
            } catch (Exception e) {
                mmi = null;
                log.info("Error fetching stats: ", e);
            } finally {
                if (client != null) {
                    ManagerClient.close(client, context);
                }
            }
            if (mmi == null) {
                sleepUninterruptibly(1, TimeUnit.SECONDS);
            }
        }
        if (mmi != null) {
            int majorCompactions = 0;
            int minorCompactions = 0;
            lookupRateTracker.startingUpdates();
            indexCacheHitTracker.startingUpdates();
            indexCacheRequestTracker.startingUpdates();
            dataCacheHitTracker.startingUpdates();
            dataCacheRequestTracker.startingUpdates();
            for (TabletServerStatus server : mmi.tServerInfo) {
                TableInfo summary = TableInfoUtil.summarizeTableStats(server);
                totalIngestRate += summary.ingestRate;
                totalIngestByteRate += summary.ingestByteRate;
                totalQueryRate += summary.queryRate;
                totalScanRate += summary.scanRate;
                totalQueryByteRate += summary.queryByteRate;
                totalEntries += summary.recs;
                totalHoldTime += server.holdTime;
                totalLookups += server.lookups;
                majorCompactions += summary.majors.running;
                minorCompactions += summary.minors.running;
                lookupRateTracker.updateTabletServer(server.name, server.lastContact, server.lookups);
                indexCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheHits);
                indexCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheRequest);
                dataCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheHits);
                dataCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheRequest);
            }
            lookupRateTracker.finishedUpdating();
            indexCacheHitTracker.finishedUpdating();
            indexCacheRequestTracker.finishedUpdating();
            dataCacheHitTracker.finishedUpdating();
            dataCacheRequestTracker.finishedUpdating();
            int totalTables = 0;
            for (TableInfo tInfo : mmi.tableMap.values()) {
                totalTabletCount += tInfo.tablets;
                totalTables++;
            }
            this.totalIngestRate = totalIngestRate;
            this.totalTables = totalTables;
            totalIngestByteRate = totalIngestByteRate / 1000000.0;
            this.totalQueryRate = totalQueryRate;
            this.totalScanRate = totalScanRate;
            totalQueryByteRate = totalQueryByteRate / 1000000.0;
            this.totalEntries = totalEntries;
            this.totalTabletCount = totalTabletCount;
            this.totalHoldTime = totalHoldTime;
            this.totalLookups = totalLookups;
            ingestRateOverTime.add(new Pair<>(currentTime, totalIngestRate));
            ingestByteRateOverTime.add(new Pair<>(currentTime, totalIngestByteRate));
            double totalLoad = 0.;
            for (TabletServerStatus status : mmi.tServerInfo) {
                if (status != null) {
                    totalLoad += status.osLoad;
                }
            }
            loadOverTime.add(new Pair<>(currentTime, totalLoad));
            minorCompactionsOverTime.add(new Pair<>(currentTime, minorCompactions));
            majorCompactionsOverTime.add(new Pair<>(currentTime, majorCompactions));
            lookupsOverTime.add(new Pair<>(currentTime, lookupRateTracker.calculateRate()));
            queryRateOverTime.add(new Pair<>(currentTime, (long) totalQueryRate));
            queryByteRateOverTime.add(new Pair<>(currentTime, totalQueryByteRate));
            scanRateOverTime.add(new Pair<>(currentTime, (long) totalScanRate));
            calcCacheHitRate(indexCacheHitRateOverTime, currentTime, indexCacheHitTracker, indexCacheRequestTracker);
            calcCacheHitRate(dataCacheHitRateOverTime, currentTime, dataCacheHitTracker, dataCacheRequestTracker);
        }
        try {
            this.problemSummary = ProblemReports.getInstance(context).summarize();
            this.problemException = null;
        } catch (Exception e) {
            log.info("Failed to obtain problem reports ", e);
            this.problemSummary = Collections.emptyMap();
            this.problemException = e;
        }
        // check for compaction coordinator host and only notify its discovery
        Optional<HostAndPort> previousHost;
        if (System.nanoTime() - coordinatorCheckNanos > fetchTimeNanos) {
            previousHost = coordinatorHost;
            coordinatorHost = ExternalCompactionUtil.findCompactionCoordinator(context);
            coordinatorCheckNanos = System.nanoTime();
            if (previousHost.isEmpty() && coordinatorHost.isPresent())
                log.info("External Compaction Coordinator found at {}", coordinatorHost.get());
        }
    } finally {
        if (coordinatorClient != null) {
            ThriftUtil.returnClient(coordinatorClient, context);
            coordinatorClient = null;
        }
        lastRecalc.set(currentTime);
        // stop fetching; log an error if this thread wasn't already fetching
        if (!fetching.compareAndSet(true, false)) {
            throw new AssertionError("Not supposed to happen; somebody broke this code");
        }
    }
}

Example 20 with ServerContext

use of org.apache.accumulo.server.ServerContext in project accumulo by apache.

the class Monitor method fetchGcStatus.

private GCStatus fetchGcStatus() {
    ServerContext context = getContext();
    GCStatus result = null;
    HostAndPort address = null;
    try {
        // Read the gc location from its lock
        ZooReaderWriter zk = context.getZooReaderWriter();
        var path = ServiceLock.path(context.getZooKeeperRoot() + Constants.ZGC_LOCK);
        List<String> locks = ServiceLock.validateAndSort(path, zk.getChildren(path.toString()));
        if (locks != null && !locks.isEmpty()) {
            address = new ServerServices(new String(zk.getData(path + "/" + locks.get(0)), UTF_8)).getAddress(Service.GC_CLIENT);
            GCMonitorService.Client client = ThriftUtil.getClient(new GCMonitorService.Client.Factory(), address, context);
            try {
                result = client.getStatus(TraceUtil.traceInfo(), context.rpcCreds());
            } finally {
                ThriftUtil.returnClient(client, context);
            }
        }
    } catch (Exception ex) {
        log.warn("Unable to contact the garbage collector at " + address, ex);
    }
    return result;
}