Examples with TServerInstance org.apache.accumulo.core.metadata.TServerInstance used on opensource projects

Example 76 with TServerInstance

use of org.apache.accumulo.core.metadata.TServerInstance in project accumulo by apache.

the class TabletServer method markUnusedWALs.

private void markUnusedWALs() {
    List<DfsLogger> closedCopy;
    synchronized (closedLogs) {
        closedCopy = copyClosedLogs(closedLogs);
    }
    ReferencedRemover refRemover = candidates -> {
        for (Tablet tablet : getOnlineTablets().values()) {
            tablet.removeInUseLogs(candidates);
            if (candidates.isEmpty()) {
                break;
            }
        }
    };
    Set<DfsLogger> eligible = findOldestUnreferencedWals(closedCopy, refRemover);
    try {
        TServerInstance session = this.getTabletSession();
        for (DfsLogger candidate : eligible) {
            log.info("Marking " + candidate.getPath() + " as unreferenced");
            walMarker.walUnreferenced(session, candidate.getPath());
        }
        synchronized (closedLogs) {
            closedLogs.removeAll(eligible);
        }
    } catch (WalMarkerException ex) {
        log.info(ex.toString(), ex);
    }
}

Example 77 with TServerInstance

use of org.apache.accumulo.core.metadata.TServerInstance in project accumulo by apache.

the class Upgrader9to10 method upgradeRootTabletMetadata.

/**
 * Improvements to the metadata and root tables were made in this version. See pull request
 * <a href="https://github.com/apache/accumulo/pull/1174">#1174</a> for more details.
 */
private void upgradeRootTabletMetadata(ServerContext context) {
    String rootMetaSer = getFromZK(context, ZROOT_TABLET);
    if (rootMetaSer == null || rootMetaSer.isEmpty()) {
        String dir = getFromZK(context, ZROOT_TABLET_PATH);
        List<LogEntry> logs = getRootLogEntries(context);
        TServerInstance last = getLocation(context, ZROOT_TABLET_LAST_LOCATION);
        TServerInstance future = getLocation(context, ZROOT_TABLET_FUTURE_LOCATION);
        TServerInstance current = getLocation(context, ZROOT_TABLET_LOCATION);
        UpgradeMutator tabletMutator = new UpgradeMutator(context);
        tabletMutator.putPrevEndRow(RootTable.EXTENT.prevEndRow());
        tabletMutator.putDirName(upgradeDirColumn(dir));
        if (last != null)
            tabletMutator.putLocation(last, LocationType.LAST);
        if (future != null)
            tabletMutator.putLocation(future, LocationType.FUTURE);
        if (current != null)
            tabletMutator.putLocation(current, LocationType.CURRENT);
        logs.forEach(tabletMutator::putWal);
        Map<String, DataFileValue> files = cleanupRootTabletFiles(context.getVolumeManager(), dir);
        files.forEach((path, dfv) -> tabletMutator.putFile(new TabletFile(new Path(path)), dfv));
        tabletMutator.putTime(computeRootTabletTime(context, files.keySet()));
        tabletMutator.mutate();
    }
    try {
        context.getZooReaderWriter().putPersistentData(context.getZooKeeperRoot() + ZROOT_TABLET_GC_CANDIDATES, new RootGcCandidates().toJson().getBytes(UTF_8), NodeExistsPolicy.SKIP);
    } catch (KeeperException | InterruptedException e) {
        throw new RuntimeException(e);
    }
    // this operation must be idempotent, so deleting after updating is very important
    delete(context, ZROOT_TABLET_CURRENT_LOGS);
    delete(context, ZROOT_TABLET_FUTURE_LOCATION);
    delete(context, ZROOT_TABLET_LAST_LOCATION);
    delete(context, ZROOT_TABLET_LOCATION);
    delete(context, ZROOT_TABLET_WALOGS);
    delete(context, ZROOT_TABLET_PATH);
}

Example 78 with TServerInstance

use of org.apache.accumulo.core.metadata.TServerInstance in project accumulo by apache.

the class ManagerReplicationCoordinatorTest method randomServerFromMany.

@Test
public void randomServerFromMany() {
    Manager manager = EasyMock.createMock(Manager.class);
    ZooReader reader = EasyMock.createMock(ZooReader.class);
    ServerContext context = EasyMock.createMock(ServerContext.class);
    EasyMock.expect(context.getConfiguration()).andReturn(config).anyTimes();
    EasyMock.expect(context.getInstanceID()).andReturn(InstanceId.of("1234")).anyTimes();
    EasyMock.expect(context.getZooReaderWriter()).andReturn(null).anyTimes();
    EasyMock.expect(manager.getInstanceID()).andReturn(InstanceId.of("1234")).anyTimes();
    EasyMock.expect(manager.getContext()).andReturn(context).anyTimes();
    EasyMock.replay(manager, context, reader);
    ManagerReplicationCoordinator coordinator = new ManagerReplicationCoordinator(manager, reader);
    EasyMock.verify(manager, reader);
    TreeSet<TServerInstance> instances = new TreeSet<>();
    TServerInstance inst1 = new TServerInstance(HostAndPort.fromParts("host1", 1234), "session");
    instances.add(inst1);
    TServerInstance inst2 = new TServerInstance(HostAndPort.fromParts("host2", 1234), "session");
    instances.add(inst2);
    assertEquals(inst1, coordinator.getRandomTServer(instances, 0));
    assertEquals(inst2, coordinator.getRandomTServer(instances, 1));
}

Example 79 with TServerInstance

use of org.apache.accumulo.core.metadata.TServerInstance in project accumulo by apache.

the class UnusedWALIT method getWALCount.

private int getWALCount(ServerContext context) throws Exception {
    WalStateManager wals = new WalStateManager(context);
    int result = 0;
    for (Entry<TServerInstance, List<UUID>> entry : wals.getAllMarkers().entrySet()) {
        result += entry.getValue().size();
    }
    return result;
}

Example 80 with TServerInstance

use of org.apache.accumulo.core.metadata.TServerInstance in project accumulo by apache.

the class SplitRecoveryIT method splitPartiallyAndRecover.

private void splitPartiallyAndRecover(ServerContext context, KeyExtent extent, KeyExtent high, KeyExtent low, double splitRatio, SortedMap<StoredTabletFile, DataFileValue> mapFiles, Text midRow, String location, int steps, ServiceLock zl) throws Exception {
    SortedMap<StoredTabletFile, DataFileValue> lowDatafileSizes = new TreeMap<>();
    SortedMap<StoredTabletFile, DataFileValue> highDatafileSizes = new TreeMap<>();
    List<StoredTabletFile> highDatafilesToRemove = new ArrayList<>();
    MetadataTableUtil.splitDatafiles(midRow, splitRatio, new HashMap<>(), mapFiles, lowDatafileSizes, highDatafileSizes, highDatafilesToRemove);
    MetadataTableUtil.splitTablet(high, extent.prevEndRow(), splitRatio, context, zl, Set.of());
    TServerInstance instance = new TServerInstance(location, zl.getSessionId());
    Assignment assignment = new Assignment(high, instance);
    TabletMutator tabletMutator = context.getAmple().mutateTablet(extent);
    tabletMutator.putLocation(assignment.server, LocationType.FUTURE);
    tabletMutator.mutate();
    if (steps >= 1) {
        Map<Long, List<TabletFile>> bulkFiles = getBulkFilesLoaded(context, high);
        ManagerMetadataUtil.addNewTablet(context, low, "lowDir", instance, lowDatafileSizes, bulkFiles, new MetadataTime(0, TimeType.LOGICAL), -1L, -1L, zl);
    }
    if (steps >= 2) {
        MetadataTableUtil.finishSplit(high, highDatafileSizes, highDatafilesToRemove, context, zl);
    }
    TabletMetadata meta = context.getAmple().readTablet(high);
    KeyExtent fixedExtent = ManagerMetadataUtil.fixSplit(context, meta, zl);
    if (steps < 2)
        assertEquals(splitRatio, meta.getSplitRatio(), 0.0);
    if (steps >= 1) {
        assertEquals(high, fixedExtent);
        ensureTabletHasNoUnexpectedMetadataEntries(context, low, lowDatafileSizes);
        ensureTabletHasNoUnexpectedMetadataEntries(context, high, highDatafileSizes);
        Map<Long, ? extends Collection<TabletFile>> lowBulkFiles = getBulkFilesLoaded(context, low);
        Map<Long, ? extends Collection<TabletFile>> highBulkFiles = getBulkFilesLoaded(context, high);
        if (!lowBulkFiles.equals(highBulkFiles)) {
            throw new Exception(" " + lowBulkFiles + " != " + highBulkFiles + " " + low + " " + high);
        }
        if (lowBulkFiles.isEmpty()) {
            throw new Exception(" no bulk files " + low);
        }
    } else {
        assertEquals(extent, fixedExtent);
        ensureTabletHasNoUnexpectedMetadataEntries(context, extent, mapFiles);
    }
}