Examples with TServerInstance org.apache.accumulo.server.master.state.TServerInstance used on opensource projects

Example 51 with TServerInstance

use of org.apache.accumulo.server.master.state.TServerInstance in project accumulo by apache.

the class HostRegexTableLoadBalancerTest method testUnassignedWithNoTServers.

@Test
public void testUnassignedWithNoTServers() {
    init(new AccumuloServerContext(instance, factory));
    Map<KeyExtent, TServerInstance> assignments = new HashMap<>();
    Map<KeyExtent, TServerInstance> unassigned = new HashMap<>();
    for (KeyExtent ke : tableExtents.get(BAR.getTableName())) {
        unassigned.put(ke, null);
    }
    SortedMap<TServerInstance, TabletServerStatus> current = createCurrent(15);
    // Remove the BAR tablet servers from current
    List<TServerInstance> removals = new ArrayList<>();
    for (Entry<TServerInstance, TabletServerStatus> e : current.entrySet()) {
        if (e.getKey().host().equals("192.168.0.6") || e.getKey().host().equals("192.168.0.7") || e.getKey().host().equals("192.168.0.8") || e.getKey().host().equals("192.168.0.9") || e.getKey().host().equals("192.168.0.10")) {
            removals.add(e.getKey());
        }
    }
    for (TServerInstance r : removals) {
        current.remove(r);
    }
    this.getAssignments(Collections.unmodifiableSortedMap(allTabletServers), Collections.unmodifiableMap(unassigned), assignments);
    Assert.assertEquals(unassigned.size(), assignments.size());
    // Ensure assignments are correct
    for (Entry<KeyExtent, TServerInstance> e : assignments.entrySet()) {
        if (!tabletInBounds(e.getKey(), e.getValue())) {
            Assert.fail("tablet not in bounds: " + e.getKey() + " -> " + e.getValue().host());
        }
    }
}

Example 52 with TServerInstance

use of org.apache.accumulo.server.master.state.TServerInstance in project accumulo by apache.

the class TableLoadBalancerTest method test.

@Test
public void test() throws Exception {
    final Instance inst = EasyMock.createMock(Instance.class);
    EasyMock.expect(inst.getInstanceID()).andReturn(UUID.nameUUIDFromBytes(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0 }).toString()).anyTimes();
    EasyMock.expect(inst.getZooKeepers()).andReturn("10.0.0.1:1234").anyTimes();
    EasyMock.expect(inst.getZooKeepersSessionTimeOut()).andReturn(30_000).anyTimes();
    EasyMock.replay(inst);
    ServerConfigurationFactory confFactory = new ServerConfigurationFactory(inst) {

        @Override
        public TableConfiguration getTableConfiguration(Table.ID tableId) {
            // create a dummy namespaceConfiguration to satisfy requireNonNull in TableConfiguration constructor
            NamespaceConfiguration dummyConf = new NamespaceConfiguration(null, inst, null);
            return new TableConfiguration(inst, tableId, dummyConf) {

                @Override
                public String get(Property property) {
                    // fake the get table configuration so the test doesn't try to look in zookeeper for per-table classpath stuff
                    return DefaultConfiguration.getInstance().get(property);
                }
            };
        }
    };
    String t1Id = TABLE_ID_MAP.get("t1"), t2Id = TABLE_ID_MAP.get("t2"), t3Id = TABLE_ID_MAP.get("t3");
    state = new TreeMap<>();
    TServerInstance svr = mkts("10.0.0.1", "0x01020304");
    state.put(svr, status(t1Id, 10, t2Id, 10, t3Id, 10));
    Set<KeyExtent> migrations = Collections.emptySet();
    List<TabletMigration> migrationsOut = new ArrayList<>();
    TableLoadBalancer tls = new TableLoadBalancer();
    tls.init(new AccumuloServerContext(inst, confFactory));
    tls.balance(state, migrations, migrationsOut);
    Assert.assertEquals(0, migrationsOut.size());
    state.put(mkts("10.0.0.2", "0x02030405"), status());
    tls = new TableLoadBalancer();
    tls.init(new AccumuloServerContext(inst, confFactory));
    tls.balance(state, migrations, migrationsOut);
    int count = 0;
    Map<Table.ID, Integer> movedByTable = new HashMap<>();
    movedByTable.put(Table.ID.of(t1Id), 0);
    movedByTable.put(Table.ID.of(t2Id), 0);
    movedByTable.put(Table.ID.of(t3Id), 0);
    for (TabletMigration migration : migrationsOut) {
        if (migration.oldServer.equals(svr))
            count++;
        Table.ID key = migration.tablet.getTableId();
        movedByTable.put(key, movedByTable.get(key) + 1);
    }
    Assert.assertEquals(15, count);
    for (Integer moved : movedByTable.values()) {
        Assert.assertEquals(5, moved.intValue());
    }
}

Example 53 with TServerInstance

use of org.apache.accumulo.server.master.state.TServerInstance in project accumulo by apache.

the class DatafileManager method bringMajorCompactionOnline.

void bringMajorCompactionOnline(Set<FileRef> oldDatafiles, FileRef tmpDatafile, FileRef newDatafile, Long compactionId, DataFileValue dfv) throws IOException {
    final KeyExtent extent = tablet.getExtent();
    long t1, t2;
    if (!extent.isRootTablet()) {
        if (tablet.getTabletServer().getFileSystem().exists(newDatafile.path())) {
            log.error("Target map file already exist " + newDatafile, new Exception());
            throw new IllegalStateException("Target map file already exist " + newDatafile);
        }
        // rename before putting in metadata table, so files in metadata table should
        // always exist
        rename(tablet.getTabletServer().getFileSystem(), tmpDatafile.path(), newDatafile.path());
        if (dfv.getNumEntries() == 0) {
            tablet.getTabletServer().getFileSystem().deleteRecursively(newDatafile.path());
        }
    }
    TServerInstance lastLocation = null;
    synchronized (tablet) {
        t1 = System.currentTimeMillis();
        IZooReaderWriter zoo = ZooReaderWriter.getInstance();
        tablet.incrementDataSourceDeletions();
        if (extent.isRootTablet()) {
            waitForScansToFinish(oldDatafiles, true, Long.MAX_VALUE);
            try {
                if (!zoo.isLockHeld(tablet.getTabletServer().getLock().getLockID())) {
                    throw new IllegalStateException();
                }
            } catch (Exception e) {
                throw new IllegalStateException("Can not bring major compaction online, lock not held", e);
            }
            // mark files as ready for deletion, but
            // do not delete them until we successfully
            // rename the compacted map file, in case
            // the system goes down
            RootFiles.replaceFiles(tablet.getTableConfiguration(), tablet.getTabletServer().getFileSystem(), tablet.getLocation(), oldDatafiles, tmpDatafile, newDatafile);
        }
        // atomically remove old files and add new file
        for (FileRef oldDatafile : oldDatafiles) {
            if (!datafileSizes.containsKey(oldDatafile)) {
                log.error("file does not exist in set {}", oldDatafile);
            }
            datafileSizes.remove(oldDatafile);
            majorCompactingFiles.remove(oldDatafile);
        }
        if (datafileSizes.containsKey(newDatafile)) {
            log.error("Adding file that is already in set {}", newDatafile);
        }
        if (dfv.getNumEntries() > 0) {
            datafileSizes.put(newDatafile, dfv);
        }
        // could be used by a follow on compaction in a multipass compaction
        majorCompactingFiles.add(newDatafile);
        tablet.computeNumEntries();
        lastLocation = tablet.resetLastLocation();
        tablet.setLastCompactionID(compactionId);
        t2 = System.currentTimeMillis();
    }
    if (!extent.isRootTablet()) {
        Set<FileRef> filesInUseByScans = waitForScansToFinish(oldDatafiles, false, 10000);
        if (filesInUseByScans.size() > 0)
            log.debug("Adding scan refs to metadata {} {}", extent, filesInUseByScans);
        MasterMetadataUtil.replaceDatafiles(tablet.getTabletServer(), extent, oldDatafiles, filesInUseByScans, newDatafile, compactionId, dfv, tablet.getTabletServer().getClientAddressString(), lastLocation, tablet.getTabletServer().getLock());
        removeFilesAfterScan(filesInUseByScans);
    }
    log.debug(String.format("MajC finish lock %.2f secs", (t2 - t1) / 1000.0));
    log.debug("TABLET_HIST {} MajC  --> {}", oldDatafiles, newDatafile);
}

Example 54 with TServerInstance

use of org.apache.accumulo.server.master.state.TServerInstance in project accumulo by apache.

the class Tablet method resetLastLocation.

public TServerInstance resetLastLocation() {
    TServerInstance result = lastLocation;
    lastLocation = null;
    return result;
}

Example 55 with TServerInstance

use of org.apache.accumulo.server.master.state.TServerInstance in project accumulo by apache.

the class CheckTabletMetadataTest method testBadTabletMetadata.

@Test
public void testBadTabletMetadata() throws Exception {
    KeyExtent ke = new KeyExtent(Table.ID.of("1"), null, null);
    TreeMap<Key, Value> tabletMeta = new TreeMap<>();
    put(tabletMeta, "1<", TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN, KeyExtent.encodePrevEndRow(null).get());
    put(tabletMeta, "1<", TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN, "/t1".getBytes());
    put(tabletMeta, "1<", TabletsSection.ServerColumnFamily.TIME_COLUMN, "M0".getBytes());
    put(tabletMeta, "1<", TabletsSection.FutureLocationColumnFamily.NAME, "4", "127.0.0.1:9997");
    TServerInstance tsi = new TServerInstance("127.0.0.1:9997", 4);
    Assert.assertNotNull(TabletServer.checkTabletMetadata(ke, tsi, tabletMeta, ke.getMetadataEntry()));
    assertFail(tabletMeta, ke, new TServerInstance("127.0.0.1:9998", 4));
    assertFail(tabletMeta, ke, new TServerInstance("127.0.0.1:9998", 5));
    assertFail(tabletMeta, ke, new TServerInstance("127.0.0.1:9997", 5));
    assertFail(tabletMeta, ke, new TServerInstance("127.0.0.2:9997", 4));
    assertFail(tabletMeta, ke, new TServerInstance("127.0.0.2:9997", 5));
    assertFail(tabletMeta, new KeyExtent(Table.ID.of("1"), null, new Text("m")), tsi);
    assertFail(tabletMeta, new KeyExtent(Table.ID.of("1"), new Text("r"), new Text("m")), tsi);
    assertFail(tabletMeta, ke, tsi, newKey("1<", TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN));
    assertFail(tabletMeta, ke, tsi, newKey("1<", TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN));
    assertFail(tabletMeta, ke, tsi, newKey("1<", TabletsSection.ServerColumnFamily.TIME_COLUMN));
    assertFail(tabletMeta, ke, tsi, newKey("1<", TabletsSection.FutureLocationColumnFamily.NAME, "4"));
    TreeMap<Key, Value> copy = new TreeMap<>(tabletMeta);
    put(copy, "1<", TabletsSection.CurrentLocationColumnFamily.NAME, "4", "127.0.0.1:9997");
    assertFail(copy, ke, tsi);
    assertFail(copy, ke, tsi, newKey("1<", TabletsSection.FutureLocationColumnFamily.NAME, "4"));
    copy = new TreeMap<>(tabletMeta);
    put(copy, "1<", TabletsSection.CurrentLocationColumnFamily.NAME, "5", "127.0.0.1:9998");
    assertFail(copy, ke, tsi);
    put(copy, "1<", TabletsSection.CurrentLocationColumnFamily.NAME, "6", "127.0.0.1:9999");
    assertFail(copy, ke, tsi);
    copy = new TreeMap<>(tabletMeta);
    put(copy, "1<", TabletsSection.FutureLocationColumnFamily.NAME, "5", "127.0.0.1:9998");
    assertFail(copy, ke, tsi);
    assertFail(new TreeMap<>(), ke, tsi);
}