Examples with TServerInstance org.apache.accumulo.server.master.state.TServerInstance used on opensource projects

Example 56 with TServerInstance

use of org.apache.accumulo.server.master.state.TServerInstance in project accumulo by apache.

the class MasterMetadataUtil method getUpdateForTabletDataFile.

/**
 * Create an update that updates a tablet
 *
 * @return A Mutation to update a tablet from the given information
 */
private static Mutation getUpdateForTabletDataFile(KeyExtent extent, FileRef path, FileRef mergeFile, DataFileValue dfv, String time, Set<FileRef> filesInUseByScans, String address, ZooLock zooLock, Set<String> unusedWalLogs, TServerInstance lastLocation, long flushId) {
    Mutation m = new Mutation(extent.getMetadataEntry());
    if (dfv.getNumEntries() > 0) {
        m.put(DataFileColumnFamily.NAME, path.meta(), new Value(dfv.encode()));
        TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(UTF_8)));
        // stuff in this location
        TServerInstance self = getTServerInstance(address, zooLock);
        self.putLastLocation(m);
        // erase the old location
        if (lastLocation != null && !lastLocation.equals(self))
            lastLocation.clearLastLocation(m);
    }
    if (unusedWalLogs != null) {
        for (String entry : unusedWalLogs) {
            m.putDelete(LogColumnFamily.NAME, new Text(entry));
        }
    }
    for (FileRef scanFile : filesInUseByScans) m.put(ScanFileColumnFamily.NAME, scanFile.meta(), new Value(new byte[0]));
    if (mergeFile != null)
        m.putDelete(DataFileColumnFamily.NAME, mergeFile.meta());
    TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(m, new Value(Long.toString(flushId).getBytes(UTF_8)));
    return m;
}

Example 57 with TServerInstance

use of org.apache.accumulo.server.master.state.TServerInstance in project accumulo by apache.

the class ChaoticLoadBalancerTest method testUnevenAssignment.

@Test
public void testUnevenAssignment() {
    servers.clear();
    for (char c : "abcdefghijklmnopqrstuvwxyz".toCharArray()) {
        String cString = Character.toString(c);
        HostAndPort fakeAddress = HostAndPort.fromParts("127.0.0.1", c);
        String fakeInstance = cString;
        TServerInstance tsi = new TServerInstance(fakeAddress, fakeInstance);
        FakeTServer fakeTServer = new FakeTServer();
        servers.put(tsi, fakeTServer);
        fakeTServer.extents.add(makeExtent(cString, null, null));
    }
    // Put more tablets on one server, but not more than the number of servers
    Entry<TServerInstance, FakeTServer> first = servers.entrySet().iterator().next();
    first.getValue().extents.add(makeExtent("newTable", "a", null));
    first.getValue().extents.add(makeExtent("newTable", "b", "a"));
    first.getValue().extents.add(makeExtent("newTable", "c", "b"));
    first.getValue().extents.add(makeExtent("newTable", "d", "c"));
    first.getValue().extents.add(makeExtent("newTable", "e", "d"));
    first.getValue().extents.add(makeExtent("newTable", "f", "e"));
    first.getValue().extents.add(makeExtent("newTable", "g", "f"));
    first.getValue().extents.add(makeExtent("newTable", "h", "g"));
    first.getValue().extents.add(makeExtent("newTable", "i", null));
    TestChaoticLoadBalancer balancer = new TestChaoticLoadBalancer();
    Set<KeyExtent> migrations = Collections.emptySet();
    // Just want to make sure it gets some migrations, randomness prevents guarantee of a defined amount, or even expected amount
    List<TabletMigration> migrationsOut = new ArrayList<>();
    while (migrationsOut.size() != 0) {
        balancer.balance(getAssignments(servers), migrations, migrationsOut);
    }
}

Example 58 with TServerInstance

use of org.apache.accumulo.server.master.state.TServerInstance in project accumulo by apache.

the class DefaultLoadBalancerTest method testUnevenAssignment.

@Test
public void testUnevenAssignment() {
    for (char c : "abcdefghijklmnopqrstuvwxyz".toCharArray()) {
        String cString = Character.toString(c);
        HostAndPort fakeAddress = HostAndPort.fromParts("127.0.0.1", c);
        String fakeInstance = cString;
        TServerInstance tsi = new TServerInstance(fakeAddress, fakeInstance);
        FakeTServer fakeTServer = new FakeTServer();
        servers.put(tsi, fakeTServer);
        fakeTServer.extents.add(makeExtent(cString, null, null));
    }
    // Put more tablets on one server, but not more than the number of servers
    Entry<TServerInstance, FakeTServer> first = servers.entrySet().iterator().next();
    first.getValue().extents.add(makeExtent("newTable", "a", null));
    first.getValue().extents.add(makeExtent("newTable", "b", "a"));
    first.getValue().extents.add(makeExtent("newTable", "c", "b"));
    first.getValue().extents.add(makeExtent("newTable", "d", "c"));
    first.getValue().extents.add(makeExtent("newTable", "e", "d"));
    first.getValue().extents.add(makeExtent("newTable", "f", "e"));
    first.getValue().extents.add(makeExtent("newTable", "g", "f"));
    first.getValue().extents.add(makeExtent("newTable", "h", "g"));
    first.getValue().extents.add(makeExtent("newTable", "i", null));
    TestDefaultLoadBalancer balancer = new TestDefaultLoadBalancer();
    Set<KeyExtent> migrations = Collections.emptySet();
    int moved = 0;
    // balance until we can't balance no more!
    while (true) {
        List<TabletMigration> migrationsOut = new ArrayList<>();
        balancer.balance(getAssignments(servers), migrations, migrationsOut);
        if (migrationsOut.size() == 0)
            break;
        for (TabletMigration migration : migrationsOut) {
            if (servers.get(migration.oldServer).extents.remove(migration.tablet))
                moved++;
            servers.get(migration.newServer).extents.add(migration.tablet);
        }
    }
    assertEquals(8, moved);
}

Example 59 with TServerInstance

use of org.apache.accumulo.server.master.state.TServerInstance in project accumulo by apache.

the class ShutdownTServerTest method testSingleShutdown.

@Test
public void testSingleShutdown() throws Exception {
    final TServerInstance tserver = EasyMock.createMock(TServerInstance.class);
    final boolean force = false;
    final ShutdownTServer op = new ShutdownTServer(tserver, force);
    final Master master = EasyMock.createMock(Master.class);
    final long tid = 1l;
    final TServerConnection tserverCnxn = EasyMock.createMock(TServerConnection.class);
    final TabletServerStatus status = new TabletServerStatus();
    status.tableMap = new HashMap<>();
    // Put in a table info record, don't care what
    status.tableMap.put("a_table", new TableInfo());
    master.shutdownTServer(tserver);
    EasyMock.expectLastCall().once();
    EasyMock.expect(master.onlineTabletServers()).andReturn(Collections.singleton(tserver));
    EasyMock.expect(master.getConnection(tserver)).andReturn(tserverCnxn);
    EasyMock.expect(tserverCnxn.getTableMap(false)).andReturn(status);
    EasyMock.replay(tserver, tserverCnxn, master);
    // FATE op is not ready
    long wait = op.isReady(tid, master);
    assertTrue("Expected wait to be greater than 0", wait > 0);
    EasyMock.verify(tserver, tserverCnxn, master);
    // Reset the mocks
    EasyMock.reset(tserver, tserverCnxn, master);
    // The same as above, but should not expect call shutdownTServer on master again
    EasyMock.expect(master.onlineTabletServers()).andReturn(Collections.singleton(tserver));
    EasyMock.expect(master.getConnection(tserver)).andReturn(tserverCnxn);
    EasyMock.expect(tserverCnxn.getTableMap(false)).andReturn(status);
    EasyMock.replay(tserver, tserverCnxn, master);
    // FATE op is not ready
    wait = op.isReady(tid, master);
    assertTrue("Expected wait to be greater than 0", wait > 0);
    EasyMock.verify(tserver, tserverCnxn, master);
}

Example 60 with TServerInstance

use of org.apache.accumulo.server.master.state.TServerInstance in project accumulo by apache.

the class LoadFiles method call.

@Override
public Repo<Master> call(final long tid, final Master master) throws Exception {
    master.updateBulkImportStatus(source, BulkImportState.LOADING);
    ExecutorService executor = getThreadPool(master);
    final AccumuloConfiguration conf = master.getConfiguration();
    VolumeManager fs = master.getFileSystem();
    List<FileStatus> files = new ArrayList<>();
    for (FileStatus entry : fs.listStatus(new Path(bulk))) {
        files.add(entry);
    }
    log.debug("tid " + tid + " importing " + files.size() + " files");
    Path writable = new Path(this.errorDir, ".iswritable");
    if (!fs.createNewFile(writable)) {
        // Maybe this is a re-try... clear the flag and try again
        fs.delete(writable);
        if (!fs.createNewFile(writable))
            throw new AcceptableThriftTableOperationException(tableId.canonicalID(), null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, "Unable to write to " + this.errorDir);
    }
    fs.delete(writable);
    final Set<String> filesToLoad = Collections.synchronizedSet(new HashSet<String>());
    for (FileStatus f : files) filesToLoad.add(f.getPath().toString());
    final int RETRIES = Math.max(1, conf.getCount(Property.MASTER_BULK_RETRIES));
    for (int attempt = 0; attempt < RETRIES && filesToLoad.size() > 0; attempt++) {
        List<Future<List<String>>> results = new ArrayList<>();
        if (master.onlineTabletServers().size() == 0)
            log.warn("There are no tablet server to process bulk import, waiting (tid = " + tid + ")");
        while (master.onlineTabletServers().size() == 0) {
            sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
        }
        // Use the threadpool to assign files one-at-a-time to the server
        final List<String> loaded = Collections.synchronizedList(new ArrayList<String>());
        final Random random = new Random();
        final TServerInstance[] servers;
        String prop = conf.get(Property.MASTER_BULK_TSERVER_REGEX);
        if (null == prop || "".equals(prop)) {
            servers = master.onlineTabletServers().toArray(new TServerInstance[0]);
        } else {
            Pattern regex = Pattern.compile(prop);
            List<TServerInstance> subset = new ArrayList<>();
            master.onlineTabletServers().forEach(t -> {
                if (regex.matcher(t.host()).matches()) {
                    subset.add(t);
                }
            });
            if (0 == subset.size()) {
                log.warn("There are no tablet servers online that match supplied regex: {}", conf.get(Property.MASTER_BULK_TSERVER_REGEX));
            }
            servers = subset.toArray(new TServerInstance[0]);
        }
        if (servers.length > 0) {
            for (final String file : filesToLoad) {
                results.add(executor.submit(new Callable<List<String>>() {

                    @Override
                    public List<String> call() {
                        List<String> failures = new ArrayList<>();
                        ClientService.Client client = null;
                        HostAndPort server = null;
                        try {
                            // get a connection to a random tablet server, do not prefer cached connections because
                            // this is running on the master and there are lots of connections to tablet servers
                            // serving the metadata tablets
                            long timeInMillis = master.getConfiguration().getTimeInMillis(Property.MASTER_BULK_TIMEOUT);
                            // Pair<String,Client> pair = ServerClient.getConnection(master, false, timeInMillis);
                            server = servers[random.nextInt(servers.length)].getLocation();
                            client = ThriftUtil.getTServerClient(server, master, timeInMillis);
                            List<String> attempt = Collections.singletonList(file);
                            log.debug("Asking " + server + " to bulk import " + file);
                            List<String> fail = client.bulkImportFiles(Tracer.traceInfo(), master.rpcCreds(), tid, tableId.canonicalID(), attempt, errorDir, setTime);
                            if (fail.isEmpty()) {
                                loaded.add(file);
                            } else {
                                failures.addAll(fail);
                            }
                        } catch (Exception ex) {
                            log.error("rpc failed server:" + server + ", tid:" + tid + " " + ex);
                        } finally {
                            ThriftUtil.returnClient(client);
                        }
                        return failures;
                    }
                }));
            }
        }
        Set<String> failures = new HashSet<>();
        for (Future<List<String>> f : results) failures.addAll(f.get());
        filesToLoad.removeAll(loaded);
        if (filesToLoad.size() > 0) {
            log.debug("tid " + tid + " attempt " + (attempt + 1) + " " + sampleList(filesToLoad, 10) + " failed");
            sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
        }
    }
    FSDataOutputStream failFile = fs.create(new Path(errorDir, BulkImport.FAILURES_TXT), true);
    try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(failFile, UTF_8))) {
        for (String f : filesToLoad) {
            out.write(f);
            out.write("\n");
        }
    }
    // return the next step, which will perform cleanup
    return new CompleteBulkImport(tableId, source, bulk, errorDir);
}