Examples with ExternalCompactionId org.apache.accumulo.core.metadata.schema.ExternalCompactionId used on opensource projects

Example 11 with ExternalCompactionId

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionId in project accumulo by apache.

the class Tablet method split.

public TreeMap<KeyExtent, TabletData> split(byte[] sp) throws IOException {
    if (sp != null && extent.endRow() != null && extent.endRow().equals(new Text(sp))) {
        throw new IllegalArgumentException("Attempting to split on EndRow " + extent.endRow() + " for " + extent);
    }
    if (sp != null && sp.length > tableConfiguration.getAsBytes(Property.TABLE_MAX_END_ROW_SIZE)) {
        String msg = "Cannot split tablet " + extent + ", selected split point too long.  Length :  " + sp.length;
        log.warn(msg);
        throw new IOException(msg);
    }
    if (extent.isRootTablet()) {
        String msg = "Cannot split root tablet";
        log.warn(msg);
        throw new RuntimeException(msg);
    }
    try {
        initiateClose(true);
    } catch (IllegalStateException ise) {
        log.debug("File {} not splitting : {}", extent, ise.getMessage());
        return null;
    }
    // obtain this info outside of synch block since it will involve opening
    // the map files... it is ok if the set of map files changes, because
    // this info is used for optimization... it is ok if map files are missing
    // from the set... can still query and insert into the tablet while this
    // map file operation is happening
    Map<TabletFile, FileUtil.FileInfo> firstAndLastRows = FileUtil.tryToGetFirstAndLastRows(context, getDatafileManager().getFiles());
    synchronized (this) {
        // java needs tuples ...
        TreeMap<KeyExtent, TabletData> newTablets = new TreeMap<>();
        long t1 = System.currentTimeMillis();
        // choose a split point
        SplitRowSpec splitPoint;
        if (sp == null) {
            splitPoint = findSplitRow(getDatafileManager().getFiles());
        } else {
            Text tsp = new Text(sp);
            splitPoint = new SplitRowSpec(FileUtil.estimatePercentageLTE(context, chooseTabletDir(), extent.prevEndRow(), extent.endRow(), getDatafileManager().getFiles(), tsp), tsp);
        }
        if (splitPoint == null || splitPoint.row == null) {
            log.info("had to abort split because splitRow was null");
            closeState = CloseState.OPEN;
            return null;
        }
        closeState = CloseState.CLOSING;
        completeClose(true, false);
        Text midRow = splitPoint.row;
        double splitRatio = splitPoint.splitRatio;
        KeyExtent low = new KeyExtent(extent.tableId(), midRow, extent.prevEndRow());
        KeyExtent high = new KeyExtent(extent.tableId(), extent.endRow(), midRow);
        String lowDirectoryName = createTabletDirectoryName(context, midRow);
        // write new tablet information to MetadataTable
        SortedMap<StoredTabletFile, DataFileValue> lowDatafileSizes = new TreeMap<>();
        SortedMap<StoredTabletFile, DataFileValue> highDatafileSizes = new TreeMap<>();
        List<StoredTabletFile> highDatafilesToRemove = new ArrayList<>();
        MetadataTableUtil.splitDatafiles(midRow, splitRatio, firstAndLastRows, getDatafileManager().getDatafileSizes(), lowDatafileSizes, highDatafileSizes, highDatafilesToRemove);
        log.debug("Files for low split {} {}", low, lowDatafileSizes.keySet());
        log.debug("Files for high split {} {}", high, highDatafileSizes.keySet());
        MetadataTime time = tabletTime.getMetadataTime();
        HashSet<ExternalCompactionId> ecids = new HashSet<>();
        compactable.getExternalCompactionIds(ecids::add);
        MetadataTableUtil.splitTablet(high, extent.prevEndRow(), splitRatio, getTabletServer().getContext(), getTabletServer().getLock(), ecids);
        ManagerMetadataUtil.addNewTablet(getTabletServer().getContext(), low, lowDirectoryName, getTabletServer().getTabletSession(), lowDatafileSizes, bulkImported, time, lastFlushID, lastCompactID, getTabletServer().getLock());
        MetadataTableUtil.finishSplit(high, highDatafileSizes, highDatafilesToRemove, getTabletServer().getContext(), getTabletServer().getLock());
        TabletLogger.split(extent, low, high, getTabletServer().getTabletSession());
        newTablets.put(high, new TabletData(dirName, highDatafileSizes, time, lastFlushID, lastCompactID, lastLocation, bulkImported));
        newTablets.put(low, new TabletData(lowDirectoryName, lowDatafileSizes, time, lastFlushID, lastCompactID, lastLocation, bulkImported));
        long t2 = System.currentTimeMillis();
        log.debug(String.format("offline split time : %6.2f secs", (t2 - t1) / 1000.0));
        closeState = CloseState.COMPLETE;
        return newTablets;
    }
}

Example 12 with ExternalCompactionId

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionId in project accumulo by apache.

the class Tablet method removeOldTemporaryFiles.

private void removeOldTemporaryFiles(Map<ExternalCompactionId, ExternalCompactionMetadata> externalCompactions) {
    // remove any temporary files created by a previous tablet server
    try {
        var extCompactionFiles = externalCompactions.values().stream().map(ecMeta -> ecMeta.getCompactTmpName().getPath()).collect(Collectors.toSet());
        for (Volume volume : getTabletServer().getVolumeManager().getVolumes()) {
            String dirUri = volume.getBasePath() + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + extent.tableId() + Path.SEPARATOR + dirName;
            for (FileStatus tmp : volume.getFileSystem().globStatus(new Path(dirUri, "*_tmp"))) {
                if (extCompactionFiles.contains(tmp.getPath())) {
                    continue;
                }
                try {
                    log.debug("Removing old temp file {}", tmp.getPath());
                    volume.getFileSystem().delete(tmp.getPath(), false);
                } catch (IOException ex) {
                    log.error("Unable to remove old temp file " + tmp.getPath() + ": " + ex);
                }
            }
        }
    } catch (IOException ex) {
        log.error("Error scanning for old temp files", ex);
    }
}

Example 13 with ExternalCompactionId

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionId in project accumulo by apache.

the class ExternalCompactionUtil method getCompactionIdsRunningOnCompactors.

public static Collection<ExternalCompactionId> getCompactionIdsRunningOnCompactors(ClientContext context) {
    final ExecutorService executor = ThreadPools.createFixedThreadPool(16, "CompactorRunningCompactions", false);
    List<Future<ExternalCompactionId>> futures = new ArrayList<>();
    getCompactorAddrs(context).forEach((q, hp) -> {
        hp.forEach(hostAndPort -> {
            futures.add(executor.submit(() -> getRunningCompactionId(hostAndPort, context)));
        });
    });
    executor.shutdown();
    HashSet<ExternalCompactionId> runningIds = new HashSet<>();
    futures.forEach(future -> {
        try {
            ExternalCompactionId ceid = future.get();
            if (ceid != null) {
                runningIds.add(ceid);
            }
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        } catch (ExecutionException e) {
            throw new RuntimeException(e);
        }
    });
    return runningIds;
}

Example 14 with ExternalCompactionId

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionId in project accumulo by apache.

the class CompactionCoordinatorTest method testCoordinatorRestartOneRunningCompaction.

@Test
public void testCoordinatorRestartOneRunningCompaction() throws Exception {
    PowerMock.resetAll();
    PowerMock.suppress(PowerMock.constructor(AbstractServer.class));
    PowerMock.suppress(PowerMock.methods(ThriftUtil.class, "returnClient"));
    PowerMock.suppress(PowerMock.methods(DeadCompactionDetector.class, "detectDeadCompactions", "detectDanglingFinalStateMarkers"));
    AccumuloConfiguration conf = PowerMock.createNiceMock(AccumuloConfiguration.class);
    ServerContext context = PowerMock.createNiceMock(ServerContext.class);
    TCredentials creds = PowerMock.createNiceMock(TCredentials.class);
    EasyMock.expect(context.rpcCreds()).andReturn(creds);
    CompactionFinalizer finalizer = PowerMock.createNiceMock(CompactionFinalizer.class);
    LiveTServerSet tservers = PowerMock.createNiceMock(LiveTServerSet.class);
    TServerInstance instance = PowerMock.createNiceMock(TServerInstance.class);
    HostAndPort tserverAddress = HostAndPort.fromString("localhost:9997");
    EasyMock.expect(instance.getHostAndPort()).andReturn(tserverAddress).anyTimes();
    EasyMock.expect(tservers.getCurrentServers()).andReturn(Sets.newHashSet(instance)).once();
    tservers.startListeningForTabletServerChanges();
    PowerMock.mockStatic(ExternalCompactionUtil.class);
    List<RunningCompaction> runningCompactions = new ArrayList<>();
    ExternalCompactionId eci = ExternalCompactionId.generate(UUID.randomUUID());
    TExternalCompactionJob job = PowerMock.createNiceMock(TExternalCompactionJob.class);
    EasyMock.expect(job.getExternalCompactionId()).andReturn(eci.toString()).anyTimes();
    TKeyExtent extent = new TKeyExtent();
    extent.setTable("1".getBytes());
    runningCompactions.add(new RunningCompaction(job, tserverAddress.toString(), "queue"));
    EasyMock.expect(ExternalCompactionUtil.getCompactionsRunningOnCompactors(context)).andReturn(runningCompactions);
    ServerAddress client = PowerMock.createNiceMock(ServerAddress.class);
    HostAndPort address = HostAndPort.fromString("localhost:10240");
    EasyMock.expect(client.getAddress()).andReturn(address).anyTimes();
    EasyMock.expect(instance.getHostPort()).andReturn("localhost:9997").anyTimes();
    TabletClientService.Client tsc = PowerMock.createNiceMock(TabletClientService.Client.class);
    TCompactionQueueSummary queueSummary = PowerMock.createNiceMock(TCompactionQueueSummary.class);
    EasyMock.expect(tsc.getCompactionQueueInfo(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Collections.singletonList(queueSummary)).anyTimes();
    EasyMock.expect(queueSummary.getQueue()).andReturn("R2DQ").anyTimes();
    EasyMock.expect(queueSummary.getPriority()).andReturn((short) 1).anyTimes();
    AuditedSecurityOperation security = PowerMock.createNiceMock(AuditedSecurityOperation.class);
    PowerMock.replayAll();
    TestCoordinator coordinator = new TestCoordinator(conf, finalizer, tservers, client, tsc, context, security);
    coordinator.resetInternals();
    assertEquals(0, coordinator.getQueues().size());
    assertEquals(0, coordinator.getIndex().size());
    assertEquals(0, coordinator.getRunning().size());
    coordinator.run();
    assertEquals(1, coordinator.getQueues().size());
    QueueAndPriority qp = QueueAndPriority.get("R2DQ".intern(), (short) 1);
    Map<Short, TreeSet<TServerInstance>> m = coordinator.getQueues().get("R2DQ".intern());
    assertNotNull(m);
    assertEquals(1, m.size());
    assertTrue(m.containsKey((short) 1));
    Set<TServerInstance> t = m.get((short) 1);
    assertNotNull(t);
    assertEquals(1, t.size());
    TServerInstance queuedTsi = t.iterator().next();
    assertEquals(instance.getHostPortSession(), queuedTsi.getHostPortSession());
    assertEquals(1, coordinator.getIndex().size());
    assertTrue(coordinator.getIndex().containsKey(queuedTsi));
    Set<QueueAndPriority> i = coordinator.getIndex().get(queuedTsi);
    assertEquals(1, i.size());
    assertEquals(qp, i.iterator().next());
    assertEquals(1, coordinator.getRunning().size());
    PowerMock.verifyAll();
    coordinator.resetInternals();
    coordinator.close();
}

Example 15 with ExternalCompactionId

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionId in project accumulo by apache.

the class CompactionCoordinatorTest method testGetCompactionJob.

@Test
public void testGetCompactionJob() throws Exception {
    PowerMock.resetAll();
    PowerMock.suppress(PowerMock.constructor(AbstractServer.class));
    PowerMock.suppress(PowerMock.methods(ThriftUtil.class, "returnClient"));
    PowerMock.suppress(PowerMock.methods(DeadCompactionDetector.class, "detectDeadCompactions", "detectDanglingFinalStateMarkers"));
    AccumuloConfiguration conf = PowerMock.createNiceMock(AccumuloConfiguration.class);
    ServerContext context = PowerMock.createNiceMock(ServerContext.class);
    TCredentials creds = PowerMock.createNiceMock(TCredentials.class);
    EasyMock.expect(context.rpcCreds()).andReturn(creds).anyTimes();
    PowerMock.mockStatic(ExternalCompactionUtil.class);
    List<RunningCompaction> runningCompactions = new ArrayList<>();
    EasyMock.expect(ExternalCompactionUtil.getCompactionsRunningOnCompactors(context)).andReturn(runningCompactions);
    CompactionFinalizer finalizer = PowerMock.createNiceMock(CompactionFinalizer.class);
    LiveTServerSet tservers = PowerMock.createNiceMock(LiveTServerSet.class);
    TServerInstance instance = PowerMock.createNiceMock(TServerInstance.class);
    EasyMock.expect(tservers.getCurrentServers()).andReturn(Collections.singleton(instance)).once();
    HostAndPort tserverAddress = HostAndPort.fromString("localhost:9997");
    EasyMock.expect(instance.getHostAndPort()).andReturn(tserverAddress).anyTimes();
    ServerAddress client = PowerMock.createNiceMock(ServerAddress.class);
    HostAndPort address = HostAndPort.fromString("localhost:10240");
    EasyMock.expect(client.getAddress()).andReturn(address).anyTimes();
    TServerInstance tsi = PowerMock.createNiceMock(TServerInstance.class);
    EasyMock.expect(tsi.getHostPort()).andReturn("localhost:9997").anyTimes();
    TabletClientService.Client tsc = PowerMock.createNiceMock(TabletClientService.Client.class);
    TCompactionQueueSummary queueSummary = PowerMock.createNiceMock(TCompactionQueueSummary.class);
    EasyMock.expect(tsc.getCompactionQueueInfo(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Collections.singletonList(queueSummary)).anyTimes();
    EasyMock.expect(queueSummary.getQueue()).andReturn("R2DQ").anyTimes();
    EasyMock.expect(queueSummary.getPriority()).andReturn((short) 1).anyTimes();
    ExternalCompactionId eci = ExternalCompactionId.generate(UUID.randomUUID());
    TExternalCompactionJob job = PowerMock.createNiceMock(TExternalCompactionJob.class);
    EasyMock.expect(job.getExternalCompactionId()).andReturn(eci.toString()).anyTimes();
    TInfo trace = TraceUtil.traceInfo();
    EasyMock.expect(tsc.reserveCompactionJob(trace, creds, "R2DQ", 1, "localhost:10241", eci.toString())).andReturn(job).anyTimes();
    AuditedSecurityOperation security = PowerMock.createNiceMock(AuditedSecurityOperation.class);
    EasyMock.expect(security.canPerformSystemActions(creds)).andReturn(true);
    PowerMock.replayAll();
    TestCoordinator coordinator = new TestCoordinator(conf, finalizer, tservers, client, tsc, context, security);
    coordinator.resetInternals();
    assertEquals(0, coordinator.getQueues().size());
    assertEquals(0, coordinator.getIndex().size());
    assertEquals(0, coordinator.getRunning().size());
    // Use coordinator.run() to populate the internal data structures. This is tested in a different
    // test.
    coordinator.run();
    assertEquals(1, coordinator.getQueues().size());
    QueueAndPriority qp = QueueAndPriority.get("R2DQ".intern(), (short) 1);
    Map<Short, TreeSet<TServerInstance>> m = coordinator.getQueues().get("R2DQ".intern());
    assertNotNull(m);
    assertEquals(1, m.size());
    assertTrue(m.containsKey((short) 1));
    Set<TServerInstance> t = m.get((short) 1);
    assertNotNull(t);
    assertEquals(1, t.size());
    TServerInstance queuedTsi = t.iterator().next();
    assertEquals(tsi.getHostPortSession(), queuedTsi.getHostPortSession());
    assertEquals(1, coordinator.getIndex().size());
    assertTrue(coordinator.getIndex().containsKey(queuedTsi));
    Set<QueueAndPriority> i = coordinator.getIndex().get(queuedTsi);
    assertEquals(1, i.size());
    assertEquals(qp, i.iterator().next());
    assertEquals(0, coordinator.getRunning().size());
    // Get the next job
    TExternalCompactionJob createdJob = coordinator.getCompactionJob(trace, creds, "R2DQ", "localhost:10241", eci.toString());
    assertEquals(eci.toString(), createdJob.getExternalCompactionId());
    assertEquals(1, coordinator.getQueues().size());
    assertEquals(1, coordinator.getIndex().size());
    assertEquals(1, coordinator.getRunning().size());
    Entry<ExternalCompactionId, RunningCompaction> entry = coordinator.getRunning().entrySet().iterator().next();
    assertEquals(eci.toString(), entry.getKey().toString());
    assertEquals("localhost:10241", entry.getValue().getCompactorAddress());
    assertEquals(eci.toString(), entry.getValue().getJob().getExternalCompactionId());
    PowerMock.verifyAll();
    coordinator.resetInternals();
    coordinator.close();
}