Examples with AccumuloConfiguration org.apache.accumulo.core.conf.AccumuloConfiguration used on opensource projects

Example 76 with AccumuloConfiguration

use of org.apache.accumulo.core.conf.AccumuloConfiguration in project accumulo by apache.

the class AccumuloFileOutputFormatTest method validateConfiguration.

@Test
public void validateConfiguration() {
    int a = 7;
    long b = 300L;
    long c = 50L;
    long d = 10L;
    String e = "snappy";
    SamplerConfiguration samplerConfig = new SamplerConfiguration(RowSampler.class.getName());
    samplerConfig.addOption("hasher", "murmur3_32");
    samplerConfig.addOption("modulus", "109");
    SummarizerConfiguration sc1 = SummarizerConfiguration.builder(VisibilitySummarizer.class).addOption(CountingSummarizer.MAX_COUNTERS_OPT, 2048).build();
    SummarizerConfiguration sc2 = SummarizerConfiguration.builder(FamilySummarizer.class).addOption(CountingSummarizer.MAX_COUNTERS_OPT, 256).build();
    JobConf job = new JobConf();
    AccumuloFileOutputFormat.configure().outputPath(new Path("somewhere")).replication(a).fileBlockSize(b).dataBlockSize(c).indexBlockSize(d).compression(e).sampler(samplerConfig).summarizers(sc1, sc2).store(job);
    AccumuloConfiguration acuconf = FileOutputConfigurator.getAccumuloConfiguration(AccumuloFileOutputFormat.class, job);
    assertEquals(7, acuconf.getCount(Property.TABLE_FILE_REPLICATION));
    assertEquals(300L, acuconf.getAsBytes(Property.TABLE_FILE_BLOCK_SIZE));
    assertEquals(50L, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE));
    assertEquals(10L, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
    assertEquals("snappy", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE));
    assertEquals(new SamplerConfigurationImpl(samplerConfig), SamplerConfigurationImpl.newSamplerConfig(acuconf));
    Collection<SummarizerConfiguration> summarizerConfigs = SummarizerConfiguration.fromTableProperties(acuconf);
    assertEquals(2, summarizerConfigs.size());
    assertTrue(summarizerConfigs.contains(sc1));
    assertTrue(summarizerConfigs.contains(sc2));
    a = 17;
    b = 1300L;
    c = 150L;
    d = 110L;
    e = "lzo";
    samplerConfig = new SamplerConfiguration(RowSampler.class.getName());
    samplerConfig.addOption("hasher", "md5");
    samplerConfig.addOption("modulus", "100003");
    job = new JobConf();
    AccumuloFileOutputFormat.configure().outputPath(new Path("somewhere")).replication(a).fileBlockSize(b).dataBlockSize(c).indexBlockSize(d).compression(e).sampler(samplerConfig).store(job);
    acuconf = FileOutputConfigurator.getAccumuloConfiguration(AccumuloFileOutputFormat.class, job);
    assertEquals(17, acuconf.getCount(Property.TABLE_FILE_REPLICATION));
    assertEquals(1300L, acuconf.getAsBytes(Property.TABLE_FILE_BLOCK_SIZE));
    assertEquals(150L, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE));
    assertEquals(110L, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
    assertEquals("lzo", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE));
    assertEquals(new SamplerConfigurationImpl(samplerConfig), SamplerConfigurationImpl.newSamplerConfig(acuconf));
    summarizerConfigs = SummarizerConfiguration.fromTableProperties(acuconf);
    assertEquals(0, summarizerConfigs.size());
}

Example 77 with AccumuloConfiguration

use of org.apache.accumulo.core.conf.AccumuloConfiguration in project accumulo by apache.

the class CompactionCoordinatorTest method testCoordinatorColdStartNoCompactions.

@Test
public void testCoordinatorColdStartNoCompactions() throws Exception {
    PowerMock.resetAll();
    PowerMock.suppress(PowerMock.constructor(AbstractServer.class));
    PowerMock.suppress(PowerMock.methods(ThriftUtil.class, "returnClient"));
    PowerMock.suppress(PowerMock.methods(DeadCompactionDetector.class, "detectDeadCompactions", "detectDanglingFinalStateMarkers"));
    AccumuloConfiguration conf = PowerMock.createNiceMock(AccumuloConfiguration.class);
    ServerContext context = PowerMock.createNiceMock(ServerContext.class);
    PowerMock.mockStatic(ExternalCompactionUtil.class);
    List<RunningCompaction> runningCompactions = new ArrayList<>();
    EasyMock.expect(ExternalCompactionUtil.getCompactionsRunningOnCompactors(context)).andReturn(runningCompactions);
    CompactionFinalizer finalizer = PowerMock.createNiceMock(CompactionFinalizer.class);
    LiveTServerSet tservers = PowerMock.createNiceMock(LiveTServerSet.class);
    EasyMock.expect(tservers.getCurrentServers()).andReturn(Collections.emptySet()).anyTimes();
    ServerAddress client = PowerMock.createNiceMock(ServerAddress.class);
    HostAndPort address = HostAndPort.fromString("localhost:10240");
    EasyMock.expect(client.getAddress()).andReturn(address).anyTimes();
    TServerInstance tsi = PowerMock.createNiceMock(TServerInstance.class);
    EasyMock.expect(tsi.getHostPort()).andReturn("localhost:9997").anyTimes();
    TabletClientService.Client tsc = PowerMock.createNiceMock(TabletClientService.Client.class);
    EasyMock.expect(tsc.getCompactionQueueInfo(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Collections.emptyList()).anyTimes();
    AuditedSecurityOperation security = PowerMock.createNiceMock(AuditedSecurityOperation.class);
    PowerMock.replayAll();
    TestCoordinator coordinator = new TestCoordinator(conf, finalizer, tservers, client, tsc, context, security);
    coordinator.resetInternals();
    assertEquals(0, coordinator.getQueues().size());
    assertEquals(0, coordinator.getIndex().size());
    assertEquals(0, coordinator.getRunning().size());
    coordinator.run();
    assertEquals(0, coordinator.getQueues().size());
    assertEquals(0, coordinator.getIndex().size());
    assertEquals(0, coordinator.getRunning().size());
    PowerMock.verifyAll();
    coordinator.resetInternals();
    coordinator.close();
}

Example 78 with AccumuloConfiguration

use of org.apache.accumulo.core.conf.AccumuloConfiguration in project accumulo by apache.

the class CompactionCoordinatorTest method testCoordinatorRestartNoRunningCompactions.

@Test
public void testCoordinatorRestartNoRunningCompactions() throws Exception {
    PowerMock.resetAll();
    PowerMock.suppress(PowerMock.constructor(AbstractServer.class));
    PowerMock.suppress(PowerMock.methods(ThriftUtil.class, "returnClient"));
    PowerMock.suppress(PowerMock.methods(DeadCompactionDetector.class, "detectDeadCompactions", "detectDanglingFinalStateMarkers"));
    AccumuloConfiguration conf = PowerMock.createNiceMock(AccumuloConfiguration.class);
    ServerContext context = PowerMock.createNiceMock(ServerContext.class);
    TCredentials creds = PowerMock.createNiceMock(TCredentials.class);
    EasyMock.expect(context.rpcCreds()).andReturn(creds);
    CompactionFinalizer finalizer = PowerMock.createNiceMock(CompactionFinalizer.class);
    LiveTServerSet tservers = PowerMock.createNiceMock(LiveTServerSet.class);
    TServerInstance instance = PowerMock.createNiceMock(TServerInstance.class);
    HostAndPort tserverAddress = HostAndPort.fromString("localhost:9997");
    EasyMock.expect(instance.getHostAndPort()).andReturn(tserverAddress).anyTimes();
    EasyMock.expect(tservers.getCurrentServers()).andReturn(Sets.newHashSet(instance)).once();
    tservers.startListeningForTabletServerChanges();
    PowerMock.mockStatic(ExternalCompactionUtil.class);
    List<RunningCompaction> runningCompactions = new ArrayList<>();
    EasyMock.expect(ExternalCompactionUtil.getCompactionsRunningOnCompactors(context)).andReturn(runningCompactions);
    ServerAddress client = PowerMock.createNiceMock(ServerAddress.class);
    HostAndPort address = HostAndPort.fromString("localhost:10240");
    EasyMock.expect(client.getAddress()).andReturn(address).anyTimes();
    EasyMock.expect(instance.getHostPort()).andReturn("localhost:9997").anyTimes();
    TabletClientService.Client tsc = PowerMock.createNiceMock(TabletClientService.Client.class);
    TCompactionQueueSummary queueSummary = PowerMock.createNiceMock(TCompactionQueueSummary.class);
    EasyMock.expect(tsc.getCompactionQueueInfo(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Collections.singletonList(queueSummary)).anyTimes();
    EasyMock.expect(queueSummary.getQueue()).andReturn("R2DQ").anyTimes();
    EasyMock.expect(queueSummary.getPriority()).andReturn((short) 1).anyTimes();
    AuditedSecurityOperation security = PowerMock.createNiceMock(AuditedSecurityOperation.class);
    PowerMock.replayAll();
    TestCoordinator coordinator = new TestCoordinator(conf, finalizer, tservers, client, tsc, context, security);
    coordinator.resetInternals();
    assertEquals(0, coordinator.getQueues().size());
    assertEquals(0, coordinator.getIndex().size());
    assertEquals(0, coordinator.getRunning().size());
    coordinator.run();
    assertEquals(1, coordinator.getQueues().size());
    QueueAndPriority qp = QueueAndPriority.get("R2DQ".intern(), (short) 1);
    Map<Short, TreeSet<TServerInstance>> m = coordinator.getQueues().get("R2DQ".intern());
    assertNotNull(m);
    assertEquals(1, m.size());
    assertTrue(m.containsKey((short) 1));
    Set<TServerInstance> t = m.get((short) 1);
    assertNotNull(t);
    assertEquals(1, t.size());
    TServerInstance queuedTsi = t.iterator().next();
    assertEquals(instance.getHostPortSession(), queuedTsi.getHostPortSession());
    assertEquals(1, coordinator.getIndex().size());
    assertTrue(coordinator.getIndex().containsKey(queuedTsi));
    Set<QueueAndPriority> i = coordinator.getIndex().get(queuedTsi);
    assertEquals(1, i.size());
    assertEquals(qp, i.iterator().next());
    assertEquals(0, coordinator.getRunning().size());
    PowerMock.verifyAll();
    coordinator.resetInternals();
    coordinator.close();
}

Example 79 with AccumuloConfiguration

use of org.apache.accumulo.core.conf.AccumuloConfiguration in project accumulo by apache.

the class Compactor method createCompactionJob.

/**
 * Create compaction runnable
 *
 * @param job
 *          compaction job
 * @param totalInputEntries
 *          object to capture total entries
 * @param totalInputBytes
 *          object to capture input file size
 * @param started
 *          started latch
 * @param stopped
 *          stopped latch
 * @param err
 *          reference to error
 * @return Runnable compaction job
 */
protected Runnable createCompactionJob(final TExternalCompactionJob job, final LongAdder totalInputEntries, final LongAdder totalInputBytes, final CountDownLatch started, final CountDownLatch stopped, final AtomicReference<Throwable> err) {
    return new Runnable() {

        @Override
        public void run() {
            // Its only expected that a single compaction runs at a time. Multiple compactions running
            // at a time could cause odd behavior like out of order and unexpected thrift calls to the
            // coordinator. This is a sanity check to ensure the expectation is met. Should this check
            // ever fail, it means there is a bug elsewhere.
            Preconditions.checkState(compactionRunning.compareAndSet(false, true));
            try {
                LOG.info("Starting up compaction runnable for job: {}", job);
                TCompactionStatusUpdate update = new TCompactionStatusUpdate(TCompactionState.STARTED, "Compaction started", -1, -1, -1);
                updateCompactionState(job, update);
                final AccumuloConfiguration tConfig;
                var extent = KeyExtent.fromThrift(job.getExtent());
                if (!job.getOverrides().isEmpty()) {
                    tConfig = new ConfigurationCopy(getContext().getTableConfiguration(extent.tableId()));
                    job.getOverrides().forEach((k, v) -> ((ConfigurationCopy) tConfig).set(k, v));
                    LOG.debug("Overriding table properties with {}", job.getOverrides());
                } else {
                    tConfig = getContext().getTableConfiguration(extent.tableId());
                }
                final TabletFile outputFile = new TabletFile(new Path(job.getOutputFile()));
                final Map<StoredTabletFile, DataFileValue> files = new TreeMap<>();
                job.getFiles().forEach(f -> {
                    files.put(new StoredTabletFile(f.getMetadataFileEntry()), new DataFileValue(f.getSize(), f.getEntries(), f.getTimestamp()));
                    totalInputEntries.add(f.getEntries());
                    totalInputBytes.add(f.getSize());
                });
                final List<IteratorSetting> iters = new ArrayList<>();
                job.getIteratorSettings().getIterators().forEach(tis -> iters.add(SystemIteratorUtil.toIteratorSetting(tis)));
                ExtCEnv cenv = new ExtCEnv(JOB_HOLDER, queueName);
                FileCompactor compactor = new FileCompactor(getContext(), extent, files, outputFile, job.isPropagateDeletes(), cenv, iters, tConfig);
                LOG.trace("Starting compactor");
                started.countDown();
                org.apache.accumulo.server.compaction.CompactionStats stat = compactor.call();
                TCompactionStats cs = new TCompactionStats();
                cs.setEntriesRead(stat.getEntriesRead());
                cs.setEntriesWritten(stat.getEntriesWritten());
                cs.setFileSize(stat.getFileSize());
                JOB_HOLDER.setStats(cs);
                LOG.info("Compaction completed successfully {} ", job.getExternalCompactionId());
                // Update state when completed
                TCompactionStatusUpdate update2 = new TCompactionStatusUpdate(TCompactionState.SUCCEEDED, "Compaction completed successfully", -1, -1, -1);
                updateCompactionState(job, update2);
            } catch (Exception e) {
                LOG.error("Compaction failed", e);
                err.set(e);
            } finally {
                stopped.countDown();
                Preconditions.checkState(compactionRunning.compareAndSet(true, false));
            }
        }
    };
}

Example 80 with AccumuloConfiguration

use of org.apache.accumulo.core.conf.AccumuloConfiguration in project accumulo by apache.

the class TCredentialsUpdatingInvocationHandlerTest method setup.

@Before
public void setup() {
    cc = new ConfigurationCopy();
    conf = new AccumuloConfiguration() {

        @Override
        public String get(Property property) {
            String value = cc.get(property);
            if (null == value) {
                return DEFAULT_CONFIG.get(property);
            }
            return value;
        }

        @Override
        public void getProperties(Map<String, String> props, Predicate<String> filter) {
            cc.getProperties(props, filter);
        }

        @Override
        public long getUpdateCount() {
            return cc.getUpdateCount();
        }
    };
    proxy = new TCredentialsUpdatingInvocationHandler<>(new Object(), conf);
}