Examples with TransactionVisibilityState org.apache.tephra.persist.TransactionVisibilityState used on opensource projects

Example 16 with TransactionVisibilityState

use of org.apache.tephra.persist.TransactionVisibilityState in project cdap by caskdata.

the class ConsumerConfigCache method updateCache.

/**
   * This forces an immediate update of the config cache. It should only be called from the refresh thread or from
   * tests, to avoid having to add a sleep for the duration of the refresh interval.
   *
   * This method is synchronized to protect from race conditions if called directly from a test. Otherwise this is
   * only called from the refresh thread, and there will not be concurrent invocations.
   *
   * @throws IOException if failed to update config cache
   */
@VisibleForTesting
public synchronized void updateCache() throws IOException {
    Map<byte[], QueueConsumerConfig> newCache = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
    long now = System.currentTimeMillis();
    TransactionVisibilityState txSnapshot = transactionSnapshotSupplier.get();
    if (txSnapshot == null) {
        LOG.debug("No transaction snapshot is available. Not updating the consumer config cache.");
        return;
    }
    HTableInterface table = hTableSupplier.getInput();
    try {
        // Scan the table with the transaction snapshot
        Scan scan = new Scan();
        scan.addFamily(QueueEntryRow.COLUMN_FAMILY);
        Transaction tx = TxUtils.createDummyTransaction(txSnapshot);
        setScanAttribute(scan, TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
        ResultScanner scanner = table.getScanner(scan);
        int configCnt = 0;
        for (Result result : scanner) {
            if (!result.isEmpty()) {
                NavigableMap<byte[], byte[]> familyMap = result.getFamilyMap(QueueEntryRow.COLUMN_FAMILY);
                if (familyMap != null) {
                    configCnt++;
                    Map<ConsumerInstance, byte[]> consumerInstances = new HashMap<>();
                    // Gather the startRow of all instances across all consumer groups.
                    int numGroups = 0;
                    Long groupId = null;
                    for (Map.Entry<byte[], byte[]> entry : familyMap.entrySet()) {
                        if (entry.getKey().length != STATE_COLUMN_SIZE) {
                            continue;
                        }
                        long gid = Bytes.toLong(entry.getKey());
                        int instanceId = Bytes.toInt(entry.getKey(), Bytes.SIZEOF_LONG);
                        consumerInstances.put(new ConsumerInstance(gid, instanceId), entry.getValue());
                        // Columns are sorted by groupId, hence if it change, then numGroups would get +1
                        if (groupId == null || groupId != gid) {
                            numGroups++;
                            groupId = gid;
                        }
                    }
                    byte[] queueName = result.getRow();
                    newCache.put(queueName, new QueueConsumerConfig(consumerInstances, numGroups));
                }
            }
        }
        long elapsed = System.currentTimeMillis() - now;
        this.configCache = newCache;
        this.lastUpdated = now;
        if (LOG.isDebugEnabled()) {
            LOG.debug("Updated consumer config cache with {} entries, took {} msec", configCnt, elapsed);
        }
    } finally {
        try {
            table.close();
        } catch (IOException ioe) {
            LOG.error("Error closing table {}", queueConfigTableName, ioe);
        }
    }
}

Example 17 with TransactionVisibilityState

use of org.apache.tephra.persist.TransactionVisibilityState in project cdap by caskdata.

the class HBaseQueueRegionObserver method start.

@Override
public void start(CoprocessorEnvironment env) {
    if (env instanceof RegionCoprocessorEnvironment) {
        HTableDescriptor tableDesc = ((RegionCoprocessorEnvironment) env).getRegion().getTableDesc();
        String hTableName = tableDesc.getNameAsString();
        String prefixBytes = tableDesc.getValue(HBaseQueueAdmin.PROPERTY_PREFIX_BYTES);
        try {
            // Default to SALT_BYTES for the older salted queue implementation.
            this.prefixBytes = prefixBytes == null ? SaltedHBaseQueueStrategy.SALT_BYTES : Integer.parseInt(prefixBytes);
        } catch (NumberFormatException e) {
            // Shouldn't happen for table created by cdap.
            LOG.error("Unable to parse value of '" + HBaseQueueAdmin.PROPERTY_PREFIX_BYTES + "' property. " + "Default to " + SaltedHBaseQueueStrategy.SALT_BYTES, e);
            this.prefixBytes = SaltedHBaseQueueStrategy.SALT_BYTES;
        }
        namespaceId = HTableNameConverter.from(tableDesc).getNamespace();
        appName = HBaseQueueAdmin.getApplicationName(hTableName);
        flowName = HBaseQueueAdmin.getFlowName(hTableName);
        Configuration conf = env.getConfiguration();
        String hbaseNamespacePrefix = tableDesc.getValue(Constants.Dataset.TABLE_PREFIX);
        final String sysConfigTablePrefix = HTableNameConverter.getSysConfigTablePrefix(hbaseNamespacePrefix);
        txStateCacheSupplier = new DefaultTransactionStateCacheSupplier(sysConfigTablePrefix, conf);
        txStateCache = txStateCacheSupplier.get();
        txSnapshotSupplier = new Supplier<TransactionVisibilityState>() {

            @Override
            public TransactionVisibilityState get() {
                return txStateCache.getLatestState();
            }
        };
        String queueConfigTableId = HBaseQueueAdmin.getConfigTableName();
        configTableName = HTableNameConverter.toTableName(hbaseNamespacePrefix, TableId.from(namespaceId, queueConfigTableId));
        cConfReader = new CConfigurationReader(conf, sysConfigTablePrefix);
        configCacheSupplier = createConfigCache(env);
        configCache = configCacheSupplier.get();
    }
}

Example 18 with TransactionVisibilityState

use of org.apache.tephra.persist.TransactionVisibilityState in project cdap by caskdata.

the class HBaseQueueRegionObserver method preCompact.

@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, InternalScanner scanner, ScanType type, CompactionRequest request) throws IOException {
    if (!e.getEnvironment().getRegion().isAvailable()) {
        return scanner;
    }
    LOG.info("preCompact, creates EvictionInternalScanner");
    TransactionVisibilityState txVisibilityState = txStateCache.getLatestState();
    reloadPruneState(e.getEnvironment());
    if (compactionState != null) {
        // Record tx state before the compaction
        compactionState.record(request, txVisibilityState);
    }
    return new EvictionInternalScanner("compaction", e.getEnvironment(), scanner, txVisibilityState);
}

Example 19 with TransactionVisibilityState

use of org.apache.tephra.persist.TransactionVisibilityState in project cdap by caskdata.

the class MessageTableRegionObserver method preFlushScannerOpen.

@Override
public InternalScanner preFlushScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException {
    LOG.info("preFlush, filter using MessageDataFilter");
    TransactionVisibilityState txVisibilityState = txStateCache.getLatestState();
    Scan scan = new Scan();
    scan.setFilter(new MessageDataFilter(c.getEnvironment(), System.currentTimeMillis(), prefixLength, topicMetadataCache, txVisibilityState));
    return new LoggingInternalScanner("MessageDataFilter", "preFlush", new StoreScanner(store, store.getScanInfo(), scan, Collections.singletonList(memstoreScanner), ScanType.COMPACT_DROP_DELETES, store.getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP), txVisibilityState);
}

Example 20 with TransactionVisibilityState

use of org.apache.tephra.persist.TransactionVisibilityState in project cdap by caskdata.

the class HBaseQueueRegionObserver method start.

@Override
public void start(CoprocessorEnvironment env) {
    if (env instanceof RegionCoprocessorEnvironment) {
        HTableDescriptor tableDesc = ((RegionCoprocessorEnvironment) env).getRegion().getTableDesc();
        String hTableName = tableDesc.getNameAsString();
        String prefixBytes = tableDesc.getValue(HBaseQueueAdmin.PROPERTY_PREFIX_BYTES);
        try {
            // Default to SALT_BYTES for the older salted queue implementation.
            this.prefixBytes = prefixBytes == null ? SaltedHBaseQueueStrategy.SALT_BYTES : Integer.parseInt(prefixBytes);
        } catch (NumberFormatException e) {
            // Shouldn't happen for table created by cdap.
            LOG.error("Unable to parse value of '" + HBaseQueueAdmin.PROPERTY_PREFIX_BYTES + "' property. " + "Default to " + SaltedHBaseQueueStrategy.SALT_BYTES, e);
            this.prefixBytes = SaltedHBaseQueueStrategy.SALT_BYTES;
        }
        namespaceId = HTableNameConverter.from(tableDesc).getNamespace();
        appName = HBaseQueueAdmin.getApplicationName(hTableName);
        flowName = HBaseQueueAdmin.getFlowName(hTableName);
        Configuration conf = env.getConfiguration();
        String hbaseNamespacePrefix = tableDesc.getValue(Constants.Dataset.TABLE_PREFIX);
        final String sysConfigTablePrefix = HTableNameConverter.getSysConfigTablePrefix(hbaseNamespacePrefix);
        txStateCacheSupplier = new DefaultTransactionStateCacheSupplier(sysConfigTablePrefix, conf);
        txStateCache = txStateCacheSupplier.get();
        txSnapshotSupplier = new Supplier<TransactionVisibilityState>() {

            @Override
            public TransactionVisibilityState get() {
                return txStateCache.getLatestState();
            }
        };
        String queueConfigTableId = HBaseQueueAdmin.getConfigTableName();
        configTableName = HTableNameConverter.toTableName(hbaseNamespacePrefix, TableId.from(namespaceId, queueConfigTableId));
        cConfReader = new CConfigurationReader(conf, sysConfigTablePrefix);
        configCacheSupplier = createConfigCache(env);
        configCache = configCacheSupplier.get();
    }
}