Examples with RegionCoprocessorEnvironment org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment used on opensource projects

Example 16 with RegionCoprocessorEnvironment

use of org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment in project phoenix by apache.

the class SequenceRegionObserver method preIncrement.

/**
     * Use PreIncrement hook of BaseRegionObserver to overcome deficiencies in Increment
     * implementation (HBASE-10254):
     * 1) Lack of recognition and identification of when the key value to increment doesn't exist
     * 2) Lack of the ability to set the timestamp of the updated key value.
     * Works the same as existing region.increment(), except assumes there is a single column to
     * increment and uses Phoenix LONG encoding.
     * 
     * @since 3.0.0
     */
@Override
public Result preIncrement(final ObserverContext<RegionCoprocessorEnvironment> e, final Increment increment) throws IOException {
    RegionCoprocessorEnvironment env = e.getEnvironment();
    // We need to set this to prevent region.increment from being called
    e.bypass();
    e.complete();
    Region region = env.getRegion();
    byte[] row = increment.getRow();
    List<RowLock> locks = Lists.newArrayList();
    TimeRange tr = increment.getTimeRange();
    region.startRegionOperation();
    try {
        acquireLock(region, row, locks);
        try {
            long maxTimestamp = tr.getMax();
            boolean validateOnly = true;
            Get get = new Get(row);
            get.setTimeRange(tr.getMin(), tr.getMax());
            for (Map.Entry<byte[], List<Cell>> entry : increment.getFamilyCellMap().entrySet()) {
                byte[] cf = entry.getKey();
                for (Cell cq : entry.getValue()) {
                    long value = Bytes.toLong(cq.getValueArray(), cq.getValueOffset());
                    get.addColumn(cf, CellUtil.cloneQualifier(cq));
                    long cellTimestamp = cq.getTimestamp();
                    // on the Increment or any of its Cells.
                    if (cellTimestamp > 0 && cellTimestamp < maxTimestamp) {
                        maxTimestamp = cellTimestamp;
                        get.setTimeRange(MetaDataProtocol.MIN_TABLE_TIMESTAMP, maxTimestamp);
                    }
                    validateOnly &= (Sequence.ValueOp.VALIDATE_SEQUENCE.ordinal() == value);
                }
            }
            Result result = region.get(get);
            if (result.isEmpty()) {
                return getErrorResult(row, maxTimestamp, SQLExceptionCode.SEQUENCE_UNDEFINED.getErrorCode());
            }
            KeyValue currentValueKV = Sequence.getCurrentValueKV(result);
            KeyValue incrementByKV = Sequence.getIncrementByKV(result);
            KeyValue cacheSizeKV = Sequence.getCacheSizeKV(result);
            long currentValue = PLong.INSTANCE.getCodec().decodeLong(currentValueKV.getValueArray(), currentValueKV.getValueOffset(), SortOrder.getDefault());
            long incrementBy = PLong.INSTANCE.getCodec().decodeLong(incrementByKV.getValueArray(), incrementByKV.getValueOffset(), SortOrder.getDefault());
            long cacheSize = PLong.INSTANCE.getCodec().decodeLong(cacheSizeKV.getValueArray(), cacheSizeKV.getValueOffset(), SortOrder.getDefault());
            // Hold timestamp constant for sequences, so that clients always only see the latest
            // value regardless of when they connect.
            long timestamp = currentValueKV.getTimestamp();
            Put put = new Put(row, timestamp);
            int numIncrementKVs = increment.getFamilyCellMap().get(PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_FAMILY_BYTES).size();
            // creates the list of KeyValues used for the Result that will be returned
            List<Cell> cells = Sequence.getCells(result, numIncrementKVs);
            //if client is 3.0/4.0 preserve the old behavior (older clients won't have newer columns present in the increment)
            if (numIncrementKVs != Sequence.NUM_SEQUENCE_KEY_VALUES) {
                currentValue += incrementBy * cacheSize;
                // Hold timestamp constant for sequences, so that clients always only see the latest value
                // regardless of when they connect.
                KeyValue newCurrentValueKV = createKeyValue(row, PhoenixDatabaseMetaData.CURRENT_VALUE_BYTES, currentValue, timestamp);
                put.add(newCurrentValueKV);
                Sequence.replaceCurrentValueKV(cells, newCurrentValueKV);
            } else {
                KeyValue cycleKV = Sequence.getCycleKV(result);
                KeyValue limitReachedKV = Sequence.getLimitReachedKV(result);
                KeyValue minValueKV = Sequence.getMinValueKV(result);
                KeyValue maxValueKV = Sequence.getMaxValueKV(result);
                boolean increasingSeq = incrementBy > 0 ? true : false;
                // if the minValue, maxValue, cycle and limitReached is null this sequence has been upgraded from
                // a lower version. Set minValue, maxValue, cycle and limitReached to Long.MIN_VALUE, Long.MAX_VALUE, true and false
                // respectively in order to maintain existing behavior and also update the KeyValues on the server 
                boolean limitReached;
                if (limitReachedKV == null) {
                    limitReached = false;
                    KeyValue newLimitReachedKV = createKeyValue(row, PhoenixDatabaseMetaData.LIMIT_REACHED_FLAG_BYTES, limitReached, timestamp);
                    put.add(newLimitReachedKV);
                    Sequence.replaceLimitReachedKV(cells, newLimitReachedKV);
                } else {
                    limitReached = (Boolean) PBoolean.INSTANCE.toObject(limitReachedKV.getValueArray(), limitReachedKV.getValueOffset(), limitReachedKV.getValueLength());
                }
                long minValue;
                if (minValueKV == null) {
                    minValue = Long.MIN_VALUE;
                    KeyValue newMinValueKV = createKeyValue(row, PhoenixDatabaseMetaData.MIN_VALUE_BYTES, minValue, timestamp);
                    put.add(newMinValueKV);
                    Sequence.replaceMinValueKV(cells, newMinValueKV);
                } else {
                    minValue = PLong.INSTANCE.getCodec().decodeLong(minValueKV.getValueArray(), minValueKV.getValueOffset(), SortOrder.getDefault());
                }
                long maxValue;
                if (maxValueKV == null) {
                    maxValue = Long.MAX_VALUE;
                    KeyValue newMaxValueKV = createKeyValue(row, PhoenixDatabaseMetaData.MAX_VALUE_BYTES, maxValue, timestamp);
                    put.add(newMaxValueKV);
                    Sequence.replaceMaxValueKV(cells, newMaxValueKV);
                } else {
                    maxValue = PLong.INSTANCE.getCodec().decodeLong(maxValueKV.getValueArray(), maxValueKV.getValueOffset(), SortOrder.getDefault());
                }
                boolean cycle;
                if (cycleKV == null) {
                    cycle = false;
                    KeyValue newCycleKV = createKeyValue(row, PhoenixDatabaseMetaData.CYCLE_FLAG_BYTES, cycle, timestamp);
                    put.add(newCycleKV);
                    Sequence.replaceCycleValueKV(cells, newCycleKV);
                } else {
                    cycle = (Boolean) PBoolean.INSTANCE.toObject(cycleKV.getValueArray(), cycleKV.getValueOffset(), cycleKV.getValueLength());
                }
                long numSlotsToAllocate = calculateNumSlotsToAllocate(increment);
                // We don't support Bulk Allocations on sequences that have the CYCLE flag set to true
                if (cycle && !SequenceUtil.isCycleAllowed(numSlotsToAllocate)) {
                    return getErrorResult(row, maxTimestamp, SQLExceptionCode.NUM_SEQ_TO_ALLOCATE_NOT_SUPPORTED.getErrorCode());
                }
                // Bulk Allocations are expressed by NEXT <n> VALUES FOR
                if (SequenceUtil.isBulkAllocation(numSlotsToAllocate)) {
                    if (SequenceUtil.checkIfLimitReached(currentValue, minValue, maxValue, incrementBy, cacheSize, numSlotsToAllocate)) {
                        // all the slots requested.
                        return getErrorResult(row, maxTimestamp, SequenceUtil.getLimitReachedErrorCode(increasingSeq).getErrorCode());
                    }
                }
                if (validateOnly) {
                    return result;
                }
                // return if we have run out of sequence values 
                if (limitReached) {
                    if (cycle) {
                        // reset currentValue of the Sequence row to minValue/maxValue
                        currentValue = increasingSeq ? minValue : maxValue;
                    } else {
                        return getErrorResult(row, maxTimestamp, SequenceUtil.getLimitReachedErrorCode(increasingSeq).getErrorCode());
                    }
                }
                // check if the limit was reached
                limitReached = SequenceUtil.checkIfLimitReached(currentValue, minValue, maxValue, incrementBy, cacheSize, numSlotsToAllocate);
                // update currentValue
                currentValue += incrementBy * (SequenceUtil.isBulkAllocation(numSlotsToAllocate) ? numSlotsToAllocate : cacheSize);
                // update the currentValue of the Result row
                KeyValue newCurrentValueKV = createKeyValue(row, PhoenixDatabaseMetaData.CURRENT_VALUE_BYTES, currentValue, timestamp);
                Sequence.replaceCurrentValueKV(cells, newCurrentValueKV);
                put.add(newCurrentValueKV);
                // set the LIMIT_REACHED column to true, so that no new values can be used
                KeyValue newLimitReachedKV = createKeyValue(row, PhoenixDatabaseMetaData.LIMIT_REACHED_FLAG_BYTES, limitReached, timestamp);
                put.add(newLimitReachedKV);
            }
            // update the KeyValues on the server
            Mutation[] mutations = new Mutation[] { put };
            region.batchMutate(mutations, HConstants.NO_NONCE, HConstants.NO_NONCE);
            // return a Result with the updated KeyValues
            return Result.create(cells);
        } finally {
            region.releaseRowLocks(locks);
        }
    } catch (Throwable t) {
        ServerUtil.throwIOException("Increment of sequence " + Bytes.toStringBinary(row), t);
        // Impossible
        return null;
    } finally {
        region.closeRegionOperation();
    }
}

Example 17 with RegionCoprocessorEnvironment

use of org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment in project phoenix by apache.

the class SequenceRegionObserver method preAppend.

/**
     * Override the preAppend for checkAndPut and checkAndDelete, as we need the ability to
     * a) set the TimeRange for the Get being done and
     * b) return something back to the client to indicate success/failure
     */
@SuppressWarnings("deprecation")
@Override
public Result preAppend(final ObserverContext<RegionCoprocessorEnvironment> e, final Append append) throws IOException {
    byte[] opBuf = append.getAttribute(OPERATION_ATTRIB);
    if (opBuf == null) {
        return null;
    }
    Sequence.MetaOp op = Sequence.MetaOp.values()[opBuf[0]];
    Cell keyValue = append.getFamilyCellMap().values().iterator().next().iterator().next();
    long clientTimestamp = HConstants.LATEST_TIMESTAMP;
    long minGetTimestamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP;
    long maxGetTimestamp = HConstants.LATEST_TIMESTAMP;
    boolean hadClientTimestamp;
    byte[] clientTimestampBuf = null;
    if (op == Sequence.MetaOp.RETURN_SEQUENCE) {
        // When returning sequences, this allows us to send the expected timestamp
        // of the sequence to make sure we don't reset any other sequence
        hadClientTimestamp = true;
        clientTimestamp = minGetTimestamp = keyValue.getTimestamp();
        maxGetTimestamp = minGetTimestamp + 1;
    } else {
        clientTimestampBuf = append.getAttribute(MAX_TIMERANGE_ATTRIB);
        if (clientTimestampBuf != null) {
            clientTimestamp = maxGetTimestamp = Bytes.toLong(clientTimestampBuf);
        }
        hadClientTimestamp = (clientTimestamp != HConstants.LATEST_TIMESTAMP);
        if (hadClientTimestamp) {
            // created.
            if (op == Sequence.MetaOp.CREATE_SEQUENCE) {
                maxGetTimestamp = clientTimestamp + 1;
            }
        } else {
            clientTimestamp = maxGetTimestamp = EnvironmentEdgeManager.currentTimeMillis();
            clientTimestampBuf = Bytes.toBytes(clientTimestamp);
        }
    }
    RegionCoprocessorEnvironment env = e.getEnvironment();
    // We need to set this to prevent region.append from being called
    e.bypass();
    e.complete();
    Region region = env.getRegion();
    byte[] row = append.getRow();
    List<RowLock> locks = Lists.newArrayList();
    region.startRegionOperation();
    try {
        acquireLock(region, row, locks);
        try {
            byte[] family = CellUtil.cloneFamily(keyValue);
            byte[] qualifier = CellUtil.cloneQualifier(keyValue);
            Get get = new Get(row);
            get.setTimeRange(minGetTimestamp, maxGetTimestamp);
            get.addColumn(family, qualifier);
            Result result = region.get(get);
            if (result.isEmpty()) {
                if (op == Sequence.MetaOp.DROP_SEQUENCE || op == Sequence.MetaOp.RETURN_SEQUENCE) {
                    return getErrorResult(row, clientTimestamp, SQLExceptionCode.SEQUENCE_UNDEFINED.getErrorCode());
                }
            } else {
                if (op == Sequence.MetaOp.CREATE_SEQUENCE) {
                    return getErrorResult(row, clientTimestamp, SQLExceptionCode.SEQUENCE_ALREADY_EXIST.getErrorCode());
                }
            }
            Mutation m = null;
            switch(op) {
                case RETURN_SEQUENCE:
                    KeyValue currentValueKV = result.raw()[0];
                    long expectedValue = PLong.INSTANCE.getCodec().decodeLong(append.getAttribute(CURRENT_VALUE_ATTRIB), 0, SortOrder.getDefault());
                    long value = PLong.INSTANCE.getCodec().decodeLong(currentValueKV.getValueArray(), currentValueKV.getValueOffset(), SortOrder.getDefault());
                    // Timestamp should match exactly, or we may have the wrong sequence
                    if (expectedValue != value || currentValueKV.getTimestamp() != clientTimestamp) {
                        return Result.create(Collections.singletonList((Cell) KeyValueUtil.newKeyValue(row, PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, currentValueKV.getTimestamp(), ByteUtil.EMPTY_BYTE_ARRAY)));
                    }
                    m = new Put(row, currentValueKV.getTimestamp());
                    m.getFamilyCellMap().putAll(append.getFamilyCellMap());
                    break;
                case DROP_SEQUENCE:
                    m = new Delete(row, clientTimestamp);
                    break;
                case CREATE_SEQUENCE:
                    m = new Put(row, clientTimestamp);
                    m.getFamilyCellMap().putAll(append.getFamilyCellMap());
                    break;
            }
            if (!hadClientTimestamp) {
                for (List<Cell> kvs : m.getFamilyCellMap().values()) {
                    for (Cell kv : kvs) {
                        ((KeyValue) kv).updateLatestStamp(clientTimestampBuf);
                    }
                }
            }
            Mutation[] mutations = new Mutation[] { m };
            region.batchMutate(mutations, HConstants.NO_NONCE, HConstants.NO_NONCE);
            long serverTimestamp = MetaDataUtil.getClientTimeStamp(m);
            // when the mutation was actually performed (useful in the case of .
            return Result.create(Collections.singletonList((Cell) KeyValueUtil.newKeyValue(row, PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, serverTimestamp, SUCCESS_VALUE)));
        } finally {
            region.releaseRowLocks(locks);
        }
    } catch (Throwable t) {
        ServerUtil.throwIOException("Increment of sequence " + Bytes.toStringBinary(row), t);
        // Impossible
        return null;
    } finally {
        region.closeRegionOperation();
    }
}

Example 18 with RegionCoprocessorEnvironment

use of org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment in project phoenix by apache.

the class Indexer method start.

@Override
public void start(CoprocessorEnvironment e) throws IOException {
    try {
        final RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
        this.environment = env;
        env.getConfiguration().setClass(RpcControllerFactory.CUSTOM_CONTROLLER_CONF_KEY, ServerRpcControllerFactory.class, RpcControllerFactory.class);
        String serverName = env.getRegionServerServices().getServerName().getServerName();
        if (env.getConfiguration().getBoolean(CHECK_VERSION_CONF_KEY, true)) {
            // make sure the right version <-> combinations are allowed.
            String errormsg = Indexer.validateVersion(env.getHBaseVersion(), env.getConfiguration());
            if (errormsg != null) {
                IOException ioe = new IOException(errormsg);
                env.getRegionServerServices().abort(errormsg, ioe);
                throw ioe;
            }
        }
        this.builder = new IndexBuildManager(env);
        // setup the actual index writer
        this.writer = new IndexWriter(env, serverName + "-index-writer");
        try {
            // get the specified failure policy. We only ever override it in tests, but we need to do it
            // here
            Class<? extends IndexFailurePolicy> policyClass = env.getConfiguration().getClass(INDEX_RECOVERY_FAILURE_POLICY_KEY, StoreFailuresInCachePolicy.class, IndexFailurePolicy.class);
            IndexFailurePolicy policy = policyClass.getConstructor(PerRegionIndexWriteCache.class).newInstance(failedIndexEdits);
            LOG.debug("Setting up recovery writter with failure policy: " + policy.getClass());
            recoveryWriter = new RecoveryIndexWriter(policy, env, serverName + "-recovery-writer");
        } catch (Exception ex) {
            throw new IOException("Could not instantiate recovery failure policy!", ex);
        }
    } catch (NoSuchMethodError ex) {
        disabled = true;
        super.start(e);
        LOG.error("Must be too early a version of HBase. Disabled coprocessor ", ex);
    }
}

Example 19 with RegionCoprocessorEnvironment

use of org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment in project phoenix by apache.

the class TestLocalTableState method testNoScannerForImmutableRows.

@Test
public void testNoScannerForImmutableRows() throws Exception {
    IndexMetaData indexMetaData = new IndexMetaData() {

        @Override
        public boolean isImmutableRows() {
            return true;
        }

        @Override
        public boolean ignoreNewerMutations() {
            return false;
        }
    };
    Put m = new Put(row);
    m.add(fam, qual, ts, val);
    // setup mocks
    Configuration conf = new Configuration(false);
    RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);
    Mockito.when(env.getConfiguration()).thenReturn(conf);
    Region region = Mockito.mock(Region.class);
    Mockito.when(env.getRegion()).thenReturn(region);
    Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenThrow(new ScannerCreatedException("Should not open scanner when data is immutable"));
    LocalHBaseState state = new LocalTable(env);
    LocalTableState table = new LocalTableState(env, state, m);
    //add the kvs from the mutation
    table.addPendingUpdates(KeyValueUtil.ensureKeyValues(m.get(fam, qual)));
    // setup the lookup
    ColumnReference col = new ColumnReference(fam, qual);
    table.setCurrentTimestamp(ts);
    //check that our value still shows up first on scan, even though this is a lazy load
    Pair<Scanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
    Scanner s = p.getFirst();
    assertEquals("Didn't get the pending mutation's value first", m.get(fam, qual).get(0), s.next());
}

Example 20 with RegionCoprocessorEnvironment

use of org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment in project phoenix by apache.

the class TestLocalTableState method testCorrectRollback.

/**
   * Test that we correctly rollback the state of keyvalue
   * @throws Exception
   */
@Test
@SuppressWarnings("unchecked")
public void testCorrectRollback() throws Exception {
    Put m = new Put(row);
    m.add(fam, qual, ts, val);
    // setup mocks
    RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);
    Region region = Mockito.mock(Region.class);
    Mockito.when(env.getRegion()).thenReturn(region);
    RegionScanner scanner = Mockito.mock(RegionScanner.class);
    Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner);
    final byte[] stored = Bytes.toBytes("stored-value");
    final KeyValue storedKv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
    storedKv.setSequenceId(2);
    Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer<Boolean>() {

        @Override
        public Boolean answer(InvocationOnMock invocation) throws Throwable {
            List<KeyValue> list = (List<KeyValue>) invocation.getArguments()[0];
            list.add(storedKv);
            return false;
        }
    });
    LocalHBaseState state = new LocalTable(env);
    LocalTableState table = new LocalTableState(env, state, m);
    // add the kvs from the mutation
    KeyValue kv = KeyValueUtil.ensureKeyValue(m.get(fam, qual).get(0));
    kv.setSequenceId(0);
    table.addPendingUpdates(kv);
    // setup the lookup
    ColumnReference col = new ColumnReference(fam, qual);
    table.setCurrentTimestamp(ts);
    // check that the value is there
    Pair<Scanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
    Scanner s = p.getFirst();
    assertEquals("Didn't get the pending mutation's value first", kv, s.next());
    // rollback that value
    table.rollback(Arrays.asList(kv));
    p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
    s = p.getFirst();
    assertEquals("Didn't correctly rollback the row - still found it!", null, s.next());
    Mockito.verify(env, Mockito.times(1)).getRegion();
    Mockito.verify(region, Mockito.times(1)).getScanner(Mockito.any(Scan.class));
}