Examples with RemoteEditLog org.apache.hadoop.hdfs.server.protocol.RemoteEditLog used on opensource projects

Example 6 with RemoteEditLog

use of org.apache.hadoop.hdfs.server.protocol.RemoteEditLog in project hadoop by apache.

the class TestPBHelper method testConvertRemoteEditLogManifest.

@Test
public void testConvertRemoteEditLogManifest() {
    List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>();
    logs.add(new RemoteEditLog(1, 10));
    logs.add(new RemoteEditLog(11, 20));
    convertAndCheckRemoteEditLogManifest(new RemoteEditLogManifest(logs, 20), logs, 20);
    convertAndCheckRemoteEditLogManifest(new RemoteEditLogManifest(logs), logs, HdfsServerConstants.INVALID_TXID);
}

Example 7 with RemoteEditLog

use of org.apache.hadoop.hdfs.server.protocol.RemoteEditLog in project hadoop by apache.

the class TestPBHelper method testConvertRemoteEditLog.

@Test
public void testConvertRemoteEditLog() {
    RemoteEditLog l = new RemoteEditLog(1, 100);
    RemoteEditLogProto lProto = PBHelper.convert(l);
    RemoteEditLog l1 = PBHelper.convert(lProto);
    compare(l, l1);
}

Example 8 with RemoteEditLog

use of org.apache.hadoop.hdfs.server.protocol.RemoteEditLog in project hadoop by apache.

the class TestPBHelper method convertAndCheckRemoteEditLogManifest.

private void convertAndCheckRemoteEditLogManifest(RemoteEditLogManifest m, List<RemoteEditLog> logs, long committedTxnId) {
    RemoteEditLogManifestProto mProto = PBHelper.convert(m);
    RemoteEditLogManifest m1 = PBHelper.convert(mProto);
    List<RemoteEditLog> logs1 = m1.getLogs();
    assertEquals(logs.size(), logs1.size());
    for (int i = 0; i < logs.size(); i++) {
        compare(logs.get(i), logs1.get(i));
    }
    assertEquals(committedTxnId, m.getCommittedTxnId());
}

Example 9 with RemoteEditLog

use of org.apache.hadoop.hdfs.server.protocol.RemoteEditLog in project hadoop by apache.

the class TestCheckpoint method testNamespaceVerifiedOnFileTransfer.

/**
   * Test that the primary NN will not serve any files to a 2NN who doesn't
   * share its namespace ID, and also will not accept any files from one.
   */
@Test
public void testNamespaceVerifiedOnFileTransfer() throws IOException {
    MiniDFSCluster cluster = null;
    Configuration conf = new HdfsConfiguration();
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).format(true).build();
        NamenodeProtocols nn = cluster.getNameNodeRpc();
        URL fsName = DFSUtil.getInfoServer(cluster.getNameNode().getServiceRpcAddress(), conf, DFSUtil.getHttpClientScheme(conf)).toURL();
        // Make a finalized log on the server side. 
        nn.rollEditLog();
        RemoteEditLogManifest manifest = nn.getEditLogManifest(1);
        RemoteEditLog log = manifest.getLogs().get(0);
        NNStorage dstImage = Mockito.mock(NNStorage.class);
        Mockito.doReturn(Lists.newArrayList(new File("/wont-be-written"))).when(dstImage).getFiles(Mockito.<NameNodeDirType>anyObject(), Mockito.anyString());
        File mockImageFile = File.createTempFile("image", "");
        FileOutputStream imageFile = new FileOutputStream(mockImageFile);
        imageFile.write("data".getBytes());
        imageFile.close();
        Mockito.doReturn(mockImageFile).when(dstImage).findImageFile(Mockito.any(NameNodeFile.class), Mockito.anyLong());
        Mockito.doReturn(new StorageInfo(1, 1, "X", 1, NodeType.NAME_NODE).toColonSeparatedString()).when(dstImage).toColonSeparatedString();
        try {
            TransferFsImage.downloadImageToStorage(fsName, 0, dstImage, false, false);
            fail("Storage info was not verified");
        } catch (IOException ioe) {
            String msg = StringUtils.stringifyException(ioe);
            assertTrue(msg, msg.contains("but the secondary expected"));
        }
        try {
            TransferFsImage.downloadEditsToStorage(fsName, log, dstImage);
            fail("Storage info was not verified");
        } catch (IOException ioe) {
            String msg = StringUtils.stringifyException(ioe);
            assertTrue(msg, msg.contains("but the secondary expected"));
        }
        try {
            TransferFsImage.uploadImageFromStorage(fsName, conf, dstImage, NameNodeFile.IMAGE, 0);
            fail("Storage info was not verified");
        } catch (IOException ioe) {
            String msg = StringUtils.stringifyException(ioe);
            assertTrue(msg, msg.contains("but the secondary expected"));
        }
    } finally {
        cleanup(cluster);
        cluster = null;
    }
}

Example 10 with RemoteEditLog

use of org.apache.hadoop.hdfs.server.protocol.RemoteEditLog in project hadoop by apache.

the class Journal method getEditLogManifest.

/**
   * @see QJournalProtocol#getEditLogManifest(String, long, boolean)
   */
public RemoteEditLogManifest getEditLogManifest(long sinceTxId, boolean inProgressOk) throws IOException {
    // No need to checkRequest() here - anyone may ask for the list
    // of segments.
    checkFormatted();
    List<RemoteEditLog> logs = fjm.getRemoteEditLogs(sinceTxId, inProgressOk);
    if (inProgressOk) {
        RemoteEditLog log = null;
        for (Iterator<RemoteEditLog> iter = logs.iterator(); iter.hasNext(); ) {
            log = iter.next();
            if (log.isInProgress()) {
                iter.remove();
                break;
            }
        }
        if (log != null && log.isInProgress()) {
            logs.add(new RemoteEditLog(log.getStartTxId(), getHighestWrittenTxId(), true));
        }
    }
    return new RemoteEditLogManifest(logs, getCommittedTxnId());
}