use of alluxio.client.file.FileInStream in project alluxio by Alluxio.
the class LocalBlockInStreamIntegrationTest method seekExceptionTest1.
/**
* Tests {@link alluxio.client.block.LocalBlockInStream#seek(long)}. Validate the expected
* exception for seeking a negative position.
*/
@Test
public void seekExceptionTest1() throws Exception {
mThrown.expect(IllegalArgumentException.class);
mThrown.expectMessage(String.format(PreconditionMessage.ERR_SEEK_NEGATIVE.toString(), -1));
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
AlluxioURI uri = new AlluxioURI(sTestPath + "/file_" + k + "_" + op.hashCode());
try (FileInStream is = sFileSystem.openFile(uri, sReadNoCache)) {
is.seek(-1);
}
}
}
}
use of alluxio.client.file.FileInStream in project alluxio by Alluxio.
the class LocalBlockInStreamIntegrationTest method seek.
/**
* Tests {@link alluxio.client.block.LocalBlockInStream#seek(long)}.
*/
@Test
public void seek() throws Exception {
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
AlluxioURI uri = new AlluxioURI(sTestPath + "/file_" + k + "_" + op.hashCode());
FileInStream is = sFileSystem.openFile(uri, sReadNoCache);
is.seek(k / 3);
Assert.assertEquals(k / 3, is.read());
is.seek(k / 2);
Assert.assertEquals(k / 2, is.read());
is.seek(k / 4);
Assert.assertEquals(k / 4, is.read());
is.close();
}
}
}
use of alluxio.client.file.FileInStream in project alluxio by Alluxio.
the class LocalCacheFileInStream method readExternalPage.
/**
* Reads a page from external storage which contains the position specified. Note that this makes
* a copy of the page.
* <p>
* This method is synchronized to ensure thread safety for positioned reads. Only a single thread
* should call this method at a time because the underlying state (mExternalFileInStream) is
* shared. Another way would be to use positioned reads instead of seek and read, but that assumes
* the underlying FileInStream implements thread safe positioned reads which are not much more
* expensive than seek and read.
* <p>
* TODO(calvin): Consider a more efficient API which does not require a data copy.
*
* @param pos the position which the page will contain
* @return a byte array of the page data
*/
private synchronized byte[] readExternalPage(long pos, ReadType readType) throws IOException {
long pageStart = pos - (pos % mPageSize);
FileInStream stream = getExternalFileInStream(pageStart);
int pageSize = (int) Math.min(mPageSize, mStatus.getLength() - pageStart);
byte[] page = new byte[pageSize];
ByteBuffer buffer = readType == ReadType.READ_INTO_BYTE_BUFFER ? ByteBuffer.wrap(page) : null;
int totalBytesRead = 0;
while (totalBytesRead < pageSize) {
int bytesRead;
switch(readType) {
case READ_INTO_BYTE_ARRAY:
bytesRead = stream.read(page, totalBytesRead, pageSize - totalBytesRead);
break;
case READ_INTO_BYTE_BUFFER:
bytesRead = stream.read(buffer);
break;
default:
throw new IOException("unsupported read type = " + readType);
}
if (bytesRead <= 0) {
break;
}
totalBytesRead += bytesRead;
}
// Bytes read from external, may be larger than requests due to reading complete pages
MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_EXTERNAL.getName()).mark(totalBytesRead);
if (totalBytesRead != pageSize) {
throw new IOException("Failed to read complete page from external storage. Bytes read: " + totalBytesRead + " Page size: " + pageSize);
}
return page;
}
use of alluxio.client.file.FileInStream in project alluxio by Alluxio.
the class AlluxioMasterRestServiceHandler method getWebUIBrowse.
/**
* Gets Web UI browse page data.
*
* @param requestPath the request path
* @param requestOffset the request offset
* @param requestEnd the request end
* @param requestLimit the request limit
* @return the response object
*/
@GET
@Path(WEBUI_BROWSE)
public Response getWebUIBrowse(@DefaultValue("/") @QueryParam("path") String requestPath, @DefaultValue("0") @QueryParam("offset") String requestOffset, @DefaultValue("") @QueryParam("end") String requestEnd, @DefaultValue("20") @QueryParam("limit") String requestLimit) {
return RestUtils.call(() -> {
MasterWebUIBrowse response = new MasterWebUIBrowse();
if (!ServerConfiguration.getBoolean(PropertyKey.WEB_FILE_INFO_ENABLED)) {
return response;
}
if (SecurityUtils.isSecurityEnabled(ServerConfiguration.global()) && AuthenticatedClientUser.get(ServerConfiguration.global()) == null) {
AuthenticatedClientUser.set(ServerUserState.global().getUser().getName());
}
response.setDebug(ServerConfiguration.getBoolean(PropertyKey.DEBUG)).setShowPermissions(ServerConfiguration.getBoolean(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_ENABLED)).setMasterNodeAddress(mMasterProcess.getRpcAddress().toString()).setInvalidPathError("");
List<FileInfo> filesInfo;
String path = URLDecoder.decode(requestPath, "UTF-8");
if (path.isEmpty()) {
path = AlluxioURI.SEPARATOR;
}
AlluxioURI currentPath = new AlluxioURI(path);
response.setCurrentPath(currentPath.toString()).setViewingOffset(0);
try {
long fileId = mFileSystemMaster.getFileId(currentPath);
FileInfo fileInfo = mFileSystemMaster.getFileInfo(fileId);
UIFileInfo currentFileInfo = new UIFileInfo(fileInfo, ServerConfiguration.global(), new MasterStorageTierAssoc().getOrderedStorageAliases());
if (currentFileInfo.getAbsolutePath() == null) {
throw new FileDoesNotExistException(currentPath.toString());
}
response.setCurrentDirectory(currentFileInfo).setBlockSizeBytes(currentFileInfo.getBlockSizeBytes());
if (!currentFileInfo.getIsDirectory()) {
long relativeOffset = 0;
long offset;
try {
if (requestOffset != null) {
relativeOffset = Long.parseLong(requestOffset);
}
} catch (NumberFormatException e) {
// ignore the exception
}
// relative to the end of the file.
if (requestEnd.equals("")) {
offset = relativeOffset;
} else {
offset = fileInfo.getLength() - relativeOffset;
}
if (offset < 0) {
offset = 0;
} else if (offset > fileInfo.getLength()) {
offset = fileInfo.getLength();
}
try {
AlluxioURI absolutePath = new AlluxioURI(currentFileInfo.getAbsolutePath());
FileSystem fs = mFsClient;
String fileData;
URIStatus status = fs.getStatus(absolutePath);
if (status.isCompleted()) {
OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build();
try (FileInStream is = fs.openFile(absolutePath, options)) {
int len = (int) Math.min(5L * Constants.KB, status.getLength() - offset);
byte[] data = new byte[len];
long skipped = is.skip(offset);
if (skipped < 0) {
// nothing was skipped
fileData = "Unable to traverse to offset; is file empty?";
} else if (skipped < offset) {
// couldn't skip all the way to offset
fileData = "Unable to traverse to offset; is offset larger than the file?";
} else {
// read may not read up to len, so only convert what was read
int read = is.read(data, 0, len);
if (read < 0) {
// stream couldn't read anything, skip went to EOF?
fileData = "Unable to read file";
} else {
fileData = WebUtils.convertByteArrayToStringWithoutEscape(data, 0, read);
}
}
}
} else {
fileData = "The requested file is not complete yet.";
}
List<UIFileBlockInfo> uiBlockInfo = new ArrayList<>();
for (FileBlockInfo fileBlockInfo : mFileSystemMaster.getFileBlockInfoList(absolutePath)) {
uiBlockInfo.add(new UIFileBlockInfo(fileBlockInfo, ServerConfiguration.global()));
}
response.setFileBlocks(uiBlockInfo).setFileData(fileData).setHighestTierAlias(mBlockMaster.getGlobalStorageTierAssoc().getAlias(0));
} catch (AlluxioException e) {
throw new IOException(e);
}
response.setViewingOffset(offset);
return response;
}
if (currentPath.isRoot()) {
response.setPathInfos(new UIFileInfo[0]);
} else {
String[] splitPath = PathUtils.getPathComponents(currentPath.toString());
UIFileInfo[] pathInfos = new UIFileInfo[splitPath.length - 1];
fileId = mFileSystemMaster.getFileId(currentPath);
pathInfos[0] = new UIFileInfo(mFileSystemMaster.getFileInfo(fileId), ServerConfiguration.global(), new MasterStorageTierAssoc().getOrderedStorageAliases());
AlluxioURI breadcrumb = new AlluxioURI(AlluxioURI.SEPARATOR);
for (int i = 1; i < splitPath.length - 1; i++) {
breadcrumb = breadcrumb.join(splitPath[i]);
fileId = mFileSystemMaster.getFileId(breadcrumb);
pathInfos[i] = new UIFileInfo(mFileSystemMaster.getFileInfo(fileId), ServerConfiguration.global(), new MasterStorageTierAssoc().getOrderedStorageAliases());
}
response.setPathInfos(pathInfos);
}
filesInfo = mFileSystemMaster.listStatus(currentPath, ListStatusContext.defaults());
} catch (FileDoesNotExistException e) {
response.setInvalidPathError("Error: Invalid Path " + e.getMessage());
return response;
} catch (InvalidPathException e) {
response.setInvalidPathError("Error: Invalid Path " + e.getLocalizedMessage());
return response;
} catch (UnavailableException e) {
response.setInvalidPathError("The service is temporarily unavailable. " + e.getMessage());
return response;
} catch (IOException e) {
response.setInvalidPathError("Error: File " + currentPath + " is not available " + e.getMessage());
return response;
} catch (AccessControlException e) {
response.setInvalidPathError("Error: File " + currentPath + " cannot be accessed " + e.getMessage());
return response;
}
List<UIFileInfo> fileInfos = new ArrayList<>(filesInfo.size());
for (FileInfo fileInfo : filesInfo) {
UIFileInfo toAdd = new UIFileInfo(fileInfo, ServerConfiguration.global(), new MasterStorageTierAssoc().getOrderedStorageAliases());
try {
if (!toAdd.getIsDirectory() && fileInfo.getLength() > 0) {
FileBlockInfo blockInfo = mFileSystemMaster.getFileBlockInfoList(new AlluxioURI(toAdd.getAbsolutePath())).get(0);
List<String> locations = new ArrayList<>();
// add the in-Alluxio block locations
for (BlockLocation location : blockInfo.getBlockInfo().getLocations()) {
WorkerNetAddress address = location.getWorkerAddress();
locations.add(address.getHost() + ":" + address.getDataPort());
}
// add underFS locations
locations.addAll(blockInfo.getUfsLocations());
toAdd.setFileLocations(locations);
}
} catch (FileDoesNotExistException e) {
response.setFileDoesNotExistException("Error: non-existing file " + e.getMessage());
return response;
} catch (InvalidPathException e) {
response.setInvalidPathException("Error: invalid path " + e.getMessage());
return response;
} catch (AccessControlException e) {
response.setAccessControlException("Error: File " + currentPath + " cannot be accessed " + e.getMessage());
return response;
}
fileInfos.add(toAdd);
}
fileInfos.sort(UIFileInfo.PATH_STRING_COMPARE);
response.setNTotalFile(fileInfos.size());
try {
int offset = Integer.parseInt(requestOffset);
int limit = Integer.parseInt(requestLimit);
limit = offset == 0 && limit > fileInfos.size() ? fileInfos.size() : limit;
limit = offset + limit > fileInfos.size() ? fileInfos.size() - offset : limit;
int sum = Math.addExact(offset, limit);
fileInfos = fileInfos.subList(offset, sum);
response.setFileInfos(fileInfos);
} catch (NumberFormatException e) {
response.setFatalError("Error: offset or limit parse error, " + e.getLocalizedMessage());
return response;
} catch (ArithmeticException e) {
response.setFatalError("Error: offset or offset + limit is out of bound, " + e.getLocalizedMessage());
return response;
} catch (IllegalArgumentException e) {
response.setFatalError(e.getLocalizedMessage());
return response;
}
return response;
}, ServerConfiguration.global());
}
use of alluxio.client.file.FileInStream in project zeppelin by apache.
the class AlluxioInterpreterTest method copyFromLocalFileToDstPathTest.
@Test
public void copyFromLocalFileToDstPathTest() throws IOException, AlluxioException {
String dataString = "copyFromLocalFileToDstPathTest";
byte[] data = dataString.getBytes();
File localDir = new File(mLocalAlluxioCluster.getAlluxioHome() + "/localDir");
localDir.mkdir();
File localFile = generateFileContent("/localDir/testFile", data);
alluxioInterpreter.interpret("mkdir /dstDir", null);
alluxioInterpreter.interpret("copyFromLocal " + localFile.getPath() + " /dstDir", null);
FileInStream fStream = fs.openFile(new AlluxioURI("/dstDir/testFile"));
long fileLength = fs.getStatus(new AlluxioURI("/dstDir/testFile")).getLength();
byte[] read = new byte[(int) fileLength];
fStream.read(read);
Assert.assertEquals(new String(read), dataString);
}
Aggregations