使用Java API操作HDFS文件系统
关键点:
- 1)创建Configuration
- 2)获取FileSystem
- 3)…就是你的HDFS API的操作
首先创建FileSystem
构造一个访问指定HDFS系统的客户端对象
第一个参数:HDFS的URI
第二个参数:客户端指定的配置参数
第三个参数:客户端的身份,说白了就是用户名
FileSystemfileSystem = FileSystem.get(new URI(“hdfs://hadoop000:8020”), new configuration, “hadoop”);
1.创建HDFS文件夹
@Test
public void mkdir() throws Exception {
fileSystem.mkdirs(new Path("/hdfsapi/test"));
}
2.查看HDFS内容
import org.apache.hadoop.io.IOUtils;
@Test
public void text()throws Exception {
FSDataInputStream in = fileSystem.open(new Path("/cdh_version.properties"));
IOUtils.copyBytes(in, System.out, 1024);
}
3.创建文件
public void create()throws Exception {
FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/b.txt"));
out.writeUTF("hello DM replication 1");
out.flush();
out.close();
}
4.测试文件名更改
@Test
public void rename() throws Exception {
Path oldPath = new Path("/hdfsapi/test/b.txt");
Path newPath = new Path("/hdfsapi/test/c.txt");
boolean result = fileSystem.rename(oldPath, newPath);
System.out.println(result);
}
5.拷贝本地文件到HDFS文件系统(put)
@Test
public void copyFromLocalFile() throws Exception {
Path src = new Path("/Users/rocky/data/hello.txt");
Path dst = new Path("/hdfsapi/test/");
fileSystem.copyFromLocalFile(src,dst);
}
6.拷贝HDFS文件到本地(get)
@Test
public void copyToLocalFile() throws Exception {
Path src = new Path("/hdfsapi/test/hello.txt");
Path dst = new Path("/Users/rocky/tmp/software");
fileSystem.copyToLocalFile(src, dst);
}
7.查看目标文件夹下的所有文件
@Test
public void listFiles() throws Exception {
FileStatus[] statuses = fileSystem.listStatus(new Path("/hdfsapi/test"));
for(FileStatus file : statuses) {
String isDir = file.isDirectory() ? "文件夹" : "文件";
String permission = file.getPermission().toString();
short replication = file.getReplication();
long length = file.getLen();
String path = file.getPath().toString();
System.out.println(isDir + "\t" + permission
+ "\t" + replication + "\t" + length
+ "\t" + path
);
}
}
8.递归查看目标文件夹下的所有文件
@Test
public void listFilesRecursive() throws Exception {
RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path("/hdfsapi/test"), true);
while (files.hasNext()) {
LocatedFileStatus file = files.next();
String isDir = file.isDirectory() ? "文件夹" : "文件";
String permission = file.getPermission().toString();
short replication = file.getReplication();
long length = file.getLen();
String path = file.getPath().toString();
System.out.println(isDir + "\t" + permission
+ "\t" + replication + "\t" + length
+ "\t" + path
);
}
}
9.查看文件块信息
@Test
public void getFileBlockLocations() throws Exception {
FileStatus fileStatus = fileSystem.getFileStatus(new Path("/hdfsapi/test/jdk.tgz"));
BlockLocation[] blocks = fileSystem.getFileBlockLocations(fileStatus,0,fileStatus.getLen());
for(BlockLocation block : blocks) {
for(String name: block.getNames()) {
System.out.println(name +" : " + block.getOffset() + " : " + block.getLength() + " : " + block.getHosts());
}
}
}
10删除文件
@Test
public void delete() throws Exception {
boolean result = fileSystem.delete(new Path("/hdfsapi/test/jdk.tgz"), true);
System.out.println(result);
}