一、HDFS集群API所需要jar包的maven配置信息
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.8.4</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.8.4</version>
</dependency>
二、从HDFS下载数据文件/上传文件到HDFS文件系统
思路：1.获取配置信息
2.设置配置信息(块大小、副本数)
3.构造客户端
4.下载数据文件/上传数据文件
5.关闭资源
(1)下载文件
/**
* @author: PrincessHug
* @date: 2019/3/18, 16:10
* @Blog: https://www.cnblogs.com/HelloBigTable/
*/
public class HdfsClientDemo02 {
public static void main(String[] args) throws URISyntaxException, IOException, InterruptedException {
//配置信息对象
Configuration conf = new Configuration();
//设置具体配置信息
conf.set("dfs.replication","2");
//构造客户端
FileSystem fs = FileSystem.get(new URI("hdfs://192.168.126.128:9000/"), conf, "root"); //下载数据到本地
fs.copyToLocalFile(new Path("/words1.txt"),new Path("f://words1.txt"));
//关闭资源
fs.close();
System.out.println("下载完成");
}
}
（2）上传文件
/**
* @author: PrincessHug
* @date: 2019/3/18, 11:53
* @Blog: https://www.cnblogs.com/HelloBigTable/
*/
public class HdfsClientDemo01 {
public static void main(String[] args) throws URISyntaxException, IOException, InterruptedException {
//配置信息
Configuration conf = new Configuration();
//配置块大小和副本数
conf.set("dfs.blocksize","64m");
conf.set("dfs.replication","2");
//构造客户端
FileSystem fs = FileSystem.get(new URI("hdfs://192.168.126.128:9000/"), conf, "root");
//上传文件到hdfs客户端
fs.copyFromLocalFile(new Path("/root/love.tsv"),new Path("/love1.tsv"));
//关闭资源
fs.close();
System.out.println("上传成功！");
}
}
三、对HDFS系统进行操作的API
/**
* @author: PrincessHug
* @date: 2019/3/18, 16:16
* @Blog: https://www.cnblogs.com/HelloBigTable/
*/
public class HdfsClientDemo { private static FileSystem fs = null; static {
Configuration conf = new Configuration();
conf.set("dfs.blocksize","64m");
conf.set("dfs.replication","3");
try {
fs = FileSystem.get(new URI("hdfs://192.168.126.128:9000/"),conf,"root");
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (URISyntaxException e) {
e.printStackTrace();
}
} /**
* 创建文件夹方法
* @throws IOException
*/
public void mkDir(String path) throws IOException {
fs.mkdirs(new Path(path));
fs.close();
} /**
* 重命名或移动文件
* @param path1
* @param path2
* @throws IOException
*/
public void hdfsRename(String path1,String path2) throws IOException {
fs.rename(new Path(path1),new Path(path2));
fs.close();
} /**
* 删除文件或文件夹
* @param path 路径
* @throws IOException
*/
public void delete(String path) throws IOException {
fs.delete(new Path(path),true);
fs.close();
} /**
* 列出hdfs指定的目录信息
* @param path
* @throws IOException
*/
public void list(String path) throws IOException {
RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(new Path(path), true);
while (iterator.hasNext()){
//拿数据
LocatedFileStatus status = iterator.next();
System.out.println("文件的路径为：" + status.getPath());
System.out.println("文件的块大小为：" + status.getBlockSize());
System.out.println("文件的块信息为：" + Arrays.toString(status.getBlockLocations()));
System.out.println("文件的长度为：" + status.getLen());
System.out.println("文件的副本数为：" + status.getReplication());
System.out.println("=====================================================");
}
fs.close();
} /**
* 判断时文件还是文件夹
* @param parh
* @throws IOException
*/
public void judgeFileOrDir(String parh) throws IOException {
//展示状态信息
FileStatus[] fileStatuses = fs.listStatus(new Path(parh));
//遍历所有文件
for (FileStatus fs:fileStatuses){
if (fs.isFile()){
System.out.println("文件-----f------" + fs.getPath().getName());
}else {
System.out.println("文件-----d------" + fs.getPath().getName());
}
}
}
} public class HdfsDriver {
public static void main(String[] args) {
HdfsClientDemo hcd = new HdfsClientDemo();
try {
//hcd.mkDir("/wyh");
hcd.judgeFileOrDir("/");
hcd.list("/");
} catch (IOException e) {
e.printStackTrace();
}
}
}
四、