一、读取HDFS文件数据、将本地文件写入HDFS中文件、使用IOUtils读写数据
**
* @author: PrincessHug
* @date: 2019/3/18, 17:24
* @Blog: https://www.cnblogs.com/HelloBigTable/
*/
public class HdfsClientDemo03 {
FileSystem fs = null;
Configuration conf = null; @Before
public void init() throws URISyntaxException, IOException, InterruptedException {
conf = new Configuration();
fs = FileSystem.get(new URI("hdfs://192.168.126.128:9000/"),conf,"root");
} /**
* 使用缓冲流读数据
* @throws IOException
*/
@Test
public void ReadData01() throws IOException {
FSDataInputStream in = fs.open(new Path("/words1.txt"));
BufferedReader br = new BufferedReader(new InputStreamReader(in, "UTF-8"));
String line = null;
//读数据
while ((line = br.readLine()) != null){
System.out.println(line);
}
//关闭资源
br.close();
in.close();
fs.close();
} /**
* 使用字节数据来接收数据
* @throws IOException
*/
@Test
public void ReadData02() throws IOException {
FSDataInputStream in = fs.open(new Path("/words1.txt"));
byte[] bytes = new byte[1024];
in.read(bytes);
System.out.println(new String(bytes));
in.close();
fs.close();
} /**
* 读取自定义偏移量的数据
* @throws IOException
*/
@Test
public void randomRead() throws IOException {
FSDataInputStream in = fs.open(new Path("/words1.txt"));
in.seek(10);
byte[] bytes = new byte[1024];
in.read(bytes);
System.out.println(new String(bytes));
in.close();
} /**
* 从本地文件读取数据写入hdfs文件中
* @throws IOException
*/
@Test
public void writeData() throws IOException {
FSDataOutputStream out = fs.create(new Path("/window1.txt"), false);
FileInputStream in = new FileInputStream("G:\\潭州课堂笔记视频作业\\java\\集合.txt");
byte[] bytes = new byte[1024];
int read = 0;
while ((read = in.read(bytes)) != -1){
out.write(bytes,0,read);
}
in.close();
out.close();
fs.close();
} /**
* 向hdfs文件中写自定义的数据
*/
@Test
public void writeData01() throws IOException {
FSDataOutputStream out = fs.create(new Path("/Wyh"));
out.write("I love dilireba".getBytes());
out.close();
fs.close();
} /**
* 使用IOUtils的传输流方法上传文件
* @throws IOException
*/
@Test
public void putFileToHdfs() throws IOException {
//从本地文件获取数据流
FileInputStream fis = new FileInputStream(new File("G:\\潭州课堂笔记视频作业\\java\\多线程.txt"));
//定义输出流对象
FSDataOutputStream fos = fs.create(new Path("/threads.txt"));
//流的传输
IOUtils.copyBytes(fis,fos,conf);
//关闭流
IOUtils.closeStream(fis);
IOUtils.closeStream(fos);
fs.close();
} @Test
public void getFileFromHdfs() throws IOException {
//从hdfs文件获取输入流
FSDataInputStream fis = fs.open(new Path("/threads.txt"));
//定义输出流对象
FileOutputStream fos = new FileOutputStream(new File("G:\\潭州课堂笔记视频作业\\java\\threads.txt"));
//流的传输
IOUtils.copyBytes(fis,fos,conf);
//关闭流
IOUtils.closeStream(fis);
IOUtils.closeStream(fos);
fs.close();
}
}