hadoop实验课-hdfs

---

HDFS实验

通过Java API操作HDFS

准备工作

导入hdfs的jar包

1.将/usr/…/share/common下；/usr/…/share/common/lib下；/usr/…/share/hdfs/下的jar包导入到hadoop项目下的lib

2.eclipse的项目单击右键build path->configure..->libraries->external jars

代码编写

文件上传

//
package sugon.edu.hadoop.hdfs;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class putFile_HDFS {
	public static void main(String[] args) throws Exception
	{
	Configuration conf = new Configuration();
	URI uri = new URI("hdfs://192.168.206.129:8020");
	FileSystem fs = FileSystem.get(uri, conf, "ttyong");
	Path src = new Path("/home/ttyong/Public/java/wordCount.txt");
	Path dst = new Path("/mydir/");
	fs.copyFromLocalFile(src, dst);
	fs.close();
	System.out.println("upload successfully");
	}
}

// 流拷贝上传
package sugon.edu.hadoop.hdfs;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

public class putFile_HDFS_byLiuKaoBei {
	public static void main(String[] args) throws Exception
	{
	Configuration conf = new Configuration();
	URI uri = new URI("hdfs://192.168.206.129:8020");
	FileSystem fs = FileSystem.get(uri, conf, "ttyong");
	InputStream is = new FileInputStream("/home/ttyong/Public/java/salaryTotal.txt");
	OutputStream os = fs.create(new Path("/mydir/salaryTotal.txt"));
	IOUtils.copyBytes(is, os, 1024);
	is.close();
	os.close();
	fs.close();
	System.out.println("upload successfully");
}
}

文件下载

package sugon.edu.hadoop.hdfs;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

public class getFile_HDFS {
	public static void main(String[] args) throws Exception
	{	
		Configuration conf = new Configuration();
		URI uri = new URI("hdfs://192.168.206.129:8020");
		FileSystem fs = FileSystem.get(uri, conf, "ttyong");
		Path src = new Path("/mydir/salaryTotal.txt");
		Path dst = new Path("/home/ttyong/Public/java/test4.txt");
		System.setProperty("HADOOP_USER_NAME", "root");
		fs.copyToLocalFile(src, dst); //linux
		fs.close();
		System.out.println("upload successfully");
}
}

// 流拷贝下载
package sugon.edu.hadoop.hdfs;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

public class getFile_HDFS_ByLiukaobei {
	public static void main(String[] args) throws Exception
	{
		Configuration conf = new Configuration();
		URI uri = new URI("hdfs://192.168.206.129:8020");
		FileSystem fs = FileSystem.get(uri, conf, "ttyong");
		InputStream is = fs.open(new Path("/mydir/salaryTotal.txt"));
		OutputStream os = new 		       FileOutputStream("/home/ttyong/Public/java/test5.txt");
		IOUtils.copyBytes(is, os, 1024);
		is.close();
		os.close();
		fs.close();
		System.out.println("upload successfully");
}
}

文件创建

package sugon.edu.hadoop.hdfs;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class CreateFile_HDFS {
	public static void main(String[] args) throws Exception{
		// TODO Auto-generated method stub
		System.setProperty("HADOOP_USER_NAME", "root");
		Configuration conf = new Configuration();
		URI uri = new URI("hdfs://192.168.206.129:8020");
		FileSystem fs = FileSystem.get(uri, conf, "ttyong");
		Path path = new Path("/mydir/test2.txt");
		FSDataOutputStream os = fs.create(path);
		os.writeBytes("hello HDFS");
		os.close();
		fs.close();
	}
}

文件删除

package sugon.edu.hadoop.hdfs;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class DeleteFile_HDFS {
	public static void main(String[] args) throws Exception
	{
	Configuration conf = new Configuration();
	URI uri = new URI("hdfs://192.168.206.129:8020");
	FileSystem fs = FileSystem.get(uri, conf, "ttyong");
	Path path = new Path("/mydir/test2.txt");
	fs.delete(path);
	fs.close();
}
}

shell命令访问HDFS

管理命令

操作命令

---