博客
关于我
强烈建议你试试无所不能的chatGPT,快点击我
大数据笔记(十二)——使用MRUnit进行单元测试
阅读量:4482 次
发布时间:2019-06-08

本文共 3957 字,大约阅读时间需要 13 分钟。

package demo.wc;import java.util.ArrayList;import java.util.List;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mrunit.mapreduce.MapDriver;import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;import org.junit.Test;public class MRUnitWordCount {    @Test    public void testMapper() throws Exception{        //设置一个环境变量(没有可能会报错)        System.setProperty("hadoop.home.dir", "D:\\temp\\hadoop-2.4.1\\hadoop-2.4.1");                //创建一个测试对象        WordCountMapper mapper = new WordCountMapper();                //创建一个MapDriver进行单元测试        MapDriver
driver = new MapDriver<>(mapper); //ָ指定Map的输入值: k1 v1 driver.withInput(new LongWritable(1), new Text("I love Beijing")); //ָ指定Map的输出值:k2 v2 ----> 期望值 driver.withOutput(new Text("I"), new IntWritable(1)) .withOutput(new Text("love"), new IntWritable(1)) .withOutput(new Text("Beijing"), new IntWritable(1)); //ִ执行单元测试,对比 期望的结果和实际的结果 driver.runTest(); } @Test public void testReducer() throws Exception{ //设置一个环境变量 System.setProperty("hadoop.home.dir", "D:\\temp\\hadoop-2.4.1\\hadoop-2.4.1"); //创建一个测试对象 WordCountReducer reducer = new WordCountReducer(); //创建一个ReduceDriver进行单元测试 ReduceDriver
driver = new ReduceDriver<>(reducer); //构造v3:List List
value3 = new ArrayList<>(); value3.add(new IntWritable(1)); value3.add(new IntWritable(1)); value3.add(new IntWritable(1)); //指定reducer的输入 driver.withInput(new Text("Beijing"), value3); //指定reducer的输出 driver.withOutput(new Text("Beijing"), new IntWritable(3)); //ִ执行测试 driver.runTest(); } @Test public void testJob() throws Exception{ //设置一个环境变量 System.setProperty("hadoop.home.dir", "D:\\temp\\hadoop-2.4.1\\hadoop-2.4.1"); //创建一个测试对象 WordCountMapper mapper = new WordCountMapper(); WordCountReducer reducer = new WordCountReducer(); //创建一个Driver //MapReduceDriver
MapReduceDriver
driver = new MapReduceDriver<>(mapper,reducer); //指定Map输入的数据 driver.withInput(new LongWritable(1), new Text("I love Beijing")) .withInput(new LongWritable(4), new Text("I love China")) .withInput(new LongWritable(7), new Text("Beijing is the capital of China")); //ָ指定Reducer的输出// driver.withOutput(new Text("I"), new IntWritable(2))// .withOutput(new Text("love"), new IntWritable(2))// .withOutput(new Text("Beijing"), new IntWritable(2))// .withOutput(new Text("China"), new IntWritable(2))// .withOutput(new Text("is"), new IntWritable(1))// .withOutput(new Text("the"), new IntWritable(1))// .withOutput(new Text("capital"), new IntWritable(1))// .withOutput(new Text("of"), new IntWritable(1)); //指定Reducer的输出(默认排序规则) driver.withOutput(new Text("Beijing"), new IntWritable(2)) .withOutput(new Text("China"), new IntWritable(2)) .withOutput(new Text("I"), new IntWritable(2)) .withOutput(new Text("capital"), new IntWritable(1)) .withOutput(new Text("is"), new IntWritable(1)) .withOutput(new Text("love"), new IntWritable(2)) .withOutput(new Text("of"), new IntWritable(1)) .withOutput(new Text("the"), new IntWritable(1)); driver.runTest(); }}

 

转载于:https://www.cnblogs.com/lingluo2017/p/8540275.html

你可能感兴趣的文章
中兴电信光纤猫F450获取管理员密码方法
查看>>
申请TexturePacker 或 PhysicsEditor free licenses
查看>>
kafka启动报错&问题解决
查看>>
nginx反向代理下没有获取到正确的clientIP问题发散
查看>>
python周报第一周
查看>>
IBM MQ 创建以及常见问题集锦
查看>>
Office文件的奥秘——.NET平台下不借助Office实现Word、Powerpoint等文件的解析(1)
查看>>
SQL Server 服务器磁盘测试之SQLIO篇(一)
查看>>
JQ插件写法 扩展JQ方法
查看>>
[LeetCode&Python] Problem 543. Diameter of Binary Tree
查看>>
226 Invert Binary Tree 翻转二叉树
查看>>
《Pro ASP.NET MVC 3 Framework》学习笔记之十六【示例项目SportsStore】
查看>>
IntelliJ IDEA的安装及永久破解
查看>>
C#多线程编程
查看>>
替换空格
查看>>
IntelliJ IDEA2018.1、2017.3激活
查看>>
Orchard后台控制面板的介绍
查看>>
大二下第一周----开学测试
查看>>
javaweb-servlet生成简单的验证码
查看>>
apache+php+mysql环境搭建时,phpinfo里面没有mysql解决办法
查看>>