hadoop ClassNotFoundException

cookqq 发布于 2013/04/24 10:46
阅读 1K+
收藏 0

StudentinfoRecord这个类确实有,并且和DBInput在同一目录下

hduser@tianbaoxing-virtual-machine:/app/hadoop/tmp/java$ hadoop DBInput

ava$ hadoop DBInputWarning: $HADOOP_HOME is deprecated.


13/04/24 10:41:59 WARN mapred.JobClient: Use GenericOptionsParser for parsing the arguments. Applications should implement Tool for the same.
13/04/24 10:41:59 WARN mapred.JobClient: No job jar file set.  User classes may not be found. See JobConf(Class) or JobConf#setJar(String).
13/04/24 10:42:00 INFO mapred.JobClient: Running job: job_201304241011_0004
13/04/24 10:42:01 INFO mapred.JobClient:  map 0% reduce 0%
13/04/24 10:42:12 INFO mapred.JobClient: Task Id : attempt_201304241011_0004_m_000000_0, Status : FAILED
java.lang.RuntimeException: java.lang.ClassNotFoundException: StudentinfoRecord
    at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
    at org.apache.hadoop.mapred.lib.db.DBConfiguration.getInputClass(DBConfiguration.java:192)
    at org.apache.hadoop.mapred.lib.db.DBInputFormat.getRecordReader(DBInputFormat.java:284)
    at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.<init>(MapTask.java:197)
    at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:418)
    at org.apache.hadoop.mapred.MapTask.run(MapTask.java:372)
    at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:396)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
    at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: java.lang.ClassNotFoundException: StudentinfoRecord
    at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
    at java.security.AccessController.doPrivileged(Native Method)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
    at java.lang.Class.forName0(Native Method)
    at java.lang.Class.forName(Class.java:247)
    at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
    at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
    ... 10 more

13/04/24 10:42:12 INFO mapred.JobClient: Task Id : attempt_201304241011_0004_m_000001_0, Status : FAILED
java.lang.RuntimeException: java.lang.ClassNotFoundException: StudentinfoRecord
    at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
    at org.apache.hadoop.mapred.lib.db.DBConfiguration.getInputClass(DBConfiguration.java:192)
    at org.apache.hadoop.mapred.lib.db.DBInputFormat.getRecordReader(DBInputFormat.java:284)
    at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.<init>(MapTask.java:197)
    at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:418)
    at org.apache.hadoop.mapred.MapTask.run(MapTask.java:372)
    at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:396)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
    at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: java.lang.ClassNotFoundException: StudentinfoRecord
    at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
    at java.security.AccessController.doPrivileged(Native Method)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
    at java.lang.Class.forName0(Native Method)
    at java.lang.Class.forName(Class.java:247)
    at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
    at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
    ... 10 more

13/04/24 10:42:21 INFO mapred.JobClient: Task Id : attempt_201304241011_0004_m_000000_1, Status : FAILED
java.lang.RuntimeException: java.lang.ClassNotFoundException: StudentinfoRecord
    at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
    at org.apache.hadoop.mapred.lib.db.DBConfiguration.getInputClass(DBConfiguration.java:192)
    at org.apache.hadoop.mapred.lib.db.DBInputFormat.getRecordReader(DBInputFormat.java:284)
    at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.<init>(MapTask.java:197)
    at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:418)
    at org.apache.hadoop.mapred.MapTask.run(MapTask.java:372)
    at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:396)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
    at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: java.lang.ClassNotFoundException: StudentinfoRecord
    at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
    at java.security.AccessController.doPrivileged(Native Method)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
    at java.lang.Class.forName0(Native Method)
    at java.lang.Class.forName(Class.java:247)
    at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
    at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
    ... 10 more

13/04/24 10:42:21 INFO mapred.JobClient: Task Id : attempt_201304241011_0004_m_000001_1, Status : FAILED
java.lang.RuntimeException: java.lang.ClassNotFoundException: StudentinfoRecord
    at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
    at org.apache.hadoop.mapred.lib.db.DBConfiguration.getInputClass(DBConfiguration.java:192)
    at org.apache.hadoop.mapred.lib.db.DBInputFormat.getRecordReader(DBInputFormat.java:284)
    at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.<init>(MapTask.java:197)
    at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:418)
    at org.apache.hadoop.mapred.MapTask.run(MapTask.java:372)
    at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:396)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
    at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: java.lang.ClassNotFoundException: StudentinfoRecord
    at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
    at java.security.AccessController.doPrivileged(Native Method)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
    at java.lang.Class.forName0(Native Method)
    at java.lang.Class.forName(Class.java:247)
    at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
    at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
    ... 10 more

13/04/24 10:42:28 INFO mapred.JobClient: Task Id : attempt_201304241011_0004_m_000000_2, Status : FAILED
java.lang.RuntimeException: java.lang.ClassNotFoundException: StudentinfoRecord
    at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
    at org.apache.hadoop.mapred.lib.db.DBConfiguration.getInputClass(DBConfiguration.java:192)
    at org.apache.hadoop.mapred.lib.db.DBInputFormat.getRecordReader(DBInputFormat.java:284)
    at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.<init>(MapTask.java:197)
    at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:418)
    at org.apache.hadoop.mapred.MapTask.run(MapTask.java:372)
    at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:396)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
    at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: java.lang.ClassNotFoundException: StudentinfoRecord
    at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
    at java.security.AccessController.doPrivileged(Native Method)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
    at java.lang.Class.forName0(Native Method)
    at java.lang.Class.forName(Class.java:247)
    at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
    at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
    ... 10 more

13/04/24 10:42:29 INFO mapred.JobClient: Task Id : attempt_201304241011_0004_m_000001_2, Status : FAILED
java.lang.RuntimeException: java.lang.ClassNotFoundException: StudentinfoRecord
    at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
    at org.apache.hadoop.mapred.lib.db.DBConfiguration.getInputClass(DBConfiguration.java:192)
    at org.apache.hadoop.mapred.lib.db.DBInputFormat.getRecordReader(DBInputFormat.java:284)
    at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.<init>(MapTask.java:197)
    at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:418)
    at org.apache.hadoop.mapred.MapTask.run(MapTask.java:372)
    at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:396)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
    at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: java.lang.ClassNotFoundException: StudentinfoRecord
    at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
    at java.security.AccessController.doPrivileged(Native Method)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
    at java.lang.Class.forName0(Native Method)
    at java.lang.Class.forName(Class.java:247)
    at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
    at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
    ... 10 more

13/04/24 10:42:40 INFO mapred.JobClient: Job complete: job_201304241011_0004
13/04/24 10:42:40 INFO mapred.JobClient: Counters: 6
13/04/24 10:42:40 INFO mapred.JobClient:   Job Counters
13/04/24 10:42:40 INFO mapred.JobClient:     SLOTS_MILLIS_MAPS=70989
13/04/24 10:42:40 INFO mapred.JobClient:     Total time spent by all reduces waiting after reserving slots (ms)=0
13/04/24 10:42:40 INFO mapred.JobClient:     Total time spent by all maps waiting after reserving slots (ms)=0
13/04/24 10:42:40 INFO mapred.JobClient:     Launched map tasks=8
13/04/24 10:42:40 INFO mapred.JobClient:     SLOTS_MILLIS_REDUCES=0
13/04/24 10:42:40 INFO mapred.JobClient:     Failed map tasks=1
13/04/24 10:42:40 INFO mapred.JobClient: Job Failed: # of failed Map Tasks exceeded allowed limit. FailedCount: 1. LastFailedTask: task_201304241011_0004_m_000000
Exception in thread "main" java.io.IOException: Job failed!
    at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:1327)
    at DBInput.main(DBInput.java:37)
hduser@tianbaoxing-virtual-machine:/app/hadoop/tmp/java$

加载中
0
逝水fox
逝水fox

13/04/24 10:41:59 WARN mapred.JobClient: No job jar file set.  User classes may not be found. See JobConf(Class) or JobConf#setJar(String)

我把hadoop从0.2x升级到1.x之后,同样的代码也出过这个问题,在代码里面加了那个setJar()的调用后就正常了,不知道是不是也是同样的问题

cookqq
cookqq
我在代码里加了: conf.setJarByClass(DBInput.class);
0
cookqq
cookqq
这个问题找到了,我使用的是1.1.2,需要打包成jar文件运行。然后就没有报错
0
chyileon
chyileon
不打包也行,需要设置HADOOP_CLASSPATH
0
Rain-suny
Rain-suny

我的HADOOP_CLASSPATH设置如下:

.:/usr/hadoop/hadoop-core-0.20.1.jar:/usr/jar/ant.jar

但是打包不打包都不行。
0
z
zhengmingzhao
13/10/30 11:30:24 WARN mapred.JobClient: Use GenericOptionsParser for parsing the arguments. Applications should implement Tool for the same.
13/10/30 11:30:24 WARN mapred.JobClient: No job jar file set.  User classes may not be found. See JobConf(Class) or JobConf#setJar(String).
13/10/30 11:30:24 INFO input.FileInputFormat: Total input paths to process : 0
13/10/30 11:30:45 INFO mapred.JobClient: Running job: job_201310291914_0010
13/10/30 11:30:46 INFO mapred.JobClient:  map 0% reduce 0%
13/10/30 11:31:41 INFO mapred.JobClient: Task Id : attempt_201310291914_0010_r_000000_0, Status : FAILED
java.lang.RuntimeException: java.lang.ClassNotFoundException: org.MyReduce
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
at org.apache.hadoop.mapreduce.JobContext.getCombinerClass(JobContext.java:211)
at org.apache.hadoop.mapred.Task$CombinerRunner.create(Task.java:1400)
at org.apache.hadoop.mapred.ReduceTask$ReduceCopier.<init>(ReduceTask.java:1939)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:385)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: java.lang.ClassNotFoundException: org.MyReduce
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:270)
at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
... 9 more


13/10/30 11:32:01 INFO mapred.JobClient: Task Id : attempt_201310291914_0010_r_000000_1, Status : FAILED
java.lang.RuntimeException: java.lang.ClassNotFoundException: org.MyReduce
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
at org.apache.hadoop.mapreduce.JobContext.getCombinerClass(JobContext.java:211)
at org.apache.hadoop.mapred.Task$CombinerRunner.create(Task.java:1400)
at org.apache.hadoop.mapred.ReduceTask$ReduceCopier.<init>(ReduceTask.java:1939)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:385)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: java.lang.ClassNotFoundException: org.MyReduce
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:270)
at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
... 9 more


13/10/30 11:32:20 INFO mapred.JobClient: Task Id : attempt_201310291914_0010_r_000000_2, Status : FAILED
java.lang.RuntimeException: java.lang.ClassNotFoundException: org.MyReduce
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:849)
at org.apache.hadoop.mapreduce.JobContext.getCombinerClass(JobContext.java:211)
at org.apache.hadoop.mapred.Task$CombinerRunner.create(Task.java:1400)
at org.apache.hadoop.mapred.ReduceTask$ReduceCopier.<init>(ReduceTask.java:1939)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:385)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1149)
at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: java.lang.ClassNotFoundException: org.MyReduce
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:270)
at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:802)
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:847)
... 9 more


13/10/30 11:33:16 INFO mapred.JobClient: Job complete: job_201310291914_0010
13/10/30 11:33:16 INFO mapred.JobClient: Counters: 6
13/10/30 11:33:16 INFO mapred.JobClient:   Job Counters 
13/10/30 11:33:16 INFO mapred.JobClient:     Launched reduce tasks=4
13/10/30 11:33:16 INFO mapred.JobClient:     SLOTS_MILLIS_MAPS=52652
13/10/30 11:33:16 INFO mapred.JobClient:     Total time spent by all reduces waiting after reserving slots (ms)=0
13/10/30 11:33:16 INFO mapred.JobClient:     Total time spent by all maps waiting after reserving slots (ms)=0
13/10/30 11:33:16 INFO mapred.JobClient:     Failed reduce tasks=1
13/10/30 11:33:16 INFO mapred.JobClient:     SLOTS_MILLIS_REDUCES=76644

0
z
zhengmingzhao
你好我的问题和你的差不多,但是不知道是哪里出错了,能帮忙看一下吗,多谢啦~
z
zhengmingzhao
回复 @田保兴 : 这样改是成功了,但是每次都要打包吗,很麻烦的,我在代码里加了conf.setJarByClass但是不行
cookqq
cookqq
你打完包,然后放到相应的class路径中。
返回顶部
顶部