getJdbcTemplate().query(getFileContentSql, new RowMapper<Void>() {
public void mapRow(ResultSet rs, int rowNum) throws SQLException {
OracleLobHandler lobHandler = new OracleLobHandler();
InputStream inputStream = lobHandler.getBlobAsBinaryStream(rs, "file_content");
ExlImporter importer = new ExlBOMImporter(inputStream);
importer.process();
}
}, fileId);
Please tell me what is Void ?
Note : V is capital.
Form the javadoc:
The Void class is an uninstantiable placeholder class to hold a
reference to the Class object representing the Java keyword void.
Related
I am using Hadoop 2.7 and I have got an issue when using a custom Writable "TextPair" (page 104 of the Definitive Guide). Basically, my program works fine when I am using just Text whereas it outputs "test.TextTuple#3b86249a test.TextTuple#63cd18fd" when using the TextPair.
Please, Any idea of what is wrong with my code (below)?
============
Mapper1:
public class KWMapper extends Mapper<LongWritable, Text, TextTuple, TextTuple> {
#Override
public void map(LongWritable k, Text v, Mapper.Context c) throws IOException, InterruptedException {
String keywordRelRecord[] = v.toString().split(",");
String subTopicID = keywordRelRecord[0];
String paperID = keywordRelRecord[1];
//set the KEY
TextTuple key = new TextTuple();
key.setNaturalKey(new Text(subTopicID));
key.setSecondaryKey(new Text("K"));
//set the VALUE
TextTuple value = new TextTuple();
value.setNaturalKey(new Text(paperID));
value.setSecondaryKey(new Text("K"));
c.write(key, value);
}
Mapper2:
public class TDMapper extends Mapper<LongWritable, Text, TextTuple, TextTuple> {
#Override
public void map(LongWritable k, Text v, Mapper.Context c) throws IOException, InterruptedException {
String topicRecord[] = v.toString().split(",");
String superTopicID = topicRecord[0];
String subTopicID = topicRecord[1].substring(1, topicRecord[1].length() - 1);
TextTuple key = new TextTuple();
key.setNaturalKey(new Text(subTopicID));
key.setSecondaryKey(new Text("T"));
TextTuple value = new TextTuple();
value.setNaturalKey(new Text(superTopicID));
value.setSecondaryKey(new Text("T"));
c.write(key, value);
}
REDUCER :
public class TDKRReducer extends Reducer<TextTuple, TextTuple, Text, Text>{
public void reduce(TextTuple k, Iterable<TextTuple> values, Reducer.Context c) throws IOException, InterruptedException{
for (TextTuple val : values) {
c.write(k.getNaturalKey(), val.getNaturalKey());
}
}
}
DRIVER:
public class TDDriver {
public static void main(String args[]) throws IOException, InterruptedException, ClassNotFoundException {
// This class support the user for the configuration of the execution;
Configuration confStage1 = new Configuration();
Job job1 = new Job(confStage1, "TopDecKeywordRel");
// Setting the driver class
job1.setJarByClass(TDDriver.class);
// Setting the input Files and processing them using the corresponding mapper class
MultipleInputs.addInputPath(job1, new Path(args[0]), TextInputFormat.class, TDMapper.class);
MultipleInputs.addInputPath(job1, new Path(args[1]), TextInputFormat.class, KWMapper.class);
job1.setMapOutputKeyClass(TextTuple.class);
job1.setMapOutputValueClass(TextTuple.class);
// Setting the Reducer Class;
job1.setReducerClass(TDKRReducer.class);
// Setting the output class for the Key-value pairs
job1.setOutputKeyClass(Text.class);
job1.setOutputValueClass(Text.class);
// Setting the output file
Path outputPA = new Path(args[2]);
FileOutputFormat.setOutputPath(job1, outputPA);
// Submitting the Job Monitoring the execution of the Job
System.exit(job1.waitForCompletion(true) ? 0 : 1);
//conf.setPartitionerClass(CustomPartitioner.class);
}
}
CUSTOM VARIABLE
public class TextTuple implements Writable, WritableComparable<TextTuple> {
private Text naturalKey;
private Text secondaryKey;
public TextTuple() {
this.naturalKey = new Text();
this.secondaryKey = new Text();
}
public void setNaturalKey(Text naturalKey) {
this.naturalKey = naturalKey;
}
public void setSecondaryKey(Text secondaryKey) {
this.secondaryKey = secondaryKey;
}
public Text getNaturalKey() {
return naturalKey;
}
public Text getSecondaryKey() {
return secondaryKey;
}
#Override
public void write(DataOutput out) throws IOException {
naturalKey.write(out);
secondaryKey.write(out);
}
#Override
public void readFields(DataInput in) throws IOException {
naturalKey.readFields(in);
secondaryKey.readFields(in);
}
//This comparator controls the sort order of the keys.
#Override
public int compareTo(TextTuple o) {
// comparing the naturalKey
int compareValue = this.naturalKey.compareTo(o.naturalKey);
if (compareValue == 0) {
compareValue = this.secondaryKey.compareTo(o.secondaryKey);
}
return -1 * compareValue;
}
}
I have the following classes for MR jobs but when i run the job the job is failing with the below exception kindly suggest.
public class MongoKey implements WritableComparable<MongoKey> {
...
private Text name;
private Text place;
public MongoKey() {
this.name = new Text();
this.place = new Text();
}
public MongoKey(Text name, Text place) {
this.name = name;
this.place = place;
}
public void readFields(DataInput in) throws IOException {
name.readFields(in);
place.readFields(in);
}
public void write(DataOutput out) throws IOException {
name.write(out);
place.write(out);
}
public int compareTo(MongoKey o) {
MongoKey other = (MongoKey)o;
int cmp = name.compareTo(other.name);
if(cmp != 0){
return cmp;
}
return place.compareTo(other.place);
}
}
public class MongoValue implements Writable {
...
public void readFields(DataInput in) throws IOException {
profession.readFields(in);
}
public void write(DataOutput out) throws IOException {
profession.write(out);
}
}
public class MongoReducer extends Reducer<MongoKey, MongoValue, MongoKey, BSONWritable> {
...
context.write(key, new BSONWritable(output)); // line 41
}
public class MongoHadoopJobRunner extends Configured implements Tool {
public int run(String[] args) throws Exception {
if (args.length != 2) {
System.out.println("usage: [input] [output]");
System.exit(-1);
}
Configuration conf = getConf();
for (String arg : args)
System.out.println(arg);
GenericOptionsParser parser = new GenericOptionsParser(conf, args);
conf.set("mongo.output.uri", "mongodb://localhost/demo.logs_aggregate");
MongoConfigUtil.setOutputURI(conf, "mongodb://localhost/demo.logs_aggregate");
MongoConfigUtil.setOutputFormat(conf, MongoOutputFormat.class);
final Job job = new Job(conf, "mongo_hadoop");
job.setOutputFormatClass(MongoOutputFormat.class);
// Job job = new Job();
job.setJarByClass(MongoHadoopJobRunner.class);
// job.setJobName("mongo_hadoop");
job.setNumReduceTasks(1);
job.setMapperClass(MongoMapper.class);
job.setReducerClass(MongoReducer.class);
job.setMapOutputKeyClass(MongoKey.class);
job.setMapOutputValueClass(MongoValue.class);
job.setOutputKeyClass(MongoKey.class);
job.setOutputValueClass(BSONWritable.class);
job.setInputFormatClass(MongoInputFormat.class);
for (String arg2 : parser.getRemainingArgs()) {
System.out.println("remaining: " + arg2);
}
Path inPath = new Path(parser.getRemainingArgs()[0]);
MongoInputFormat.addInputPath(job, inPath);
job.waitForCompletion(true);
return 0;
}
public static void main(String[] pArgs) throws Exception {
Configuration conf = new Configuration();
for (String arg : pArgs) {
System.out.println(arg);
}
GenericOptionsParser parser = new GenericOptionsParser(conf, pArgs);
for (String arg2 : parser.getRemainingArgs()) {
System.out.println("ree" + arg2);
}
System.exit(ToolRunner.run(conf, new MongoHadoopJobRunner(), parser
.getRemainingArgs()));
}
}
With the following exception
java.lang.Exception: java.lang.IllegalArgumentException: can't serialize class com.name.custom.MongoKey
...
...
at com.mongodb.hadoop.output.MongoRecordWriter.write(MongoRecordWriter.java:93)
at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:558)
at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89)
at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105)
at com.name.custom.MongoReducer.reduce(MongoReducer.java:41)
at com.name.custom.MongoReducer.reduce(MongoReducer.java:11)
It seems there should not be any issue with the code but why its unable to serialize the fields i am totally clueless.
Thanks very much in advance
As i see from MongoRecordWriter source code it does not support arbitrary WritableComparable object as key. You can use one of these classes as key: BSONWritable, BSONObject, Text, UTF8, simple wrappers like IntWritable. Also i think you can use Serializable object as key. So i can suggest you two workarounds:
Make your MongoKey serializable (implements Serializable, implement writeObject, readObject methods).
Use one of supported classes as key, for example you can use Text as key: Text key = new Text(name.toString() + "\t" + place.toString());
This:
java.lang.Exception: java.lang.IllegalArgumentException: can't serialize class com.name.custom.MongoKey
exception is raised because MongoKey doesn't implement java.io.Serializable.
Add the Serializable to your class declaration
I cannot figure out an answer to the following problem. Hope somebody can help me. I am mapping JAVA class to a Struct as described here:
http://docs.oracle.com/cd/F49540_01/DOC/java.815/a64685/samapp4.htm
I have an Oracle Object:
create or replace TYPE DK1 AS OBJECT( zahl CHAR(1) );
and corresponding JAVA class:
public class DK1 implements SQLData {
private String sql_type;
public static final int _SQL_TYPECODE = OracleTypes.STRUCT;
private String zahl;
public DK1(String sql_type, String z) {
this.sql_type = sql_type;
setZahl(z);
}
public String getSQLTypeName() throws SQLException {
return sql_type;
}
public void readSQL(SQLInput stream, String typeName) throws SQLException {
sql_type = typeName;
this.setZahl(stream.readString());
}
public void writeSQL(SQLOutput stream) throws SQLException {
stream.writeString(getZahl());
}
public String getSql_type() {
return sql_type;
}
public void setSql_type(String sql_type) {
this.sql_type = sql_type;
}
public String getZahl() {
return zahl;
}
public void setZahl(String zahl) {
this.zahl = zahl;
}
}
my test method is the following:
public class SQLDataExample {
public static void main(String args[]) throws Exception {
DriverManager.registerDriver(new oracle.jdbc.driver.OracleDriver());
OracleConnection conn = (OracleConnection) DriverManager.getConnection(
"jdbc:oracle:thin:#................", "...",
"...");
Dictionary<String, Class<?>> map = (Dictionary) conn.getTypeMap();
map.put("BONI.DK1", Class.forName("com.gwb.db.objects.DK1"));
OracleCallableStatement cs = (OracleCallableStatement) conn.prepareCall("{call BOX.DK(?)}");
DK1 dd = new DK1("BONI.DK1", "1");
cs.setObject(1, dd);
cs.registerOutParameter(1, OracleTypes.STRUCT, "BONI.DK1");
cs.execute();
DK1 df = (DK1) cs.getObject(1);
}
}
Now, the last step
DK1 df = (DK1) cs.getObject(1);
in this procedure fails and although I have tried so many things in the last couple of days, I cannot get it to run! I get a
Exception in thread "main" java.sql.SQLException: Inconsistent Java-
ans SQL-Objecttypes: InstantiationException: com.gwb.db.objects.DK1
If I replace getObject with getSTRUCT I see that the DB procedure works and returns values as expected. I cannnot figure out why a I unable to map a JAVA Object.
I would be very grateful for any help or tipps!
Thank you in advance
I forgot the default constructor
public DK1() { }
I have this Class:
public class Stripe implements WritableComparable<Stripe>{
private List<Term> occorrenze = new ArrayList<Term>();
public Stripe(){}
#Override
public void readFields(DataInput in) throws IOException {
}
}
public class Term implements WritableComparable<Term> {
private Text key;
private IntWritable frequency;
#Override
public void readFields(DataInput in) throws IOException {
this.key.readFields(in);
this.frequency.readFields(in);
}
Stripe is a List of Term (pair of Text and intWritable).
how can I set the method "readField" for read the complex type Stripe from DataInput?
To serialize an list you'll need to write out the length of the list, followed by the elements themselves. A simple readFields / write method pair for Stripe could be:
#Override
public void readFields(DataInput in) throws IOException {
occorrenze.clear();
int cnt = in.readInt();
for (int x = 0; x < cnt; x++) {
Term term = new Term();
term.readFields(in);
occorrence.add(term);
}
}
#Override
public void write(DataOutput out) throws IOException {
out.writeInt(occorrenze.size());
for (Term term : occorrenze) {
term.write(out);
}
}
You could make this more efficient by using a VInt rather than an int, and by using a pool of Terms which can be re-used to save on object creation / garbage collection in the readFields method
You could use ArrayWritable which is a list of writables of the same type.
I am new to Hadoop and Java, and I feel there is something obvious I am just missing. I am using Hadoop 1.0.3 if that means anything.
My goal for using hadoop is to take a bunch of files and parse them one file at a time (as opposed to line by line). Each file will produce multiple key-values, but context to the other lines is important. The key and value are multi-value/composite, so I have implemented WritableCompare for the key and Writable for the value. Because the processing of each file take a bit of CPU, I want to save the output of the mapper, then run multiple reducers later on.
For the composite keys, I followed [http://stackoverflow.com/questions/12427090/hadoop-composite-key][1]
The problem is, the output is just Java object references as opposed to the composite key and value. Example:
LinkKeyWritable#bd2f9730 LinkValueWritable#8752408c
I am not sure if the problem is related to not reducing the data at all or
Here is my main class:
public static void main(String[] args) throws Exception {
JobConf conf = new JobConf(Parser.class);
conf.setJobName("raw_parser");
conf.setOutputKeyClass(LinkKeyWritable.class);
conf.setOutputValueClass(LinkValueWritable.class);
conf.setMapperClass(RawMap.class);
conf.setNumMapTasks(0);
conf.setInputFormat(PerFileInputFormat.class);
conf.setOutputFormat(TextOutputFormat.class);
PerFileInputFormat.setInputPaths(conf, new Path(args[0]));
FileOutputFormat.setOutputPath(conf, new Path(args[1]));
JobClient.runJob(conf);
}
And my Mapper class:
public class RawMap extends MapReduceBase implements
Mapper {
public void map(NullWritable key, Text value,
OutputCollector<LinkKeyWritable, LinkValueWritable> output,
Reporter reporter) throws IOException {
String json = value.toString();
SerpyReader reader = new SerpyReader(json);
GoogleParser parser = new GoogleParser(reader);
for (String page : reader.getPages()) {
String content = reader.readPageContent(page);
parser.addPage(content);
}
for (Link link : parser.getLinks()) {
LinkKeyWritable linkKey = new LinkKeyWritable(link);
LinkValueWritable linkValue = new LinkValueWritable(link);
output.collect(linkKey, linkValue);
}
}
}
Link is basically a struct of various information that get's split between LinkKeyWritable and LinkValueWritable
LinkKeyWritable:
public class LinkKeyWritable implements WritableComparable<LinkKeyWritable>{
protected Link link;
public LinkKeyWritable() {
super();
link = new Link();
}
public LinkKeyWritable(Link link) {
super();
this.link = link;
}
#Override
public void readFields(DataInput in) throws IOException {
link.batchDay = in.readLong();
link.source = in.readUTF();
link.domain = in.readUTF();
link.path = in.readUTF();
}
#Override
public void write(DataOutput out) throws IOException {
out.writeLong(link.batchDay);
out.writeUTF(link.source);
out.writeUTF(link.domain);
out.writeUTF(link.path);
}
#Override
public int compareTo(LinkKeyWritable o) {
return ComparisonChain.start().
compare(link.batchDay, o.link.batchDay).
compare(link.domain, o.link.domain).
compare(link.path, o.link.path).
result();
}
#Override
public int hashCode() {
return Objects.hashCode(link.batchDay, link.source, link.domain, link.path);
}
#Override
public boolean equals(final Object obj){
if(obj instanceof LinkKeyWritable) {
final LinkKeyWritable o = (LinkKeyWritable)obj;
return Objects.equal(link.batchDay, o.link.batchDay)
&& Objects.equal(link.source, o.link.source)
&& Objects.equal(link.domain, o.link.domain)
&& Objects.equal(link.path, o.link.path);
}
return false;
}
}
LinkValueWritable:
public class LinkValueWritable implements Writable{
protected Link link;
public LinkValueWritable() {
link = new Link();
}
public LinkValueWritable(Link link) {
this.link = new Link();
this.link.type = link.type;
this.link.description = link.description;
}
#Override
public void readFields(DataInput in) throws IOException {
link.type = in.readUTF();
link.description = in.readUTF();
}
#Override
public void write(DataOutput out) throws IOException {
out.writeUTF(link.type);
out.writeUTF(link.description);
}
#Override
public int hashCode() {
return Objects.hashCode(link.type, link.description);
}
#Override
public boolean equals(final Object obj){
if(obj instanceof LinkKeyWritable) {
final LinkKeyWritable o = (LinkKeyWritable)obj;
return Objects.equal(link.type, o.link.type)
&& Objects.equal(link.description, o.link.description);
}
return false;
}
}
I think the answer is in the implementation of the TextOutputFormat. Specifically, the LineRecordWriter's writeObject method:
/**
* Write the object to the byte stream, handling Text as a special
* case.
* #param o the object to print
* #throws IOException if the write throws, we pass it on
*/
private void writeObject(Object o) throws IOException {
if (o instanceof Text) {
Text to = (Text) o;
out.write(to.getBytes(), 0, to.getLength());
} else {
out.write(o.toString().getBytes(utf8));
}
}
As you can see, if your key or value is not a Text object, it calls the toString method on it and writes that out. Since you've left toString unimplemented in your key and value, it's using the Object class's implementation, which is writing out the reference.
I'd say that you should try writing an appropriate toString function or using a different OutputFormat.
It looks like you have a list of objects just like you wanted. You need to implement toString() on your writable if you want a human-readable version printed out instead of an ugly java reference.