Hive to Hbase comparision of data in tables - hadoop

I am into DW testing and need to compare data from source to target.Source data is stored in hive/RDBMS while the target data is loaded in Hbase. I am new to Hbase .Can any one help me with the approach that i can take. What I am looking for is a similar function as that of "MINUS" . Is it possible ?

You should write java file in that you can combine:
HBase:
import java.io.IOException;
// HBASE
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
public class RetriveData{
public static void main(String[] args) throws IOException, Exception{
// Instantiating Configuration class
Configuration config = HBaseConfiguration.create();
// Instantiating HTable class
HTable table = new HTable(config, "emp");
// Instantiating Get class
Get g = new Get(Bytes.toBytes("row1"));
// Reading the data
Result result = table.get(g);
// Reading values from Result class object
byte [] value = result.getValue(Bytes.toBytes("personal"),Bytes.toBytes("name"));
byte [] value1 = result.getValue(Bytes.toBytes("personal"),Bytes.toBytes("city"));
// Printing the values
String name = Bytes.toString(value);
String city = Bytes.toString(value1);
**// CALL THE HIVE CLASS(HiveQLOrderBy)...YOU CAN COMPARE**
System.out.println("name: " + name + " city: " + city);
}
}
//HIVE
import java.sql.SQLException;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import java.sql.DriverManager;
public class HiveQLOrderBy {
private static String driverName = "org.apache.hadoop.hive.jdbc.HiveDriver";
public static void main(String[] args) throws SQLException {
// Register driver and create driver instance
Class.forName(driverName);
// get connection
Connection con = DriverManager.getConnection("jdbc:hive://localhost:10000/userdb", "", "");
// create statement
Statement stmt = con.createStatement();
// execute statement
Resultset res = stmt.executeQuery("SELECT * FROM employee ORDER BY DEPT;");
System.out.println(" ID \t Name \t Salary \t Designation \t Dept ");
while (res.next()) {
System.out.println(res.getInt(1) + " " + res.getString(2) + " " + res.getDouble(3) + " " + res.getString(4) + " " + res.getString(5));
}
con.close();
}
}

Related

while inserting values in a table through JDBC java.lang.ArrayIndexOutOfBoundsException:

getting this error for the below code
java.lang.ArrayIndexOutOfBoundsException: Index 0 out of bounds for length 0
at oracle.jdbc.driver.OracleSql.main(OracleSql.java:1717)
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
public class Z1 {
public static void main(String[] args) throws Exception
{
// String query = "select * from Employee";
//String sql = "CREATE TABLE people(\r\n"+ "id INT NOT NULL PRIMARY KEY,\r\n"+ "name VARCHAR2(50) \r\n"+ ");";
Class.forName("oracle.jdbc.driver.OracleDriver");
//Class.forName("com.mysql.jdbc.Driver");
Connection con = DriverManager.getConnection("jdbc:oracle:thin:#localhost:1521:","system","palak");
Statement stmt = con.createStatement();
String sql = "insert into test1 values(100)";
stmt.execute(sql);
System.out.println("dOne");
}
}

Find count of rows with empty value in Hbase

I have populated a Hbase table with rowid and vrious information pertaining to tweet such as clean-text,url,hashtag etc. as follows
902221655086211073 column=clean-tweet:clean-text-cta, timestamp=1514793745304, value=democrat mayor order hurricane harvey stand houston
However while populating I noticed that the some of the rows are empty like
902487280543305728 column=clean-tweet:clean-text-cta, timestamp=1514622371008, value=
Now how do I find the count of rows that are having data?
Please help me in this
There is no provision to do this in HBase shell as of now. May be you can use a simple code like this to get a number of records with no value for the provided column qualifier.
CountAndFilter [tableName] [columnFamily] [columnQualifier]
import java.io.IOException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
public class CountAndFilter {
private static Connection conn;
private static int recordsWithoutValue = 0;
public static Admin getConnection() throws IOException {
if (conn == null) {
conn = ConnectionFactory.createConnection(HBaseConfiguration.create());
}
return conn.getAdmin();
}
public static void main(String args[]) throws IOException {
getConnection();
scan(args[0], args[1], args[2]);
System.out.println("Records with empty value : " + recordsWithoutValue);
}
public static void scan(String tableName, String columnFamily, String columnQualifier) throws IOException {
Table table = conn.getTable(TableName.valueOf(tableName));
ResultScanner rs = table.getScanner(new Scan().addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(columnQualifier)));
Result res = null;
try {
while ((res = rs.next()) != null) {
if (res.containsEmptyColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(columnQualifier))){
recordsWithoutValue++;
}
}
} finally {
rs.close();
}
}
}

Error Handling Spring JdbcTemplate batchUpdate

I am trying to update thousands of rows in a table using batchUpdate. My requirements are:
1) Assume there are 1000 records in a batch. Record No 235 caused an error. How do I find out which record caused the error.
2) Assume that record 600 did not result in an update (reason could be no record matching the where clause). How can I find out records that did not result in an update.
3) In both scenarios above how can I continue processing the remaining records.
The only solution after long search and debug is to go to BatchUpdateException class and find the negative element and deduce the value of the insertion that is in error from the MAP.
import java.sql.BatchUpdateException;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.Map;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
#Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
#Repository("dao_")
public class YouDao extends CommunDao implements IyouDao {
public void bulkInsert(final List<Map<String, String>> map)
throws BusinessException {
try {
String sql = " insert into your_table " + "( aa,bb )"
+ "values " + "( ?,? )";
BatchPreparedStatementSetter batchPreparedStatementSetter = new BatchPreparedStatementSetter() {
#Override
public void setValues(PreparedStatement ps, int i)
throws SQLException {
Map<String, String> bean = map.get(i);
ps.setString(1, bean.get("aa"));
ps.setString(2, bean.get("bb"));
//..
//..
}
#Override
public int getBatchSize() {
return map.size();
}
};
getJdbcTemplate().batchUpdate(sql, batchPreparedStatementSetter);
}
catch (Exception e) {
if (e.getCause() instanceof BatchUpdateException) {
BatchUpdateException be = (BatchUpdateException) e.getCause();
int[] batchRes = be.getUpdateCounts();
if (batchRes != null && batchRes.length > 0) {
for (int index = 0; index < batchRes.length; index++) {
if (batchRes[index] == Statement.EXECUTE_FAILED) {
logger.error("Error execution >>>>>>>>>>>"
+ index + " --- , codeFail : " + batchRes[index]
+ "---, line " + map.get(index));
}
}
}
}
throw new BusinessException(e);
}
}
}
int[] rows =jdbcTemplate.batchUpdate(TbCareQueryConstant.SQL_UPDATE_BANKDETAILS_OF_USER, new BatchPreparedStatementSetter(){
.....
your code
}
for(int i=0 ; i < rows.length ; i++){
if(rows[i] == 0){
}
}

SQLException Handling

I am trying to run a simple Java program which fetches data from the oracle database and display it. I connected the oracle database. Here is my code:
DataHandler Class:
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.ResultSet;
import oracle.jdbc.pool.OracleDataSource;
public class DataHandler {
public DataHandler() {
super();
}
String jdbcUrl = "jdbc:oracle:thin:#localhost:1521:ORCL";
//I already added the above line but still getting error.
String userid = "scott";
String password = "tiger";
Connection conn;
public void getDBConnection() throws SQLException{
OracleDataSource ds;
ds = new OracleDataSource();
ds.setUser(jdbcUrl);
conn = ds.getConnection(userid,password);
}
Statement stmt;
ResultSet rset;
String query;
String sqlString;
public ResultSet getAllEmployees() throws SQLException{
getDBConnection();
stmt = conn.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_READ_ONLY);
query = "SELECT * FROM emp ORDER BY empno";
System.out.println("\nExecuting query: " + query);
rset = stmt.executeQuery(query);
return rset;
}
}
and the JavaClient Class as
JavaCLient CLass:
import java.sql.ResultSet;
public class JavaClient {
public JavaClient() {
super();
}
public static void main(String[] args) throws Exception{
DataHandler datahandler = new DataHandler();
ResultSet rset = datahandler.getAllEmployees();
while (rset.next()) {
System.out.println(rset.getInt(1) + " " +
rset.getString(2) + " " +
rset.getString(3) + " " +
rset.getString(4));
}
}
}
I get no compilation error but while running it I get following exception error
Error:
Exception in thread "main" java.sql.SQLException: Invalid Oracle URL specified: OracleDataSource.makeURL
at oracle.jdbc.pool.OracleDataSource.makeURL(OracleDataSource.java:1277)
at oracle.jdbc.pool.OracleDataSource.getConnection(OracleDataSource.java:185)
at student_attendence_iem.DataHandler.getDBConnection(DataHandler.java:22)
at student_attendence_iem.DataHandler.getAllEmployees(DataHandler.java:31)
at student_attendence_iem.JavaClient.main(JavaClient.java:9)
Process exited with exit code 1.
Please help me. Thanks in advance. :)
You have not set URL of your database.
Add setURL(url) method which takes URL of database as parameter. Below is the code.
OracleDataSource ds;
ds = new OracleDataSource();
ds.setURL(jdbcUrl);
Also, with ds.setUser(jdbcUrl); you are trying to setUser with the URL of database which is wrong.
You don't have to setUser like this as you are already doing that in the following line of code conn = ds.getConnection(userid,password);

NDEF format a Mifare classik card with NFC Tools java

Does anyone know how to directly format a Mifare classic 1k card as NDEF using java NFC tools library? I am using an ACR 122U reader/writer
I use this code to format a Mifare Classik card to NDEF format
package name.leiqin.test.nfc;
import java.io.IOException;
import org.nfctools.NfcAdapter;
import org.nfctools.api.TagInfo;
import org.nfctools.mf.classic.MfClassicNfcTagListener;
import org.nfctools.mf.ul.Type2NfcTagListener;
import org.nfctools.ndef.NdefOperations;
import org.nfctools.ndef.NdefOperationsListener;
import org.nfctools.scio.Terminal;
import org.nfctools.scio.TerminalHandler;
import org.nfctools.scio.TerminalMode;
import org.nfctools.spi.acs.AcsTerminal;
import org.nfctools.spi.scm.SclTerminal;
import org.nfctools.utils.LoggingUnknownTagListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FormatCard implements NdefOperationsListener {
private static final Logger logger = LoggerFactory
.getLogger(FormatCard.class);
public static void main(String[] args) throws IOException {
TerminalHandler terminalHandler = new TerminalHandler();
terminalHandler.addTerminal(new AcsTerminal());
terminalHandler.addTerminal(new SclTerminal());
Terminal terminal = terminalHandler.getAvailableTerminal(null);
logger.info("terminal : {}", terminal);
NfcAdapter adapter = new NfcAdapter(terminal, TerminalMode.INITIATOR);
FormatCard formatListener = new FormatCard();
adapter.registerTagListener(new Type2NfcTagListener(formatListener));
adapter.registerTagListener(new MfClassicNfcTagListener(formatListener));
adapter.registerUnknownTagListerner(new LoggingUnknownTagListener());
logger.info("nfc adapter : {}", adapter);
adapter.startListening();
System.out.println("Waiting for tags, press ENTER to exit");
System.in.read();
}
#Override
public void onNdefOperations(NdefOperations ndefOperations) {
TagInfo tagInfo = ndefOperations.getTagInfo();
logger.info("tag id : {}", toHex(tagInfo.getId(), ""));
logger.info("tag type : {}", tagInfo.getTagType());
if (ndefOperations.isWritable()) {
if (ndefOperations.isFormatted()) {
logger.info("tag is already formatred");
} else {
logger.info("format start");
ndefOperations.format();
logger.info("format end");
}
} else {
logger.info("Tag not writable");
}
}
public static String toHex(byte[] bs, String separator) {
if (separator == null)
separator = " ";
StringBuffer sb = new StringBuffer();
for (byte b : bs) {
int i = b & 0xFF;
String hex = Integer.toHexString(i).toUpperCase();
if (hex.length() == 1)
hex = "0" + hex;
sb.append(hex);
sb.append(separator);
}
sb.setLength(sb.length() - separator.length());
return sb.toString();
}
}

Resources