Set Request Not Working SNMP4j - snmp

I have searched everywhere but I haven't found the solution to my problem. I've tried all solutions involving changing the community name, certain different set classes and I still can't get the set to work for SNMP4j. The code executes fine and can be accessed from iReasoning MIB browser but will not change the value of the OID after the set executes. The SNMP classes and tester classes are below.
SNMPManager code:
public class SNMPManager {
Snmp snmp = null;
String address = null;
/**
* Constructor
*
* #param add
*/
public SNMPManager(String add) {
address = add;
}
public static void main(String[] args) throws IOException {
/**
* Port 161 is used for Read and Other operations Port 162 is used for the trap
* generation
*/
SNMPManager client = new SNMPManager("udp:127.0.0.1/161");
client.start();
/**
* OID - .1.3.6.1.2.1.1.1.0 => SysDec OID - .1.3.6.1.2.1.1.5.0 => SysName => MIB
* explorer will be usefull here, as discussed in previous article
*/
String sysDescr = client.getAsString(new OID(".1.3.6.1.2.1.1.1.0"));
System.out.println(sysDescr);
}
/**
* Start the Snmp session. If you forget the listen() method you will not get
* any answers because the communication is asynchronous and the listen() method
* listens for answers.
*
* #throws IOException
*/
public void start() throws IOException {
TransportMapping<?> transport = new DefaultUdpTransportMapping();
snmp = new Snmp(transport);
// Do not forget this line!
transport.listen();
}
/**
* Method which takes a single OID and returns the response from the agent as a
* String.
*
* #param oid
* #return
* #throws IOException
*/
public String getAsString(OID oid) throws IOException {
ResponseEvent event = get(new OID[] { oid });
return event.getResponse().get(0).getVariable().toString();
}
/**
* This method is capable of handling multiple OIDs
*
* #param oids
* #return
* #throws IOException
*/
public ResponseEvent get(OID oids[]) throws IOException {
PDU pdu = new PDU();
for (OID oid : oids) {
pdu.add(new VariableBinding(oid));
}
pdu.setType(PDU.GET);
ResponseEvent event = snmp.send(pdu, getTarget(), null);
if (event != null) {
return event;
}
throw new RuntimeException("GET timed out");
}
public ResponseEvent set(OID oid, String val) throws IOException {
PDU pdu = new PDU();
VariableBinding varBind = new VariableBinding(oid, new OctetString(val));
pdu.add(varBind);
pdu.setType(PDU.SET);
//pdu.setRequestID(new Integer32(1));
Target target = getTarget();
ResponseEvent event = snmp.set(pdu, target);
if (event != null) {
System.out.println("\nResponse:\nGot Snmp Set Response from Agent");
System.out.println("Snmp Set Request = " + event.getRequest().getVariableBindings());
PDU responsePDU = event.getResponse();
System.out.println("\nresponsePDU = " + responsePDU);
if (responsePDU != null) {
int errorStatus = responsePDU.getErrorStatus();
int errorIndex = responsePDU.getErrorIndex();
String errorStatusText = responsePDU.getErrorStatusText();
System.out.println("\nresponsePDU = " + responsePDU);
if (errorStatus == PDU.noError) {
System.out.println("Snmp Set Response = " + responsePDU.getVariableBindings());
} else {
System.out.println("errorStatus = " + responsePDU);
System.out.println("Error: Request Failed");
System.out.println("Error Status = " + errorStatus);
System.out.println("Error Index = " + errorIndex);
System.out.println("Error Status Text = " + errorStatusText);
}
}
return event;
}
throw new RuntimeException("SET timed out");
}
/**
* This method returns a Target, which contains information about where the data
* should be fetched and how.
*
* #return
*/
private Target getTarget() {
Address targetAddress = GenericAddress.parse(address);
CommunityTarget target = new CommunityTarget();
target.setCommunity(new OctetString("public"));
target.setAddress(targetAddress);
target.setRetries(2);
target.setTimeout(1500);
target.setVersion(SnmpConstants.version2c);
return target;
}
}
SNMPAgent code:
public class SNMPAgent extends BaseAgent {
private String address;
/**
*
* #param address
* #throws IOException
*/
public SNMPAgent(String address) throws IOException {
/**
* Creates a base agent with boot-counter, config file, and a CommandProcessor
* for processing SNMP requests. Parameters: "bootCounterFile" - a file with
* serialized boot-counter information (read/write). If the file does not exist
* it is created on shutdown of the agent. "configFile" - a file with serialized
* configuration information (read/write). If the file does not exist it is
* created on shutdown of the agent. "commandProcessor" - the CommandProcessor
* instance that handles the SNMP requests.
*/
super(new File("conf.agent"), new File("bootCounter.agent"),
new CommandProcessor(new OctetString(MPv3.createLocalEngineID())));
this.address = address;
}
/**
* Adds community to security name mappings needed for SNMPv1 and SNMPv2c.
*/
#Override
protected void addCommunities(SnmpCommunityMIB communityMIB) {
Variable[] com2sec = new Variable[] { new OctetString("public"), new OctetString("cpublic"), // security name
getAgent().getContextEngineID(), // local engine ID
new OctetString("public"), // default context name
new OctetString(), // transport tag
new Integer32(StorageType.nonVolatile), // storage type
new Integer32(RowStatus.active) // row status
};
MOTableRow<?> row = communityMIB.getSnmpCommunityEntry()
.createRow(new OctetString("public2public").toSubIndex(true), com2sec);
communityMIB.getSnmpCommunityEntry().addRow((SnmpCommunityEntryRow) row);
}
/**
* Adds initial notification targets and filters.
*/
#Override
protected void addNotificationTargets(SnmpTargetMIB arg0, SnmpNotificationMIB arg1) {
// TODO Auto-generated method stub
}
/**
* Adds all the necessary initial users to the USM.
*/
#Override
protected void addUsmUser(USM arg0) {
// TODO Auto-generated method stub
}
/**
* Adds initial VACM configuration.
*/
#Override
protected void addViews(VacmMIB vacm) {
vacm.addGroup(SecurityModel.SECURITY_MODEL_SNMPv2c, new OctetString("cpublic"), new OctetString("v1v2group"),
StorageType.nonVolatile);
vacm.addAccess(new OctetString("v1v2group"), new OctetString("public"), SecurityModel.SECURITY_MODEL_ANY,
SecurityLevel.NOAUTH_NOPRIV, MutableVACM.VACM_MATCH_EXACT, new OctetString("fullReadView"),
new OctetString("fullWriteView"), new OctetString("fullNotifyView"), StorageType.nonVolatile);
vacm.addViewTreeFamily(new OctetString("fullReadView"), new OID("1.3"), new OctetString(),
VacmMIB.vacmViewIncluded, StorageType.nonVolatile);
}
/**
* Unregister the basic MIB modules from the agent's MOServer.
*/
#Override
protected void unregisterManagedObjects() {
// TODO Auto-generated method stub
}
/**
* Register additional managed objects at the agent's server.
*/
#Override
protected void registerManagedObjects() {
// TODO Auto-generated method stub
}
#SuppressWarnings("unchecked")
protected void initTransportMappings() throws IOException {
transportMappings = new TransportMapping[1];
Address addr = GenericAddress.parse(address);
TransportMapping<?> tm = TransportMappings.getInstance().createTransportMapping(addr);
transportMappings[0] = tm;
}
/**
* Start method invokes some initialization methods needed to start the agent
*
* #throws IOException
*/
public void start() throws IOException {
init();
// This method reads some old config from a file and causes
// unexpected behavior.
// loadConfig(ImportModes.REPLACE_CREATE);
addShutdownHook();
getServer().addContext(new OctetString("public"));
finishInit();
run();
sendColdStartNotification();
}
/**
* Clients can register the MO they need
*/
public void registerManagedObject(ManagedObject mo) {
try {
server.register(mo, null);
} catch (DuplicateRegistrationException ex) {
throw new RuntimeException(ex);
}
}
public void unregisterManagedObject(MOGroup moGroup) {
moGroup.unregisterMOs(server, getContext(moGroup));
}
}
MOCreator code:
public class MOCreator {
public static MOScalar<Variable> createReadOnly(OID oid,Object value ){
return new MOScalar<Variable>(oid,
MOAccessImpl.ACCESS_READ_WRITE,
getVariable(value));
}
private static Variable getVariable(Object value) {
if(value instanceof String) {
return new OctetString((String)value);
}
throw new IllegalArgumentException("Unmanaged Type: " + value.getClass());
}
}
Tester class:
public class TestSNMPAgent {
String siteOIDNumber = "1";
String dataOIDNumber = "1";
String sensorOIDNumber = "1";
OID newOIDdata = new OID("1.3.6.1.4.1.1234.5." + siteOIDNumber + ".2." + dataOIDNumber + "." + sensorOIDNumber + ".0");
OID newOIDdata2 = new OID("1.3.6.1.4.1.1234.5.1.2.1.2.0");
OID newOIDdata3 = new OID("1.3.6.1.4.1.1234.5.1.2.1.3.0");
public static void main(String[] args) throws IOException {
TestSNMPAgent client = new TestSNMPAgent("udp:127.0.0.1/161");
client.init();
}
SNMPAgent agent = null;
/**
* This is the client which we have created earlier
*/
SNMPManager client = null;
String address = null;
/**
* Constructor
*
* #param add
*/
public TestSNMPAgent(String add) {
address = add;
}
/**
* Initiates the testing of the SNMP Agent.
* #throws IOException
*/
private void init() throws IOException {
/*agent = new SNMPAgent("172.21.1.103/2001");*/
agent = new SNMPAgent("172.21.1.103/2010");
agent.start();
// Since BaseAgent registers some MIBs by default we need to unregister
// one before we register our own sysDescr. Normally you would
// override that method and register the MIBs that you need
agent.unregisterManagedObject(agent.getSnmpv2MIB());
//agent.registerManagedObject(MOCreator.createReadOnly(sysDescr,"This Description is set By KGrewe"));
agent.registerManagedObject(MOCreator.createReadOnly(newOIDdata, "50"));
agent.registerManagedObject(MOCreator.createReadOnly(newOIDdata2, "NaN"));
agent.registerManagedObject(MOCreator.createReadOnly(newOIDdata3, "3"));
SNMPManager client1 = new SNMPManager("udp:127.21.1.103/2010");
client1.start();
client1.set(newOIDdata, "30");
//System.out.println(newOIDdata);
// Get back Value which is set
//System.out.println(client1.getAsString(newOIDdata));
while(true) {
}
}
}
Thanks for the help in advance.

Related

How to convert file to a string using SpringIntegration

I am trying to read a text file and convert it to a string using SpringIntegration.
Need help in transforming file to a string.
Git Link: https://github.com/ravikalla/spring-integration
Source Code -
#Bean
#InboundChannelAdapter(value = "payorFileSource", poller = #Poller(fixedDelay = "10000"))
public MessageSource<File> fileReadingMessageSource() {
FileReadingMessageSource sourceReader = new FileReadingMessageSource();
sourceReader.setDirectory(new File(INPUT_DIR));
sourceReader.setFilter(new SimplePatternFileListFilter(FILE_PATTERN));
return sourceReader;
}
#Bean
#Transformer(inputChannel="payorFileSource", outputChannel="payorFileContent")
public FileToStringTransformer transformFileToString() {
FileToStringTransformer objFileToStringTransformer = new FileToStringTransformer();
return objFileToStringTransformer;
}
Error -
SEVERE: org.springframework.integration.handler.ReplyRequiredException: No reply produced by handler 'fileCopyConfig.transformPayorStringToObject.transformer.handler', and its 'requiresReply' property is set to true., failedMessage=GenericMessage [payload=1|test1, headers={sequenceNumber=1, file_name=payor.txt, sequenceSize=4, correlationId=ff1fef7d-7011-ee99-8d71-96146ac9ea07, file_originalFile=source/payor.txt, id=fd4f950b-afcf-70e6-a053-7d59ff593add, file_relativePath=payor.txt, timestamp=1554875904858}]
at org.springframework.integration.handler.AbstractReplyProducingMessageHandler.handleMessageInternal(AbstractReplyProducingMessageHandler.java:119)
You can convert the file into an InputStream and the use
IOUtils.toString(inputStream) to convert it into a String.
That error is coming from somewhere else; the FTST can't return null.
I haven't looked at all your code, but this looks suspicious:
#Bean
#Transformer(inputChannel="payorRawStringChannel", outputChannel="payorRawObjectChannel")
public GenericTransformer<String, Payor> transformPayorStringToObject() {
return new GenericTransformer<String, Payor>() {
#Override
public Payor transform(String strPayor) {
String[] arrPayorData = strPayor.split(",");
Payor objPayor = null;
if (null != arrPayorData && arrPayorData.length > 1)
objPayor = new Payor(Integer.parseInt(arrPayorData[0]), arrPayorData[1]);
return objPayor;
}
};
}
It can return null; transformers are not allowed to do that.
Turn on DEBUG logging and follow the message flow to see which component is at fault.
package org.springframework.integration.samples.tcpclientserver;
import java.io.UnsupportedEncodingException;
import org.springframework.core.convert.converter.Converter;
/**
* Simple byte array to String converter; allowing the character set
* to be specified.
*
* #author Gary Russell
* #since 2.1
*
*/
public class ByteArrayToStringConverter implements Converter<byte[], String> {
private String charSet = "UTF-8";
public String convert(byte[] bytes) {
try {
return new String(bytes, this.charSet);
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
return new String(bytes);
}
}
/**
* #return the charSet
*/
public String getCharSet() {
return charSet;
}
/**
* #param charSet the charSet to set
*/
public void setCharSet(String charSet) {
this.charSet = charSet;
}
}

GCM XMPP - Error on start web application server

My back end java has the goal to receive mobile messages using GCM XMPP. My web applications have spring 4.1.4 and smack 4.1.4.
Smack dependencies:
<dependency>
<groupId>org.igniterealtime.smack</groupId>
<artifactId>smack-core</artifactId>
<version>4.1.4</version>
</dependency>
<dependency>
<groupId>org.igniterealtime.smack</groupId>
<artifactId>smack-tcp</artifactId>
<version>4.1.4</version>
</dependency>
<dependency>
<groupId>org.igniterealtime.smack</groupId>
<artifactId>smack-extensions</artifactId>
<version>4.1.4</version>
</dependency>
<dependency>
<groupId>org.igniterealtime.smack</groupId>
<artifactId>smack-java7</artifactId>
<version>4.0.1</version>
</dependency>
Bean for XMPP Connection:
#Component("CcsClientImpl")
public class CcsClientImpl {
private static final String GCM_ELEMENT_NAME = "gcm";
private static final String GCM_NAMESPACE = "google:mobile:data";
private static XMPPTCPConnection connection;
/**
* Indicates whether the connection is in draining state, which means that it
* will not accept any new downstream messages.
*/
protected static volatile boolean connectionDraining = false;
private static final Logger logger = LoggerFactory.getLogger("CcsClientImpl");
#Value("${sender.id}")
private String mSenderId;
#Value("${server.api.key}")
private String mServerApiKey;
#Value("${gcm.xmpp.host}")
private String mHost;
#Value("${gcm.xmpp.port}")
private int mPort;
#Value("${gcm.xmpp.debuggable}")
private boolean mDebuggable;
#Autowired
private ProcessorFactory processorFactory;
//#Autowired
public CcsClientImpl() {
ProviderManager.addExtensionProvider(GCM_ELEMENT_NAME, GCM_NAMESPACE, new ExtensionElementProvider<ExtensionElement>() {
#Override
public DefaultExtensionElement parse(XmlPullParser parser,int initialDepth) throws org.xmlpull.v1.XmlPullParserException, IOException {
String json = parser.nextText();
return new GcmPacketExtension(json);
}
});
try {
connect(mSenderId, mServerApiKey);
} catch (XMPPException ex) {
logger.error("ERRO AO CONECTAR COM GCM XMPP", ex);
} catch (SmackException ex) {
logger.error("ERRO AO CONECTAR COM GCM XMPP", ex);
} catch (IOException ex) {
logger.error("ERRO AO CONECTAR COM GCM XMPP", ex);
}
}
/**
* Sends a downstream message to GCM.
*
* #return true if the message has been successfully sent.
*/
public boolean sendDownstreamMessage(String jsonRequest) throws
NotConnectedException {
if (!connectionDraining) {
send(jsonRequest);
return true;
}
logger.info("Dropping downstream message since the connection is draining");
return false;
}
/**
* Returns a random message id to uniquely identify a message.
*
* <p>Note: This is generated by a pseudo random number generator for
* illustration purpose, and is not guaranteed to be unique.
*/
public String nextMessageId() {
return "m-" + UUID.randomUUID().toString();
}
/**
* Sends a packet with contents provided.
*/
protected void send(String jsonRequest) throws NotConnectedException {
Stanza request = new GcmPacketExtension(jsonRequest).toPacket();
connection.sendStanza(request);
}
/// new: customized version of the standard handleIncomingDateMessage method
/**
* Handles an upstream data message from a device application.
*/
public void handleIncomingDataMessage(CcsMessage msg) {
if (msg.getPayload().get("action") != null) {
PayloadProcessor processor = processorFactory.getProcessor(msg.getPayload().get("action"));
processor.handleMessage(msg);
}
}
/**
* Handles an ACK.
*
* <p>Logs a INFO message, but subclasses could override it to
* properly handle ACKs.
*/
protected void handleAckReceipt(Map<String, Object> jsonObject) {
String messageId = (String) jsonObject.get("message_id");
String from = (String) jsonObject.get("from");
logger.info("handleAckReceipt() from: " + from + ",messageId: " + messageId);
}
/**
* Handles a NACK.
*
* <p>Logs a INFO message, but subclasses could override it to
* properly handle NACKs.
*/
protected void handleNackReceipt(Map<String, Object> jsonObject) {
String messageId = (String) jsonObject.get("message_id");
String from = (String) jsonObject.get("from");
logger.info("handleNackReceipt() from: " + from + ",messageId: " + messageId);
}
protected void handleControlMessage(Map<String, Object> jsonObject) {
logger.info("handleControlMessage(): " + jsonObject);
String controlType = (String) jsonObject.get("control_type");
if ("CONNECTION_DRAINING".equals(controlType)) {
connectionDraining = true;
} else {
logger.info("Unrecognized control type: %s. This could happen if new features are " + "added to the CCS protocol.",
controlType);
}
}
/**
* Creates a JSON encoded GCM message.
*
* #param to RegistrationId of the target device (Required).
* #param messageId Unique messageId for which CCS sends an
* "ack/nack" (Required).
* #param payload Message content intended for the application. (Optional).
* #param collapseKey GCM collapse_key parameter (Optional).
* #param timeToLive GCM time_to_live parameter (Optional).
* #param delayWhileIdle GCM delay_while_idle parameter (Optional).
* #return JSON encoded GCM message.
*/
public static String createJsonMessage(String to, String messageId,
Map<String, String> payload, String collapseKey, Long timeToLive,
Boolean delayWhileIdle) {
Map<String, Object> message = new HashMap<String, Object>();
message.put("to", to);
if (collapseKey != null) {
message.put("collapse_key", collapseKey);
}
if (timeToLive != null) {
message.put("time_to_live", timeToLive);
}
if (delayWhileIdle != null && delayWhileIdle) {
message.put("delay_while_idle", true);
}
message.put("message_id", messageId);
message.put("data", payload);
return JSONValue.toJSONString(message);
}
/**
* Creates a JSON encoded ACK message for an upstream message received
* from an application.
*
* #param to RegistrationId of the device who sent the upstream message.
* #param messageId messageId of the upstream message to be acknowledged to CCS.
* #return JSON encoded ack.
*/
protected static String createJsonAck(String to, String messageId) {
Map<String, Object> message = new HashMap<String, Object>();
message.put("message_type", "ack");
message.put("to", to);
message.put("message_id", messageId);
return JSONValue.toJSONString(message);
}
/**
* Connects to GCM Cloud Connection Server using the supplied credentials.
*
* #param senderId Your GCM project number
* #param apiKey API Key of your project
*/
public void connect(String senderId, String serverApiKey)
throws XMPPException, IOException, SmackException {
XMPPTCPConnectionConfiguration config =
XMPPTCPConnectionConfiguration.builder()
.setServiceName(mHost)
.setHost(mHost)
.setCompressionEnabled(false)
.setPort(mPort)
.setConnectTimeout(30000)
.setSecurityMode(SecurityMode.disabled)
.setSendPresence(false)
.setDebuggerEnabled(mDebuggable)
.setSocketFactory(SSLSocketFactory.getDefault())
.build();
connection = new XMPPTCPConnection(config);
//disable Roster as I don't think this is supported by GCM
Roster roster = Roster.getInstanceFor(connection);
roster.setRosterLoadedAtLogin(false);
logger.info("Connecting...");
connection.connect();
connection.addConnectionListener(new LoggingConnectionListener());
// Handle incoming packets
connection.addAsyncStanzaListener(new MyStanzaListener() , new MyStanzaFilter() );
// Log all outgoing packets
connection.addPacketInterceptor(new MyStanzaInterceptor(), new MyStanzaFilter() );
connection.login(senderId + "#gcm.googleapis.com" , serverApiKey);
logger.info("Logged in: " + mSenderId);
}
private CcsMessage getMessage(Map<String, Object> jsonObject) {
String from = jsonObject.get("from").toString();
// PackageName of the application that sent this message.
String category = jsonObject.get("category").toString();
// unique id of this message
String messageId = jsonObject.get("message_id").toString();
#SuppressWarnings("unchecked")
Map<String, String> payload = (Map<String, String>) jsonObject.get("data");
CcsMessage msg = new CcsMessage(from, category, messageId, payload);
return msg;
}
private class MyStanzaFilter implements StanzaFilter {
#Override
public boolean accept(Stanza arg0) {
// TODO Auto-generated method stub
if (arg0.getClass() == Stanza.class) {
return true;
} else {
if (arg0.getTo() != null) {
if (arg0.getTo().startsWith(mSenderId)) {
return true;
}
}
}
return false;
}
}
private class MyStanzaListener implements StanzaListener{
#Override
public void processPacket(Stanza packet) {
logger.info("Received: " + packet.toXML());
Message incomingMessage = (Message) packet;
GcmPacketExtension gcmPacket =
(GcmPacketExtension) incomingMessage.
getExtension(GCM_NAMESPACE);
String json = gcmPacket.getJson();
try {
#SuppressWarnings("unchecked")
Map<String, Object> jsonObject =
(Map<String, Object>) JSONValue.
parseWithException(json);
// present for "ack"/"nack", null otherwise
Object messageType = jsonObject.get("message_type");
if (messageType == null) {
// Normal upstream data message
CcsMessage msg = getMessage(jsonObject);
handleIncomingDataMessage(msg);
// Send ACK to CCS
String messageId = (String) jsonObject.get("message_id");
String from = (String) jsonObject.get("from");
String ack = createJsonAck(from, messageId);
send(ack);
} else if ("ack".equals(messageType.toString())) {
// Process Ack
handleAckReceipt(jsonObject);
} else if ("nack".equals(messageType.toString())) {
// Process Nack
handleNackReceipt(jsonObject);
} else if ("control".equals(messageType.toString())) {
// Process control message
handleControlMessage(jsonObject);
} else {
logger.warn("Unrecognized message type (%s)",
messageType.toString());
}
} catch (ParseException e) {
logger.info("Error parsing JSON " + json, e);
} catch (Exception e) {
logger.info("Failed to process packet", e);
}
}
}
private class MyStanzaInterceptor implements StanzaListener
{
#Override
public void processPacket(Stanza packet) {
logger.info("Sent: {0}", packet.toXML());
}
}
// public static void main(String[] args) throws Exception {
//
// SmackCcsClient ccsClient = new SmackCcsClient();
//
// ccsClient.connect(YOUR_PROJECT_ID, YOUR_API_KEY);
//
// // Send a sample hello downstream message to a device.
// String messageId = ccsClient.nextMessageId();
// Map<String, String> payload = new HashMap<String, String>();
// payload.put("Message", "Ahha, it works!");
// payload.put("CCS", "Dummy Message");
// payload.put("EmbeddedMessageId", messageId);
// String collapseKey = "sample";
// Long timeToLive = 10000L;
// String message = createJsonMessage(YOUR_PHONE_REG_ID, messageId, payload,
// collapseKey, timeToLive, true);
//
// ccsClient.sendDownstreamMessage(message);
// logger.info("Message sent.");
//
// //crude loop to keep connection open for receiving messages
// while(true)
// {;}
// }
/**
* XMPP Packet Extension for GCM Cloud Connection Server.
*/
private static final class GcmPacketExtension extends DefaultExtensionElement {
private final String json;
public GcmPacketExtension(String json) {
super(GCM_ELEMENT_NAME, GCM_NAMESPACE);
this.json = json;
}
public String getJson() {
return json;
}
#Override
public String toXML() {
return String.format("<%s xmlns=\"%s\">%s</%s>",
GCM_ELEMENT_NAME, GCM_NAMESPACE,
StringUtils.escapeForXML(json), GCM_ELEMENT_NAME);
}
public Stanza toPacket() {
Message message = new Message();
message.addExtension(this);
return message;
}
}
private static final class LoggingConnectionListener
implements ConnectionListener {
#Override
public void connected(XMPPConnection xmppConnection) {
logger.info("Connected.");
}
#Override
public void reconnectionSuccessful() {
logger.info("Reconnecting..");
}
#Override
public void reconnectionFailed(Exception e) {
logger.info("Reconnection failed.. ", e);
}
#Override
public void reconnectingIn(int seconds) {
logger.info("Reconnecting in %d secs", seconds);
}
#Override
public void connectionClosedOnError(Exception e) {
logger.info("Connection closed on error.");
}
#Override
public void connectionClosed() {
logger.info("Connection closed.");
}
#Override
public void authenticated(XMPPConnection arg0, boolean arg1) {
// TODO Auto-generated method stub
}
}
#PreDestroy
public void cleanUp() throws Exception {
logger.info("Bean do cliente XMPP está sendo destruído...");
if (connection.isConnected()) {
logger.info("Conexão GCM XMPP está aberta. Desconectando...");
connection.disconnect();
}
}
}
When starts tomcat happens the error:
Caused by: java.lang.NoClassDefFoundError: org/jivesoftware/smack/initializer/SmackAndOsgiInitializer
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:760)
at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
at org.apache.catalina.loader.WebappClassLoaderBase.findClassInternal(WebappClassLoaderBase.java:2476)
at org.apache.catalina.loader.WebappClassLoaderBase.findClass(WebappClassLoaderBase.java:857)
at org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1282)
at org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1164)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:348)
at org.jivesoftware.smack.SmackInitialization.loadSmackClass(SmackInitialization.java:213)
at org.jivesoftware.smack.SmackInitialization.parseClassesToLoad(SmackInitialization.java:193)
at org.jivesoftware.smack.SmackInitialization.processConfigFile(SmackInitialization.java:163)
at org.jivesoftware.smack.SmackInitialization.processConfigFile(SmackInitialization.java:148)
at org.jivesoftware.smack.SmackInitialization.<clinit>(SmackInitialization.java:116)
at org.jivesoftware.smack.SmackConfiguration.getVersion(SmackConfiguration.java:96)
at org.jivesoftware.smack.provider.ProviderManager.<clinit>(ProviderManager.java:121)
at br.com.soma.service.gcm.xmpp.CcsClientImpl.<init>(CcsClientImpl.java:73)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
at org.springframework.beans.BeanUtils.instantiateClass(BeanUtils.java:147)
... 60 more
Caused by: java.lang.ClassNotFoundException: org.jivesoftware.smack.initializer.SmackAndOsgiInitializer
at org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1313)
at org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1164)
... 82 more
Can someone help me ? Thanks!
It works fine, now. I just set a variable for smack as you suggested. Thanks!!
<properties>
<smack.version>4.1.4</smack.version>
</properties>
<dependency>
<groupId>org.igniterealtime.smack</groupId>
<artifactId>smack-core</artifactId>
<version>${smack.version}</version>
</dependency>
<dependency>
<groupId>org.igniterealtime.smack</groupId>
<artifactId>smack-tcp</artifactId>
<version>${smack.version}</version>
</dependency>
<dependency>
<groupId>org.igniterealtime.smack</groupId>
<artifactId>smack-extensions</artifactId>
<version>${smack.version}</version>
</dependency>
<dependency>
<groupId>org.igniterealtime.smack</groupId>
<artifactId>smack-java7</artifactId>
<version>${smack.version}</version>
</dependency>

Quartz doesn't recognize schema job_scheduling_data_2_0.xsd present in quartz jar file

I am getting below exception on server startup.
I am using quartz 2.2.21 with spring 3.2.
I have enabled quartz plugin (org.quartz.plugins.xml.XMLSchedulingDataProcessorPlugin).
Please find below the start tag of our XML file:
During server startup we are getting below log information and stacktrace:
Error Message:
Unable to load local schema packaged in quartz distribution jar. Utilizing schema online at http://www.quartz-scheduler.org/xml/job_scheduling_data_2_0.xsd
Exception:
Caused by: org.xml.sax.SAXParseException; systemId: file:///quartz_job_data.xml; lineNumber: 5; columnNumber: 104;
schema_reference.4: Failed to read schema document 'http://www.quartz-scheduler.org/xml/job_scheduling_data_2_0.xsd', because 1) could not find the document; 2) the document could not be read; 3) the root element of the document is not <xsd:schema>.
I have the same problem. I'm using the 7.1.1 of Jboss and the problem appears when you don't have connection to the internet. This is easy as putting a fake address that's unreachable in hosts.
I tried to force to local copy but it does not work.
What I finally did is to partially overwrite the functionality until this is fixed. Watch: https://jira.spring.io/browse/SPR-13706
public class CustomXMLSchedulingDataProcessor extends org.quartz.xml.XMLSchedulingDataProcessor {
public static final String QUARTZ_XSD_PATH_IN_JAR_CLASSPATH = "classpath:org/quartz/xml/job_scheduling_data_2_0.xsd";
public CustomXMLSchedulingDataProcessor(ClassLoadHelper clh) throws ParserConfigurationException {
super(clh);
}
#Override
protected Object resolveSchemaSource() {
InputSource inputSource;
InputStream is = null;
try {
is = classLoadHelper.getResourceAsStream(QUARTZ_XSD_PATH_IN_JAR_CLASSPATH);
} finally {
if (is != null) {
inputSource = new InputSource(is);
inputSource.setSystemId(QUARTZ_SCHEMA_WEB_URL);
}
else {
return QUARTZ_SCHEMA_WEB_URL;
}
}
return inputSource;
}
}
And I did a new plugin XMLSchedulingDataProcessorPlugin overwritting just the instanciation of above class.
public class XMLSchedulingDataProcessorPlugin
extends SchedulerPluginWithUserTransactionSupport
implements FileScanListener {
/*
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*
* Data members.
*
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*/
private static final int MAX_JOB_TRIGGER_NAME_LEN = 80;
private static final String JOB_INITIALIZATION_PLUGIN_NAME = "JobSchedulingDataLoaderPlugin";
private static final String FILE_NAME_DELIMITERS = ",";
private boolean failOnFileNotFound = true;
private String fileNames = CustomXMLSchedulingDataProcessor.QUARTZ_XML_DEFAULT_FILE_NAME;
// Populated by initialization
private Map<String, JobFile> jobFiles = new LinkedHashMap<String, JobFile>();
private long scanInterval = 0;
boolean started = false;
protected ClassLoadHelper classLoadHelper = null;
private Set<String> jobTriggerNameSet = new HashSet<String>();
/*
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*
* Constructors.
*
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*/
public XMLSchedulingDataProcessorPlugin() {
}
/*
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*
* Interface.
*
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*/
/**
* Comma separated list of file names (with paths) to the XML files that should be read.
*/
public String getFileNames() {
return fileNames;
}
/**
* The file name (and path) to the XML file that should be read.
*/
public void setFileNames(String fileNames) {
this.fileNames = fileNames;
}
/**
* The interval (in seconds) at which to scan for changes to the file.
* If the file has been changed, it is re-loaded and parsed. The default
* value for the interval is 0, which disables scanning.
*
* #return Returns the scanInterval.
*/
public long getScanInterval() {
return scanInterval / 1000;
}
/**
* The interval (in seconds) at which to scan for changes to the file.
* If the file has been changed, it is re-loaded and parsed. The default
* value for the interval is 0, which disables scanning.
*
* #param scanInterval The scanInterval to set.
*/
public void setScanInterval(long scanInterval) {
this.scanInterval = scanInterval * 1000;
}
/**
* Whether or not initialization of the plugin should fail (throw an
* exception) if the file cannot be found. Default is <code>true</code>.
*/
public boolean isFailOnFileNotFound() {
return failOnFileNotFound;
}
/**
* Whether or not initialization of the plugin should fail (throw an
* exception) if the file cannot be found. Default is <code>true</code>.
*/
public void setFailOnFileNotFound(boolean failOnFileNotFound) {
this.failOnFileNotFound = failOnFileNotFound;
}
/*
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*
* SchedulerPlugin Interface.
*
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*/
/**
* <p>
* Called during creation of the <code>Scheduler</code> in order to give
* the <code>SchedulerPlugin</code> a chance to initialize.
* </p>
*
* #throws org.quartz.SchedulerConfigException
* if there is an error initializing.
*/
public void initialize(String name, final Scheduler scheduler, ClassLoadHelper schedulerFactoryClassLoadHelper)
throws SchedulerException {
super.initialize(name, scheduler);
this.classLoadHelper = schedulerFactoryClassLoadHelper;
getLog().info("Registering Quartz Job Initialization Plug-in.");
// Create JobFile objects
StringTokenizer stok = new StringTokenizer(fileNames, FILE_NAME_DELIMITERS);
while (stok.hasMoreTokens()) {
final String fileName = stok.nextToken();
final JobFile jobFile = new JobFile(fileName);
jobFiles.put(fileName, jobFile);
}
}
#Override
public void start(UserTransaction userTransaction) {
try {
if (jobFiles.isEmpty() == false) {
if (scanInterval > 0) {
getScheduler().getContext().put(JOB_INITIALIZATION_PLUGIN_NAME + '_' + getName(), this);
}
Iterator<JobFile> iterator = jobFiles.values().iterator();
while (iterator.hasNext()) {
JobFile jobFile = iterator.next();
if (scanInterval > 0) {
String jobTriggerName = buildJobTriggerName(jobFile.getFileBasename());
TriggerKey tKey = new TriggerKey(jobTriggerName, JOB_INITIALIZATION_PLUGIN_NAME);
// remove pre-existing job/trigger, if any
getScheduler().unscheduleJob(tKey);
JobDetail job = newJob().withIdentity(jobTriggerName, JOB_INITIALIZATION_PLUGIN_NAME).ofType(FileScanJob.class)
.usingJobData(FileScanJob.FILE_NAME, jobFile.getFileName())
.usingJobData(FileScanJob.FILE_SCAN_LISTENER_NAME, JOB_INITIALIZATION_PLUGIN_NAME + '_' + getName())
.build();
SimpleTrigger trig = newTrigger().withIdentity(tKey).withSchedule(
simpleSchedule().repeatForever().withIntervalInMilliseconds(scanInterval))
.forJob(job)
.build();
getScheduler().scheduleJob(job, trig);
getLog().debug("Scheduled file scan job for data file: {}, at interval: {}", jobFile.getFileName(), scanInterval);
}
processFile(jobFile);
}
}
} catch(SchedulerException se) {
getLog().error("Error starting background-task for watching jobs file.", se);
} finally {
started = true;
}
}
/**
* Helper method for generating unique job/trigger name for the
* file scanning jobs (one per FileJob). The unique names are saved
* in jobTriggerNameSet.
*/
private String buildJobTriggerName(
String fileBasename) {
// Name w/o collisions will be prefix + _ + filename (with '.' of filename replaced with '_')
// For example: JobInitializationPlugin_jobInitializer_myjobs_xml
String jobTriggerName = JOB_INITIALIZATION_PLUGIN_NAME + '_' + getName() + '_' + fileBasename.replace('.', '_');
// If name is too long (DB column is 80 chars), then truncate to max length
if (jobTriggerName.length() > MAX_JOB_TRIGGER_NAME_LEN) {
jobTriggerName = jobTriggerName.substring(0, MAX_JOB_TRIGGER_NAME_LEN);
}
// Make sure this name is unique in case the same file name under different
// directories is being checked, or had a naming collision due to length truncation.
// If there is a conflict, keep incrementing a _# suffix on the name (being sure
// not to get too long), until we find a unique name.
int currentIndex = 1;
while (jobTriggerNameSet.add(jobTriggerName) == false) {
// If not our first time through, then strip off old numeric suffix
if (currentIndex > 1) {
jobTriggerName = jobTriggerName.substring(0, jobTriggerName.lastIndexOf('_'));
}
String numericSuffix = "_" + currentIndex++;
// If the numeric suffix would make the name too long, then make room for it.
if (jobTriggerName.length() > (MAX_JOB_TRIGGER_NAME_LEN - numericSuffix.length())) {
jobTriggerName = jobTriggerName.substring(0, (MAX_JOB_TRIGGER_NAME_LEN - numericSuffix.length()));
}
jobTriggerName += numericSuffix;
}
return jobTriggerName;
}
/**
* Overriden to ignore <em>wrapInUserTransaction</em> because shutdown()
* does not interact with the <code>Scheduler</code>.
*/
#Override
public void shutdown() {
// Since we have nothing to do, override base shutdown so don't
// get extranious UserTransactions.
}
private void processFile(JobFile jobFile) {
if (jobFile == null || !jobFile.getFileFound()) {
return;
}
try {
CustomXMLSchedulingDataProcessor processor =
new CustomXMLSchedulingDataProcessor(this.classLoadHelper);
processor.addJobGroupToNeverDelete(JOB_INITIALIZATION_PLUGIN_NAME);
processor.addTriggerGroupToNeverDelete(JOB_INITIALIZATION_PLUGIN_NAME);
processor.processFileAndScheduleJobs(
jobFile.getFileName(),
jobFile.getFileName(), // systemId
getScheduler());
} catch (Exception e) {
getLog().error("Error scheduling jobs: " + e.getMessage(), e);
}
}
public void processFile(String filePath) {
processFile((JobFile)jobFiles.get(filePath));
}
/**
* #see org.quartz.jobs.FileScanListener#fileUpdated(java.lang.String)
*/
public void fileUpdated(String fileName) {
if (started) {
processFile(fileName);
}
}
class JobFile {
private String fileName;
// These are set by initialize()
private String filePath;
private String fileBasename;
private boolean fileFound;
protected JobFile(String fileName) throws SchedulerException {
this.fileName = fileName;
initialize();
}
protected String getFileName() {
return fileName;
}
protected boolean getFileFound() {
return fileFound;
}
protected String getFilePath() {
return filePath;
}
protected String getFileBasename() {
return fileBasename;
}
private void initialize() throws SchedulerException {
InputStream f = null;
try {
String furl = null;
File file = new File(getFileName()); // files in filesystem
if (!file.exists()) {
URL url = classLoadHelper.getResource(getFileName());
if(url != null) {
try {
furl = URLDecoder.decode(url.getPath(), "UTF-8");
} catch (UnsupportedEncodingException e) {
furl = url.getPath();
}
file = new File(furl);
try {
f = url.openStream();
} catch (IOException ignor) {
// Swallow the exception
}
}
} else {
try {
f = new java.io.FileInputStream(file);
}catch (FileNotFoundException e) {
// ignore
}
}
if (f == null) {
if (isFailOnFileNotFound()) {
throw new SchedulerException(
"File named '" + getFileName() + "' does not exist.");
} else {
getLog().warn("File named '" + getFileName() + "' does not exist.");
}
} else {
fileFound = true;
}
filePath = (furl != null) ? furl : file.getAbsolutePath();
fileBasename = file.getName();
} finally {
try {
if (f != null) {
f.close();
}
} catch (IOException ioe) {
getLog().warn("Error closing jobs file " + getFileName(), ioe);
}
}
}
}
}
That way you only have to use this plugin in your configuration and everything will work by default.
org.quartz.plugin.jobInitializer.class =
com.level2.quartz.processor.plugin.XMLSchedulingDataProcessorPlugin

isSplitable in combineFileInputFormat does not work

I have thousands of small files, and I want to process them with combineFileInputFormat.
In combineFileInputFormat, there are multiple small files for one mapper, each file will not be split.
the snippet of one of the small input files like this,
vers,3
period,2015-01-26-18-12-00,438469546,449329626,complete
config,libdvm.so,chromeview
pkgproc,com.futuredial.digitchat,10021,,0ns:10860078
pkgpss,com.futuredial.digitchat,10021,,0ns:9:6627:6627:6637:5912:5912:5912
pkgsvc-run,com.futuredial.digitchat,10021,.LiveScreenService,1,0n:10860078
pkgsvc-start,com.futuredial.digitchat,10021,.LiveScreenService,1,0n:10860078
pkgproc,com.google.android.youtube,10103,,0ns:10860078
pkgpss,com.google.android.youtube,10103,,0ns:9:12986:13000:13021:11552:11564:11580
pkgsvc- run,com.google.android.youtube,10103,com.google.android.apps.youtube.app.offline.transfer.OfflineTransferService,1,0n:10860078
pkgsvc- start,com.google.android.youtube,10103,com.google.android.apps.youtube.app.offline.transfer.OfflineTransferService,1,0n:10860078
I want to pass whole file content to the mapper. However, hadoop split the file to half.
For example, the above file may be split into
vers,3
period,2015-01-26-18-12-00,438469546,449329626,complete
config,libdvm.so,chromeview
pkgproc,com.futuredial.digitchat,#the line has been cut
But I want the content of whole file to be processed.
Here is my code, which reference Reading file as single record in hadoop
The driven code
public class CombineSmallfiles {
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (otherArgs.length != 2) {
System.err.println("Usage: conbinesmallfiles <in> <out>");
System.exit(2);
}
conf.setInt("mapred.min.split.size", 1);
conf.setLong("mapred.max.split.size", 26214400); // 25m
//conf.setLong("mapred.max.split.size", 134217728); // 128m
//conf.setInt("mapred.reduce.tasks", 5);
Job job = new Job(conf, "combine smallfiles");
job.setJarByClass(CombineSmallfiles.class);
job.setMapperClass(CombineSmallfileMapper.class);
//job.setReducerClass(IdentityReducer.class);
job.setNumReduceTasks(0);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
MultipleOutputs.addNamedOutput(job,"pkgproc",TextOutputFormat.class,Text.class,Text.class);
MultipleOutputs.addNamedOutput(job,"pkgpss",TextOutputFormat.class,Text.class,Text.class);
MultipleOutputs.addNamedOutput(job,"pkgsvc",TextOutputFormat.class,Text.class,Text.class);
job.setInputFormatClass(CombineSmallfileInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
int exitFlag = job.waitForCompletion(true) ? 0 : 1;
System.exit(exitFlag);
}
}
My Mapper code
public class CombineSmallfileMapper extends Mapper<NullWritable, Text, Text, Text> {
private Text file = new Text();
private MultipleOutputs mos;
private String period;
private Long elapsed;
#Override
public void setup(Context context) throws IOException, InterruptedException {
mos = new MultipleOutputs(context);
}
#Override
protected void map(NullWritable key, Text value, Context context) throws IOException, InterruptedException {
String file_name = context.getConfiguration().get("map.input.file.name");
String [] filename_tokens = file_name.split("_");
String uuid = filename_tokens[0];
String [] datetime_tokens;
try{
datetime_tokens = filename_tokens[1].split("-");
}catch(ArrayIndexOutOfBoundsException err){
throw new ArrayIndexOutOfBoundsException(file_name);
}
String year,month,day,hour,minute,sec,msec;
year = datetime_tokens[0];
month = datetime_tokens[1];
day = datetime_tokens[2];
hour = datetime_tokens[3];
minute = datetime_tokens[4];
sec = datetime_tokens[5];
msec = datetime_tokens[6];
String datetime = year+"-"+month+"-"+"-"+day+" "+hour+":"+minute+":"+sec+"."+msec;
String content = value.toString();
String []lines = content.split("\n");
for(int u = 0;u<lines.length;u++){
String line = lines[u];
String []tokens = line.split(",");
if(tokens[0].equals("period")){
period = tokens[1];
try{
long startTime = Long.valueOf(tokens[2]);
long endTime = Long.valueOf(tokens[3]);
elapsed = endTime-startTime;
}catch(NumberFormatException err){
throw new NumberFormatException(line);
}
}else if(tokens[0].equals("pkgproc")){
String proc_info = "";
try{
proc_info += period+","+String.valueOf(elapsed)+","+tokens[2]+","+tokens[3];
}catch(ArrayIndexOutOfBoundsException err){
throw new ArrayIndexOutOfBoundsException("pkgproc: "+content+ "line:"+line);
}
for(int i = 4;i<tokens.length;i++){
String []state_info = tokens[i].split(":");
String state = "";
state += ","+state_info[0].charAt(0)+","+state_info[0].charAt(1)+","+state_info[0].charAt(2)+","+state_info[1];
mos.write("pkgproc",new Text(tokens[1]), new Text(proc_info+state+','+uuid+','+datetime));
}
}else if(tokens[0].equals("pkgpss")){
String proc_info = "";
proc_info += period+","+String.valueOf(elapsed)+","+tokens[2]+","+tokens[3];
for(int i = 4;i<tokens.length;i++){
String []state_info = tokens[i].split(":");
String state = "";
state += ","+state_info[0].charAt(0)+","+state_info[0].charAt(1)+","+state_info[0].charAt(2)+","+state_info[1]+","+state_info[2]+","+state_info[3]+","+state_info[4]+","+state_info[5]+","+state_info[6]+","+state_info[7];
mos.write("pkgpss",new Text(tokens[1]), new Text(proc_info+state+','+uuid+','+datetime));
}
}else if(tokens[0].startsWith("pkgsvc")){
String []stateName = tokens[0].split("-");
String proc_info = "";
//tokens[2] = uid, tokens[3] = serviceName
proc_info += stateName[1]+','+period+","+String.valueOf(elapsed)+","+tokens[2]+","+tokens[3];
String opcount = tokens[4];
for(int i = 5;i<tokens.length;i++){
String []state_info = tokens[i].split(":");
String state = "";
state += ","+state_info[0].charAt(0)+","+state_info[0].charAt(1)+","+state_info[1];
mos.write("pkgsvc",new Text(tokens[1]), new Text(proc_info+state+','+opcount+','+uuid+','+datetime));
}
}
}
}
}
My CombineFileInputFormat, which overrides isSplitable and return false
public class CombineSmallfileInputFormat extends CombineFileInputFormat<NullWritable, Text> {
#Override
public RecordReader<NullWritable, Text> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException {
return new CombineFileRecordReader<NullWritable,Text>((CombineFileSplit) split,context,WholeFileRecordReader.class);
}
#Override
protected boolean isSplitable(JobContext context,Path file ){
return false;
}
}
The WholeFileRecordReader
public class WholeFileRecordReader extends RecordReader<NullWritable, Text> {
//private static final Logger LOG = Logger.getLogger(WholeFileRecordReader.class);
/** The path to the file to read. */
private final Path mFileToRead;
/** The length of this file. */
private final long mFileLength;
/** The Configuration. */
private final Configuration mConf;
/** Whether this FileSplit has been processed. */
private boolean mProcessed;
/** Single Text to store the file name of the current file. */
// private final Text mFileName;
/** Single Text to store the value of this file (the value) when it is read. */
private final Text mFileText;
/**
* Implementation detail: This constructor is built to be called via
* reflection from within CombineFileRecordReader.
*
* #param fileSplit The CombineFileSplit that this will read from.
* #param context The context for this task.
* #param pathToProcess The path index from the CombineFileSplit to process in this record.
*/
public WholeFileRecordReader(CombineFileSplit fileSplit, TaskAttemptContext context,
Integer pathToProcess) {
mProcessed = false;
mFileToRead = fileSplit.getPath(pathToProcess);
mFileLength = fileSplit.getLength(pathToProcess);
mConf = context.getConfiguration();
context.getConfiguration().set("map.input.file.name", mFileToRead.getName());
assert 0 == fileSplit.getOffset(pathToProcess);
//if (LOG.isDebugEnabled()) {
//LOG.debug("FileToRead is: " + mFileToRead.toString());
//LOG.debug("Processing path " + pathToProcess + " out of " + fileSplit.getNumPaths());
//try {
//FileSystem fs = FileSystem.get(mConf);
//assert fs.getFileStatus(mFileToRead).getLen() == mFileLength;
//} catch (IOException ioe) {
//// oh well, I was just testing.
//}
//}
//mFileName = new Text();
mFileText = new Text();
}
/** {#inheritDoc} */
#Override
public void close() throws IOException {
mFileText.clear();
}
/**
* Returns the absolute path to the current file.
*
* #return The absolute path to the current file.
* #throws IOException never.
* #throws InterruptedException never.
*/
#Override
public NullWritable getCurrentKey() throws IOException, InterruptedException {
return NullWritable.get();
}
/**
* <p>Returns the current value. If the file has been read with a call to NextKeyValue(),
* this returns the contents of the file as a BytesWritable. Otherwise, it returns an
* empty BytesWritable.</p>
*
* <p>Throws an IllegalStateException if initialize() is not called first.</p>
*
* #return A BytesWritable containing the contents of the file to read.
* #throws IOException never.
* #throws InterruptedException never.
*/
#Override
public Text getCurrentValue() throws IOException, InterruptedException {
return mFileText;
}
/**
* Returns whether the file has been processed or not. Since only one record
* will be generated for a file, progress will be 0.0 if it has not been processed,
* and 1.0 if it has.
*
* #return 0.0 if the file has not been processed. 1.0 if it has.
* #throws IOException never.
* #throws InterruptedException never.
*/
#Override
public float getProgress() throws IOException, InterruptedException {
return (mProcessed) ? (float) 1.0 : (float) 0.0;
}
/**
* All of the internal state is already set on instantiation. This is a no-op.
*
* #param split The InputSplit to read. Unused.
* #param context The context for this task. Unused.
* #throws IOException never.
* #throws InterruptedException never.
*/
#Override
public void initialize(InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
// no-op.
}
/**
* <p>If the file has not already been read, this reads it into memory, so that a call
* to getCurrentValue() will return the entire contents of this file as Text,
* and getCurrentKey() will return the qualified path to this file as Text. Then, returns
* true. If it has already been read, then returns false without updating any internal state.</p>
*
* #return Whether the file was read or not.
* #throws IOException if there is an error reading the file.
* #throws InterruptedException if there is an error.
*/
#Override
public boolean nextKeyValue() throws IOException, InterruptedException {
if (!mProcessed) {
if (mFileLength > (long) Integer.MAX_VALUE) {
throw new IOException("File is longer than Integer.MAX_VALUE.");
}
byte[] contents = new byte[(int) mFileLength];
FileSystem fs = mFileToRead.getFileSystem(mConf);
FSDataInputStream in = null;
try {
// Set the contents of this file.
in = fs.open(mFileToRead);
IOUtils.readFully(in, contents, 0, contents.length);
mFileText.set(contents, 0, contents.length);
} finally {
IOUtils.closeQuietly(in);
}
mProcessed = true;
return true;
}
return false;
}
}
I want every mapper to parse multiple small files and each small file can not be split.
However, above code will cut(split) my input file and will raise a parsing error (since my parser will split the line into tokens).
In my concept, combineFileInputFormat will gather multiple files into one split, and each split will feed into one mapper. Therefore, one mapper can handle multiple files.
In my code, the max input split is set to 25MB, so I think the problem is that combineFileInputFormat will split the last part of small file of input split to satisfy the split size limit.
However, I have override isSplitable and return false, but it still splits the small file.
What is the correct way to do that?
I am not sure if it is possible to specify number of files to a mapper, rather than specify input split size?
Use setMaxSplitSize() method in your constructor code, it should work,
It ideally tells the split size,
public class CFInputFormat extends CombineFileInputFormat<FileLineWritable, Text> {
public CFInputFormat(){
super();
setMaxSplitSize(67108864); // 64 MB, default block size on hadoop
}
public RecordReader<FileLineWritable, Text> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException{
return new CombineFileRecordReader<FileLineWritable, Text>((CombineFileSplit)split, context, CFRecordReader.class);
}
#Override
protected boolean isSplitable(JobContext context, Path file){
return false;
}
}

Serializing a long string in Hadoop

I have a class which implements WritableComparable class in Hadoop. This class has two string variables, one short and one very long. I use writeChars to write these variables and readLine to read them but it seems like I get some sort of error. What is the best way to serialize such a long String in Hadoop?
I think you can use byteswritable to make it more efficient. Check the below custom key which has BytesWritable type as callId.
public class CustomMRKey implements WritableComparable<CustomMRKey> {
private BytesWritable callId;
private IntWritable mapperType;
/**
* #default constructor
*/
public CustomMRKey() {
set(new BytesWritable(), new IntWritable());
}
/**
* Constructor
*
* #param callId
* #param mapperType
*/
public CustomMRKey(BytesWritable callId, IntWritable mapperType) {
set(callId, mapperType);
}
/**
* sets the call id and mapper type
*
* #param callId
* #param mapperType
*/
public void set(BytesWritable callId, IntWritable mapperType) {
this.callId = callId;
this.mapperType = mapperType;
}
/**
* This method returns the callId
*
* #return callId
*/
public BytesWritable getCallId() {
return callId;
}
/**
* This method sets the callId given a callId
*
* #param callId
*/
public void setCallId(BytesWritable callId) {
this.callId = callId;
}
/**
* This method returns the mapper type
*
*
* #return
*/
public IntWritable getMapperType() {
return mapperType;
}
/**
* This method is set to store the mapper type
*
* #param mapperType
*/
public void setMapperType(IntWritable mapperType) {
this.mapperType = mapperType;
}
#Override
public void readFields(DataInput in) throws IOException {
callId.readFields(in);
mapperType.readFields(in);
}
#Override
public void write(DataOutput out) throws IOException {
callId.write(out);
mapperType.write(out);
}
#Override
public boolean equals(Object obj) {
if (obj instanceof CustomMRCdrKey) {
CustomMRCdrKey key = (CustomMRCdrKey) obj;
return callId.equals(key.callId)
&& mapperType.equals(key.mapperType);
}
return false;
}
#Override
public int compareTo(CustomMRCdrKey key) {
int cmp = callId.compareTo(key.getCallId());
if (cmp != 0) {
return cmp;
}
return mapperType.compareTo(key.getMapperType());
}
}
To use in say mapper code say you can generate the key of BytesWritable form using something as following :-
You can call as :
CustomMRKey customKey=new CustomMRKey(new BytesWritable(),new IntWritable());
customKey.setCallId(makeKey(value, this.resultKey));
customKey.setMapperType(this.mapTypeIndicator);
Then makeKey method is something like below :-
public BytesWritable makeKey(Text value, BytesWritable key) throws IOException {
try {
ByteArrayOutputStream byteKey = new ByteArrayOutputStream(Constants.MR_DEFAULT_KEY_SIZE);
for (String field : keyFields) {
byte[] bytes = value.getString(field).getBytes();
byteKey.write(bytes,0,bytes.length);
}
if(key==null){
return new BytesWritable(byteKey.toByteArray());
}else{
key.set(byteKey.toByteArray(), 0, byteKey.size());
return key;
}
} catch (Exception ex) {
throw new IOException("Could not generate key", ex);
}
}
Hope this may help.

Resources