ASM - java.lang.VerifyError: Operand stack overflow Exception - java-8

I am using ASM 5.0.3 byte code library with Tomcat 8 and JDK 8.
Also I am using
ClassWriter classWriter = new ClassWriter(classReader, ClassWriter.COMPUTE_FRAMES);
and
classReader.accept(myClassVisitor, ClassReader.SKIP_FRAMES);
Below exception is thrown by ASM
Caused by: java.lang.VerifyError: Operand stack overflow
Exception Details:
Location: org/apache/tomcat/websocket/server/WsServerContainer.addEndpoint(Ljavax/websocket/server/ServerEndpointConfig;)V #0: aload_0
Reason:
Exceeded max stack size.
Current Frame:
bci: #0
flags: { }
locals: { 'org/apache/tomcat/websocket/server/WsServerContainer', 'javax/websocket/server/ServerEndpointConfig' }
stack: { }
Bytecode:
0x0000000: 2ab4 000a 9900 1a2a b400 0b9a 0013 bb01
0x0000010: 3d59 b200 4412 45b6 0046 b700 47bf 2ab4
0x0000020: 000e c700 13bb 0043 59b2 0044 1248 b600
0x0000030: 46b7 0047 bf2b b900 4901 004d bb00 4a59
0x0000040: 2bb9 004b 0100 2bb9 004c 0100 2cb7 004d
0x0000050: 4e2d b600 4ec7 0018 2db6 004f c700 112d
0x0000060: b600 50c7 000a 2db6 0051 9900 122b b900
0x0000070: 5201 0012 532d b900 5403 0057 bb00 5559
0x0000080: 2cb7 0056 3a04 1904 b600 5799 0087 1904
0x0000090: b600 58b8 0059 3a05 2ab4 0008 1905 b600
0x00000a0: 5ac0 005b 3a06 1906 c700 29bb 005c 59b8
0x00000b0: 005d b700 5e3a 062a b400 0819 0519 06b6
0x00000c0: 005f 572a b400 0819 05b6 005a c000 5b3a
0x00000d0: 0619 06bb 0060 592b 1904 b700 61b9 0062
0x00000e0: 0200 9a00 2dbb 0043 59b2 0044 1263 06bd
0x00000f0: 0064 5903 2c53 5904 2bb9 004b 0100 5359
0x0000100: 052b b900 4b01 0053 b600 65b7 0047 bfa7
0x0000110: 0043 2ab4 0007 2c2b b900 5403 00c0 0066
0x0000120: 3a05 1905 c600 2ebb 0043 59b2 0044 1263
0x0000130: 06bd 0064 5903 2c53 5904 1905 b900 4b01
0x0000140: 0053 5905 2bb9 004b 0100 53b6 0065 b700
0x0000150: 47bf 2a04 b500 0db1
at org.apache.tomcat.websocket.server.WsSci.init(WsSci.java:131)
at org.apache.tomcat.websocket.server.WsSci.onStartup(WsSci.java:47)
at org.apache.catalina.core.StandardContext.startInternal(StandardContext.java:5244)
at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:150)
How to resolve this issue without using -noverify option?
Edit #1: I also tried using simply COMPUTE_FRAMES as in:
ClassWriter classWriter = new ClassWriter(classReader, ClassWriter.COMPUTE_FRAMES);
and I omitted the below SKIP_FRAMES statement:
//classReader.accept(myClassVisitor, ClassReader.SKIP_FRAMES);
I continue to get the same "Caused by: java.lang.VerifyError: Operand stack overflow" error.
Edit #2: I have tried all of the following combinations and am getting the same error: java.lang.VerifyError: Operand stack overflow Reason:Exceeded max stack size.
ClassWriter.COMPUTE_FRAMES with ClassReader.EXPAND_FRAMES and mv.visitMaxs(maxStack, maxLocals);
ClassWriter.COMPUTE_FRAMES with ClassReader.EXPAND_FRAMES and mv.visitMaxs(-1, -1);
ClassWriter.COMPUTE_FRAMES with ClassReader.SKIP_FRAMES and mv.visitMaxs(maxStack, maxLocals);
ClassWriter.COMPUTE_FRAMES with ClassReader.SKIP_FRAMES and mv.visitMaxs(-1, -1);
Edit #3. Here is my Advice adapter code
public class PojoMethodAdviceAdapter extends AdviceAdapter {
private String methodName;
private String className;
private String description;
private boolean excludeCheckFlag = false;
private int okFlag = newLocal(Type.getType("Z")); //newLocal(Type.BOOLEAN_TYPE);
private int classFileVersion;
Label startFinally = new Label();
private static Hashtable excludeMethods = new Hashtable();
private static final String yesString = "Yes";
private boolean isValid;
static{
excludeMethods.put("<clinit>", yesString);
excludeMethods.put("<init>", yesString);
}
public PojoMethodAdviceAdapter(int access , MethodVisitor mv , String methodName, String description, String className, int classFileVersion){
super(Opcodes.ASM5 , mv, access, methodName, description);
this.className = className;
this.methodName = methodName;
this.description = description;
this.excludeCheckFlag = true;
this.isValid = true;
this.classFileVersion = classFileVersion;
String yesStr = (String)excludeMethods.get(this.methodName);
if(yesStr!=null && yesStr.equals(yesString))
isValid = false;
if(this.methodName.indexOf("$") > 0)
isValid = false;
if(isValid){
System.out.println(" [POJO MethodAdviceAdapter] :"+className+" \t "+methodName +" \t "+description);
}
}
public void visitCode() {
super.visitCode();
mv.visitLabel(startFinally);
}
protected void onMethodEnter(){
if(isValid) {
mv.visitInsn(Opcodes.ICONST_0);
mv.visitVarInsn(ISTORE, okFlag);
mv.visitLdcInsn(className);
mv.visitLdcInsn(methodName);
mv.visitLdcInsn(description);
mv.visitMethodInsn(Opcodes.INVOKESTATIC, "com/mini/agent/trace/RootTracer", "pojoMethodBegin", "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)Z", false);
mv.visitVarInsn(ISTORE, okFlag);
}
}
protected void onMethodExit(int opcode){
if(opcode!=ATHROW) {
onFinally(opcode);
}
}
public void visitMaxs(int maxStack, int maxLocals){
Label endFinally = new Label();
mv.visitTryCatchBlock(startFinally, endFinally, endFinally, null);
mv.visitLabel(endFinally);
onFinally(ATHROW);
mv.visitInsn(ATHROW);
if(classFileVersion <= 50){
mv.visitMaxs(maxStack, maxLocals);
}
else{
mv.visitMaxs(0, 0);
}
}
private void onFinally(int opcode){
if(isValid){
if(opcode == ATHROW){
mv.visitInsn(Opcodes.DUP);
mv.visitLdcInsn(className);
mv.visitLdcInsn(methodName);
mv.visitLdcInsn(description);
mv.visitVarInsn(ILOAD, okFlag);
mv.visitMethodInsn(Opcodes.INVOKESTATIC, "com/mini/agent/trace/RootTracer", "recordPOJOException", "(Ljava/lang/Object;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Z)V", false);
}
mv.visitLdcInsn(className);
mv.visitLdcInsn(methodName);
mv.visitLdcInsn(description);
mv.visitVarInsn(ILOAD, okFlag);
mv.visitLdcInsn(opcode);
mv.visitMethodInsn(Opcodes.INVOKESTATIC, "com/mini/agent/trace/RootTracer", "pojoMethodEnd", "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZI)V", false);
}
}
}

Try using the TraceClassVisitor to see where it is getting messed up. In my experience this happens when a Label is written incorrectly, forcing it to write jsr/ret instructions (the JVM should simply reject the class in that case though along with asm crashing when calculating frames), and most often, forgetting to call visitMaxs in the methodVisitor before calling visitEnd. In my code I'm using
visitMaxs(mv.getMaxLocals(),mv.getMaxLocals());
I don't remember what my reasoning for doing that was but my ClassWriter works when I do, but I hope this helps.

Related

Chainlink: Contract cannot retrieve large-response data type from external adapter

I tried to use the large-response type to fulfil the request but somehow it does not show up in my contract I tried to fulfil the status value, and the job works completely as shown in my chainlink node but it does not change the status value, it stays 0x as it is. So, I wonder that is my contract or job spec wrong?
This is my contract
// SPDX-License-Identifier: MIT
pragma solidity ^0.8.7;
import "#chainlink/contracts/src/v0.8/ChainlinkClient.sol";
contract APIConsumer is ChainlinkClient {
using Chainlink for Chainlink.Request;
bytes public status;
string public statusString;
address private oracle;
bytes32 private jobId;
uint256 private fee;
event RequestFulfilled(bytes32 indexed requestId,bytes indexed data);
/**
* Network: Kovan
* Oracle: 0xc57B33452b4F7BB189bB5AfaE9cc4aBa1f7a4FD8 (Chainlink Devrel
* Node)
* Job ID: d5270d1c311941d0b08bead21fea7747
* Fee: 0.1 LINK
*/
constructor() {
setPublicChainlinkToken();
oracle = 0xDFE5e6C5C624724384b55719b7da79d3EbB60057;
fee = 1 * 10 ** 18; // (Varies by network and job)
}
function requesData(string memory _jobId) public returns (bytes32 requestId)
{
Chainlink.Request memory request = buildChainlinkRequest(stringToBytes32(_jobId), address(this), this.fulfill.selector);
// Set the URL to perform the GET request on
request.add("trackingNo", "HF123456789DL");
return sendChainlinkRequestTo(oracle, request, fee);
}
/**
* Receive the response in the form of uint256
*/
function fulfill(bytes32 _requestId, bytes memory bytesData) public recordChainlinkFulfillment(_requestId)
{
emit RequestFulfilled(_requestId, bytesData);
status = bytesData;
statusString = string(status);
}
// function withdrawLink() external {} - Implement a withdraw function to avoid locking your LINK in the contract
// function getStatus() public view returns (string memory) {
// return bytes32ToString(status);
// }
function bytes32ToString(bytes32 _bytes32)
public
pure
returns (string memory)
{
uint8 i = 0;
while (i < 32 && _bytes32[i] != 0) {
i++;
}
bytes memory bytesArray = new bytes(i);
for (i = 0; i < 32 && _bytes32[i] != 0; i++) {
bytesArray[i] = _bytes32[i];
}
return string(bytesArray);
}
function stringToBytes32(string memory source)
public
pure
returns (bytes32 result)
{
bytes memory tempEmptyStringTest = bytes(source);
if (tempEmptyStringTest.length == 0) {
return 0x0;
}
assembly {
// solhint-disable-line no-inline-assembly
result := mload(add(source, 32))
}
}
}
This is my job spec.
type = "directrequest"
schemaVersion = 1
name = "Halffin-Data-EA-Create-Tracking8"
externalJobID = "3f706a6b-efdd-44ac-8167-f880a6ca63ac"
maxTaskDuration = "0s"
contractAddress = "0xDFE5e6C5C624724384b55719b7da79d3EbB60057"
minIncomingConfirmations = 0
observationSource = """
decode_log [type=ethabidecodelog
abi="OracleRequest(bytes32 indexed specId, address requester, bytes32 requestId, uint256 payment, address callbackAddr, bytes4 callbackFunctionId, uint256 cancelExpiration, uint256 dataVersion, bytes data)"
data="$(jobRun.logData)"
topics="$(jobRun.logTopics)"]
decode_cbor [type=cborparse data="$(decode_log.data)"]
fetch [type=bridge name="halffin-data" requestData="{\\"id\\": $(jobSpec.externalJobID), \\"data\\": { \\"trackingNo\\": $(decode_cbor.trackingNo)}}"]
parse [type=jsonparse path="data,tracking,slug" data="$(fetch)"]
encode_data [type=ethabiencode abi="(bytes value)" data="{ \\"value\\": $(parse) }"]
encode_tx [type=ethabiencode
abi="fulfillOracleRequest(bytes32 requestId, uint256 payment, address callbackAddress, bytes4 callbackFunctionId, uint256 expiration, bytes data)"
data="{\\"requestId\\": $(decode_log.requestId), \\"payment\\": $(decode_log.payment), \\"callbackAddress\\": $(decode_log.callbackAddr), \\"callbackFunctionId\\": $(decode_log.callbackFunctionId), \\"expiration\\": $(decode_log.cancelExpiration), \\"data\\": $(encode_data)}"
]
submit_tx [type=ethtx to="0xDFE5e6C5C624724384b55719b7da79d3EbB60057" data="$(encode_tx)"]
decode_log -> decode_cbor -> fetch -> parse -> encode_data -> encode_tx -> submit_tx
"""
These are logs from completed job
fetch
"{\"jobRunID\":\"3f706a6b-efdd-44ac-8167-f880a6ca63ac\",\"data\":{\"tracking\":{\"id\":2,\"slug\":\"halffin-logistics\",\"tracking_number\":\"HF123456789DL\"},\"result\":null},\"result\":null}"
name: halffin-data
requestData: {"id": $(jobSpec.externalJobID), "data": { "trackingNo": $(decode_cbor.trackingNo)}}
parse
"halffin-logistics"
path: data,tracking,slug
data: $(fetch)
encode_data
"0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000001168616c6666696e2d6c6f67697374696373000000000000000000000000000000"
abi: (bytes value)
data: { "value": $(parse) }
encode_tx
"0x728853aa63b008d8b908b2d431b9ea703268ba10e60ab40603941ec91a2955278f219c1e0000000000000000000000000000000000000000000000000de0b6b3a7640000000000000000000000000000136e61cdeae727926aa768574e2f979c724d6cad7c1de7e1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000062586d6800000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000001168616c6666696e2d6c6f67697374696373000000000000000000000000000000"
abi: fulfillOracleRequest(bytes32 requestId, uint256 payment, address callbackAddress, bytes4 callbackFunctionId, uint256 expiration, bytes data)
data: {"requestId": $(decode_log.requestId), "payment": $(decode_log.payment), "callbackAddress": $(decode_log.callbackAddr), "callbackFunctionId": $(decode_log.callbackFunctionId), "expiration": $(decode_log.cancelExpiration), "data": $(encode_data)}
submit_tx
"{\"logs\": [], \"root\": \"0x\", \"status\": \"0x0\", \"gasUsed\": \"0x5c49\", \"blockHash\": \"0x5b55db677b2776bb637fdb9ba2077e7db21de8e8beba60fb79e1384ae51f39a8\", \"logsBloom\": \"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\", \"blockNumber\": \"0x1d94cd5\", \"contractAddress\": \"0x0000000000000000000000000000000000000000\", \"transactionHash\": \"0xc5371c5ce1692b39835e44fb61c47d2deeeb509f71e32fccb0c5d42eec3be443\", \"transactionIndex\": \"0x1\", \"cumulativeGasUsed\": \"0x39418\"}"
to: 0xDFE5e6C5C624724384b55719b7da79d3EbB60057
data: $(encode_tx)
In case, you might wonder why I use the large-response data type. I followed this link
The contractAddress specified in the (Get > Large bytes) must be pointed at operator.sol not oracle.sol. (Get > Uint256) is pointed at oracle.sol. Oracle.sol is not meant to handle Large bytes, nor multi-variable Uint256 output.
Here is the code to deploy the current version of operator.sol on remix to then obtain the correct contractAddress to the associated job-spec within the chainlink node GUI.
// SPDX-License-Identifier: MIT
pragma solidity ^0.7.0;
import "#chainlink/contracts/src/v0.7/Operator.sol";

ESP8266 Skips some data from GET request over HTTPS

Starting from the WiFiClient HTTPS Request example, I'm trying to get some code from a Github Gist using their API, but some parts of the file are missing
[...]
const char* host = "api.github.com";
const int httpsPort = 443;
String url = "/gists/0b3d5d67342c055cc9ba34b59d075da7";
// Use web browser to view and copy
// SHA1 fingerprint of the certificate
const char fingerprint[] PROGMEM = "59 74 61 88 13 CA 12 34 15 4D 11 0A C1 7F E6 67 07 69 42 F5";
void setup() {
Serial.begin(115200);
wifiSetup();
// Use WiFiClientSecure class to create TLS connection
WiFiClientSecure client;
Serial.print("Connecting to ");
Serial.println(host);
Serial.printf("Using fingerprint '%s'\n", fingerprint);
client.setFingerprint(fingerprint);
if (!client.connect(host, httpsPort)) {
Serial.println("Connection failed");
return;
}
Serial.print("Requesting URL: ");
Serial.println(url);
client.print(String("GET ") + url + " HTTP/1.1\r\n" +
"Host: " + host + "\r\n" +
"User-Agent: ESP8266\r\n" +
"Connection: close\r\n\r\n");
Serial.println("Request sent");
while (client.connected()) {
String line = client.readStringUntil('\n');
if (line == "\r") {
Serial.println("Headers received");
break;
}
}
String line = client.readStringUntil('\n');
line = line.substring(line.indexOf("content\":") + 10, line.indexOf("},") - 2);
Serial.println("Reply was:");
Serial.println("==========");
Serial.println(toHTML(line));
Serial.println("==========");
Serial.println("Setup end");
}
void loop() {
}
String toHTML(String source) {
Serial.print("toHTML length before/after: ");
Serial.print(source.length());
Serial.print("/");
source.replace("\\\\n", "==NEWLINE==");//Preserve document newlines
source.replace("\\n", "\n");//Replace newline
source.replace("==NEWLINE==", "\\n");
source.replace("\\\"", "\"");//Replace quotes
source.replace("\\t", "\t");//Replace tabs
Serial.println(source.length());
return source;
}
File I'm trying to get:
[...]
function printDebugMap() {
map.forEach((value, key) => {
printDebug(`${key} = [${value}]`);
});
}
</script>
</head>
<body>
<h1 id="AppName">Spreadsheet Automatico</h1>
<form onsubmit="return doNameInput()">
<p>Nome giocatori:</p>
<input type="text" id="nameBox">
<input type="submit" value="Submit">
</form>
<hr/>
What I actually get: some text is skipped
[...]
function printDebugMap() {
map.forEach((value, key) => {
printDebug(`${ype="text" id="nameBox">
<input type="submit" value="Submit">
</form>
<hr/>
when I add some random unrelated code which part is missing part changes, so I don't know what I'm doing wrong

Use HttpClientUpgradeHandler to establish Websocketconnection?

I try to establish a websocketconnection. But my websocketcontent keeps being rejected as not being readable. Right now I establish a connection by using the following handler stack:
p.addLast(ClientSslHandler),
p.addLast(new HttpClientCodec()),
p.addLast(new HttpObjectAggregator(8192)),
p.addLast(new WebSocketClientProtocolHandler(...)),
p.addlast(new myHandler);
The problem is (I think) that HttpClientCodec keeps encoding my Websocketframes as Httpframes.
Thats why i think I can use HttpClientUpgradeHandler so deal with the handshake and then change to a pure Websocketconnection. Is this a viable solution? Or are there other workarounds?
Edit: changed WebSocketClientProtocolHandler to HttpClientUpgradeHandler
Edit2:
public class ShipSocketClient {
static final String URL = System.getProperty("url", "wss://10.2.3.44:4712");
public static void main(String[] args) throws Exception{
URI uri = new URI(URL);
String scheme = uri.getScheme() == null? "ws" : uri.getScheme();
final String host = uri.getHost() == null? "127.0.0.1" : uri.getHost();
final int port;
if (uri.getPort() == -1) {
if ("ws".equalsIgnoreCase(scheme)) {
port = 80;
} else if ("wss".equalsIgnoreCase(scheme)) {
port = 443;
} else {
port = -1;
}
} else {
port = uri.getPort();
}
if (!"ws".equalsIgnoreCase(scheme) && !"wss".equalsIgnoreCase(scheme)) {
System.err.println("Only WS(S) is supported.");
return;
}
final boolean ssl = "wss".equalsIgnoreCase(scheme);
final SslContext sslCtx;
if (ssl) {
List<String> CipherList = new ArrayList<String>();
CipherList.add("TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256");
//CipherList.add("TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8");
CipherList.add("TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256");
File vidksFile = new File("C:\\Users\\LETTAU\\Videos\\key.pem");
File vidcertFile = new File("C:\\Users\\LETTAU\\Videos\\cert.pem");
//CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509");
sslCtx = SslContextBuilder.forClient()
.trustManager(InsecureTrustManagerFactory.INSTANCE)
.ciphers(CipherList)
.protocols("TLSv1.2")
.clientAuth(ClientAuth.REQUIRE)
.keyManager(vidcertFile, vidksFile)
.build();
}
else{
sslCtx = null;
}
EventLoopGroup group = new NioEventLoopGroup();
try{
WebSocketClientHandshaker wsHandShaker = WebSocketClientHandshakerFactory.newHandshaker(
uri, WebSocketVersion.V13, "ship", false, new DefaultHttpHeaders());
WebSocketClientProtocolHandler wsPrClHandler = new WebSocketClientProtocolHandler(wsHandShaker,true);
Bootstrap b = new Bootstrap();
b.group(group)
.channel(NioSocketChannel.class)
.handler(new ChannelInitializer<SocketChannel>() {
#Override
protected void initChannel(SocketChannel ch) {
ChannelPipeline p = ch.pipeline();
SslHandler ClientSslHandler = sslCtx.newHandler(ch.alloc(), host, port);
HttpClientCodec sourceCodec = new HttpClientCodec();
//New approach with HttpClientUpgradeHandler
//ShipClientProtocolHandler is a class which extends WebSocketClientProtocolHandler implements HttpClientUpgradeHandler.UpgradeCodec
HttpClientUpgradeHandler httpToWsUpgrader = new HttpClientUpgradeHandler(sourceCodec, new ShipClientProtocolHandler(wsHandShaker,true), 8096);
if (sslCtx != null) {
p.addLast(new LoggingHandler(LogLevel.INFO));
p.addLast(ClientSslHandler);
}
p.addLast(
new LoggingHandler(LogLevel.INFO),
sourceCodec,
new HttpObjectAggregator(8192),
//httpToWsUpgrader,
new LoggingHandler(LogLevel.INFO),
wsPrClHandler);
}
}
);
Channel ch = b.connect(uri.getHost(), port).sync().channel();
BufferedReader console = new BufferedReader(new InputStreamReader(System.in));
while (true) {
String msg = console.readLine();
if (msg == null) {
break;
} else if ("ping".equals(msg.toLowerCase())) {
WebSocketFrame frame = new PingWebSocketFrame(Unpooled.wrappedBuffer(new byte[] { 8, 1, 8, 1 }));
ch.writeAndFlush(frame);
} else if ("cmi".equals(msg.toLowerCase())) {
ByteBuf bBuf = Unpooled.buffer(16);
bBuf.writeByte(0);
bBuf.writeByte(0);
BinaryWebSocketFrame bytebuffer = new BinaryWebSocketFrame(bBuf);
ch.writeAndFlush(bytebuffer);
} else if ("cdp".equals(msg.toLowerCase())) {
ByteBuf bBuf = Unpooled.buffer(1000);
//bBuf.writeByte(0);
//bBuf.writeByte(1);
ByteBufUtil.writeUtf8(bBuf,"{\"connectionHello\":[{\"phase\":\"ready\"},{\"waiting\":60000}]}");
BinaryWebSocketFrame bytebuffer = new BinaryWebSocketFrame(bBuf);
ch.writeAndFlush(bytebuffer);
} else {
WebSocketFrame frame = new TextWebSocketFrame(msg);
ch.writeAndFlush(frame);
}
}
} finally {
group.shutdownGracefully();
}
}
Edit3: Some addtitional Information on why i think HttpClientCodec is the problem:
This is what a received package looks like:
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
|00000000| 88 1d 11 30 53 54 41 54 45 5f 57 41 49 54 5f 43 |...0STATE_WAIT_C|
|00000010| 4c 4f 53 45 5f 43 4f 4e 4e 45 43 54 49 4f 4e |LOSE_CONNECTION |
+--------+-------------------------------------------------+----------------+
This is what an outgoing package looks like:
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
|00000000| 82 9d 12 1e 8c 5a 12 1f f7 78 71 71 e2 34 77 7d |.....Z...xqq.4w}|
|00000010| f8 33 7d 70 c4 3f 7e 72 e3 78 28 3c fe 3f 73 7a |.3}p.?~r.x(<.?sz|
|00000020| f5 78 6f |.xo |
+--------+-------------------------------------------------+----------------+
If you want to upgrade to WebSockets you should use the WebSocketClientHandshaker.
Netty itself contains an example for it:
https://github.com/netty/netty/tree/netty-4.1.28.Final/example/src/main/java/io/netty/example/http/websocketx/client

Algorithm to divide up linked objects (like contiguous US states) by some metric?

Use case: consider the 50 US states on a map, with each state corresponding to some cost function (i.e., market size). Is there an algorithm (and ideal an R/Python package) that divides the 50 states into N groups such that the variation in the sum of the cost function in each group is minimized? The only constraint is that all states in each group are contiguous (i.e., they form one large mass / connected to each other)
There is a related question with a smaller search space that might possibly be more tractable, or at least easier to code for. Given N groups, pick N states and call them roots. One way to divide states into groups is to say that the group associated with a root is the set of states closer to this root than to any other root. To allow you to fine tune this a bit, have a constant K for each root, and divide up states according to the sum of the distance to the root and the constant K for that root.
Groups created in this way always consist of contiguous states, because if a state is in a group, all of the states on the shortest path from that state to the root of the group are also in that group. If one of them was not in that group, that state would have a shorter path to a different root, which means that the original state would also have a shorter path, via that intermediate state, to the different root, which is a contradiction.
This still isn't a tractable problem, but you call hill-climb on the constants to try and even out the allocations, and you can hill-climb on the choice of roots. If you hill-climb from multiple random starts, you can hope that you will get some decent answers, and you might hope to get an idea of how good your best answer seen so far is by looking at the other hilltops found - I would be optimistic if the best answer seen occurred multiple times.
Here follows some Java with comments trimmed due to limits on bodies
import java.util.ArrayList;
import java.util.Arrays;
import java.awt.BorderLayout;
import java.io.BufferedReader;
import java.util.Collections;
import java.awt.Color;
import java.awt.Container;
import java.awt.Dimension;
import java.io.File;
import java.io.FileReader;
import java.awt.Graphics;
import java.util.HashMap;
import java.util.HashSet;
import java.io.IOException;
import javax.swing.JPanel;
import javax.swing.JFrame;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.awt.Rectangle;
import javax.swing.SwingUtilities;
public class States
{
/** Info held for State */
private static class StateInfo
{
private final String m_name;
/** abbreviation of state - drawn on graph */
private final String m_abbreviation;
private final double m_latitudeDegrees;
private final double m_longitudeDegrees;
private int m_currentX;
private int m_currentY;
/** 0-up number allocated and offset in ArrayList */
private final int m_number;
/** link to neighbouring states */
private final ArrayList m_neighbours =
new ArrayList();
/** colour to print in. Modified when states are grouped */
private Color m_color = Color.BLACK;
/** weight when computing variance */
private double m_weight = 1.0;
public String toString()
{
StringBuilder sb = new StringBuilder();
sb.append(m_abbreviation + " color " + m_color);
return sb.toString();
}
public StateInfo(String name, String abbreviation,
double latitudeDegrees, double longitudeDegrees, int number)
{
m_name = name;
m_abbreviation = abbreviation;
m_latitudeDegrees = latitudeDegrees;
m_longitudeDegrees = longitudeDegrees;
m_number = number;
}
public double getLatitudeDegrees()
{
return m_latitudeDegrees;
}
public double getLongitudeDegrees()
{
return m_longitudeDegrees;
}
public String getAbbreviation()
{
return m_abbreviation;
}
public void addNeighbour(StateInfo si)
{
m_neighbours.add(si);
}
public List getNeighbours()
{
return Collections.unmodifiableList(m_neighbours);
}
public void setCurrentPos(int x, int y)
{
m_currentX = x;
m_currentY = y;
}
public int getX()
{
return m_currentX;
}
public int getY()
{
return m_currentY;
}
public int getNumber()
{
return m_number;
}
public Color getColor()
{
return m_color;
}
public void setColor(Color newValue)
{
m_color = newValue;
}
public double getWeight()
{
return m_weight;
}
public void setWeight(double newValue)
{
m_weight = newValue;
}
}
public static class StatePanel extends JPanel
{
#Override
public void paint(Graphics g)
{
Rectangle rect = getBounds();
double minLat = Double.MAX_VALUE;
double maxLat = -Double.MAX_VALUE;
double minLon = Double.MAX_VALUE;
double maxLon = -Double.MAX_VALUE;
for (StateInfo si: m_stateInfo)
{
double lat = si.getLatitudeDegrees();
double lon = si.getLongitudeDegrees();
if (lat maxLat)
{
maxLat = lat;
}
if (lon maxLon)
{
maxLon = lon;
}
}
int numStates = m_stateInfo.size();
double gapAdd = 1.0 / (double)(numStates + 1);
double gapFactor = (numStates - 1) / (double)(numStates + 1);
for (StateInfo si: m_stateInfo)
{
double x = (si.getLongitudeDegrees() - minLon) / (maxLon - minLon);
x = x * gapFactor + gapAdd;
double y = (maxLat - si.getLatitudeDegrees()) / (maxLat - minLat);
y = y * gapFactor + gapAdd;
int xpos = (int)Math.round(rect.x + x * rect.width);
int ypos = (int)Math.round(rect.y + y * rect.height);
g.setColor(si.getColor());
g.drawString(si.getAbbreviation(), xpos, ypos);
si.setCurrentPos(xpos, ypos);
}
g.setColor(Color.BLACK);
for (StateInfo si: m_stateInfo)
{
int xp = si.getX();
int yp = si.getY();
for (StateInfo ti: si.getNeighbours())
{
if (si.getNumber() rows = new ArrayList();
for (;;)
{
String line = br.readLine();
if (line == null)
{
break;
}
int len = line.length();
int l1 = len - 1;
ArrayList cells = new ArrayList();
StringBuilder currentCell = new StringBuilder();
for (int i = 0; i = l1)
{
throw new IllegalArgumentException("Unterminated quote in line " + line);
}
ch = line.charAt(++i);
if (ch != '"')
{
currentCell.append(ch);
continue;
}
if ((i >= l1) || (line.charAt(i + 1) != '"'))
{
break;
}
currentCell.append(ch);
i++;
}
}
if (len > 0)
{
cells.add(currentCell.toString());
}
String[] row = new String[cells.size()];
row = cells.toArray(row);
rows.add(row);
}
String[][] result = new String[rows.size()][];
result = rows.toArray(result);
return result;
}
finally
{
fr.close();
}
}
/** Read locations of states and links between them from file */
public void readLocations(String filename, String linkFile) throws Exception
{
// First two lines of file are
// # State,Latitude,Longitude,Abbreviation,Population
// Alabama,32.806671,-86.79113,AL,4.859
// Population is in millions. File is result of hand-editing together info from web
String[][] data = readCsv(new File(filename));
m_stateInfo = new ArrayList();
m_stateInfoByAbbreviation = new HashMap();
int num = 0;
for (String[] row: data)
{
if (row.length rootOffsetByColor = new HashMap();
for (int i = 0; i reducedWeights[i])
{
reducedWeights[i] = reduced;
}
}
if (secondBest totalByColor = new HashMap();
for (;;)
{
allocateViaDistance(m_roots, initialWeights, m_colors);
double varianceHere = getVariance(totalByColor);
double total = 0.0;
for (Double d: totalByColor.values())
{
total += d.doubleValue();
}
double target = total / m_roots.length;
minimumNecessary(m_roots, initialWeights, m_colors, decrease, increase);
boolean improved = false;
// Don't need to change weight[0] as we can regard everything else as
// compared to that as a standard
for (int i = 1; i totalByColor)
{
totalByColor.clear();
for (StateInfo si: m_stateInfo)
{
Color key = si.getColor();
Double d = new Double(0.0);
if (totalByColor.containsKey(key))
{
d = totalByColor.get(key);
}
totalByColor.put(key, new Double(d.doubleValue() + si.getWeight()));
}
int numUsed = totalByColor.size();
if (numUsed totalByColor = new HashMap();
return getVariance(totalByColor);
}
public StateInfo getState(String abbreviation)
{
return m_stateInfoByAbbreviation.get(abbreviation);
}
public void setColorsRoots(Color[] colors, StateInfo[] roots)
{
m_roots = roots;
m_colors = colors;
}
public StateInfo[] getRoots()
{
return m_roots;
}
public double climbRoots(double[] weights)
{
for (;;)
{
double current = getVariance();
boolean improved = false;
for (int i = 0; i chosen = new HashSet();
m_roots = new StateInfo[weights.length];
for (int j = 0; j m_stateInfo;
private Map m_stateInfoByAbbreviation;
private double[][] m_distances;
private StateInfo[] m_roots;
private Color[] m_colors;
}
/** compute all-pairs distances (Floyd - Warshall) */
public static void allPairs(double[][] distances)
{
int pass = 0;
for (boolean changed = true; changed;)
{
changed = false;
System.err.println("Pass " + (pass++));
for (int i = 0; i totalByColor = new HashMap();
var = sp.getVariance(totalByColor);
System.out.println("Multiple start Climbed to variance " + var);
System.out.println("Roots " + Arrays.toString(sp.getRoots()));
System.out.println("Totals " + totalByColor.values());
}
System.out.println("Allocated result is " + sp.getVariance());
}
catch (Exception ex)
{
System.err.println("Exception setting up: " + ex);
ex.printStackTrace();
return;
}
sp.setPreferredSize(new Dimension(400, 400));
JFrame w = new JFrame();
w.setTitle("States");
Container cp = w.getContentPane();
cp.setLayout(new BorderLayout());
cp.add(sp, BorderLayout.CENTER);
w.pack();
w.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
w.setVisible(true);
}
};
SwingUtilities.invokeAndWait(r);
}
}
Here's some example code of how you can generate all possible groups of states up to a reasonable number, trick is representing the ith state by two to the power of i. And a group of states is a 64 bit integer with bits corresponding to its states.
data = """AK WA
AL FL
AL GA
AL MS
AL TN
AR LA
AR MO
AR MS
AR OK
AR TN
AR TX
AZ CA
AZ CO
AZ NM
AZ NV
AZ UT
CA HI
CA NV
CA OR
CO KS
CO NE
CO NM
CO OK
CO UT
CO WY
CT MA
CT NY
CT RI
DC MD
DC VA
DE MD
DE NJ
DE PA
FL GA
GA NC
GA SC
GA TN
IA IL
IA MN
IA MO
IA NE
IA SD
IA WI
ID MT
ID NV
ID OR
ID UT
ID WA
ID WY
IL IN
IL KY
IL MO
IL WI
IN KY
IN MI
IN OH
KS MO
KS NE
KS OK
KY MO
KY OH
KY TN
KY VA
KY WV
LA MS
LA TX
MA NH
MA NY
MA RI
MA VT
MD PA
MD VA
MD WV
ME NH
MI OH
MI WI
MN ND
MN SD
MN WI
MO NE
MO OK
MO TN
MS TN
MT ND
MT SD
MT WY
NC SC
NC TN
NC VA
ND SD
NE SD
NE WY
NH VT
NJ NY
NJ PA
NM OK
NM TX
NM UT
NV OR
NV UT
NY PA
NY VT
OH PA
OH WV
OK TX
OR WA
PA WV
SD WY
TN VA
UT WY
VA WV""".splitlines()
edges = [line.split() for line in data]
states = sorted(set([node for pair in edges for node in pair]))
state_map = {}
groups = [set() for _ in range(52)]
neighbours = [[] for _ in range(52)]
for i, state in enumerate(states):
state_map[state] = i
state_map[i] = state
groups[1].add(1 << i)
for a, b in edges:
i, j = state_map[a], state_map[b]
neighbours[i].append(j)
neighbours[j].append(i)
for group_size in range(2, 20):
prev_groups = groups[group_size-1]
print "group size: %d, count: %d" % (group_size - 1, len(prev_groups))
current_groups = groups[group_size]
for group in prev_groups:
for i in range(51):
state = 1 << i
if state & group == 0:
for j in neighbours[i]:
if group & (1 << j) > 0:
current_groups.add(group | (1<<i))
group size: 1, count: 51
group size: 2, count: 111
group size: 3, count: 323
group size: 4, count: 1063
group size: 5, count: 3714
group size: 6, count: 13219
group size: 7, count: 46858
group size: 8, count: 163360
group size: 9, count: 556059
group size: 10, count: 1838076
group size: 11, count: 5873753
group size: 12, count: 18076855
group size: 13, count: 53409207

ThreadLocal and generics

I would like to create some thread local static arrays, using generics. The size of the array depends on the type. I'm trying to do something like:
type LeftoverPool private () =
static let instance = new ThreadLocal<'T[]>(fun () -> Array.zeroCreate (Vector<'T>.Count))
static member Instance = instance.Value
But that constrains T to int.
I've tried making the instance method generic, or the type, but can't get this to work.
From a type perspective, what you're trying to do at the moment really doesn't make sense.
You have a static property Instance but its type its completely ambiguous because it's impossible to resolve the meaning of 'T. In order to do this you need to propagate the type annotation up to the containing type, e.g.:
type LeftoverPool<'T> private () =
static let instance = new ThreadLocal<'T[]>(fun () -> Array.zeroCreate<'T> (Vector<'T>.Count))
static member Instance = instance.Value
Create a generic type, then its static members will be specific for the type. Resharper (for C#) will issue a warning, but here is what you really need.
Also, you need to benchmark ThreadStatic vs ThreadLocal as well, the later is effectively a pool of objects and it my have some lookup overheads compared to thread-static fields that just reside in a special memory region for each thread.
Update
I was going to test it for a while anyway, here is the code and output. ThreadStatic is faster.
[TestFixture]
public class BuffersTests {
public static class LocalBuffers<T> {
[ThreadStatic]
private static T[] _threadStatic;
private static ThreadLocal<T[]> _threadLocal = new ThreadLocal<T[]>(() => new T[10]);
public static T[] ThreadStatic => _threadStatic ?? (_threadStatic = new T[10]);
public static T[] ThreadLocal => _threadLocal.Value;
}
[Test, Ignore]
public void ThreadStaticVsThreadLocal() {
for (int r = 0; r < 10; r++) {
const int count = 100000000;
var sw = new Stopwatch();
sw.Restart();
var sum = 0L;
for (var i = 0; i < count; i++) {
var buffer = LocalBuffers<int>.ThreadStatic;
buffer[0] = 123;
sum += buffer[0] + buffer[1];
}
Assert.IsTrue(sum > 0);
sw.Stop();
Console.WriteLine($"ThreadStatic {sw.ElapsedMilliseconds}");
sw.Restart();
sum = 0L;
for (var i = 0; i < count; i++) {
var buffer = LocalBuffers<int>.ThreadLocal;
buffer[0] = 123;
sum += buffer[0] + buffer[1];
}
Assert.IsTrue(sum > 0);
sw.Stop();
Console.WriteLine($"ThreadLocal {sw.ElapsedMilliseconds}");
Console.WriteLine("---------------------");
}
}
}
ThreadStatic 1286
ThreadLocal 1860
---------------------
ThreadStatic 1312
ThreadLocal 1849
---------------------
ThreadStatic 1334
ThreadLocal 1933
---------------------
ThreadStatic 1390
ThreadLocal 2076
---------------------
ThreadStatic 1438
ThreadLocal 2088
---------------------
ThreadStatic 1295
ThreadLocal 2216
---------------------
ThreadStatic 1317
ThreadLocal 1972
---------------------
ThreadStatic 1380
ThreadLocal 1943
---------------------
ThreadStatic 1410
ThreadLocal 1970
---------------------

Resources