I'm trying to execute a rather trivial WebAssembly benchmark with Google's V8 engine (both in-browser using the current Version of Google Chrome (Version 83.0.4103.106, 64-bit) and via embedding V8 (Version 8.5.183) in a C++ program. All benchmarks are executed on macOS 10.14.6 with an Intel i7 8850H processor. No RAM swap has been used.
I am using the following C code as a benchmark. (Note that runtime is in the order of seconds on a current Intel Core i7)
static void init(int n, int path[1000][1000]) {
for (int i = 0; i < n; i++) {
for (int j = 0; j < n; j++) {
path[i][j] = i*j%7+1;
if ((i+j)%13 == 0 || (i+j)%7==0 || (i+j)%11 == 0) {
path[i][j] = 999;
}
}
}
}
static void kernel(int n, int path[1000][1000]) {
for (int k = 0; k < n; k++) {
for(int i = 0; i < n; i++) {
for (int j = 0; j < n; j++) {
path[i][j] = path[i][j] < path[i][k] + path[k][j] ? path[i][j] : path[i][k] + path[k][j];
}
}
}
}
int path[1000][1000];
int main(void) {
int n = 1000;
init(n, path);
kernel(n, path);
return 0;
}
This can be easily executed via https://wasdk.github.io/WasmFiddle/. The corresponding JS code measuring time in the most basic way is the following:
var wasmModule = new WebAssembly.Module(wasmCode);
var wasmInstance = new WebAssembly.Instance(wasmModule, wasmImports);
var a = new Date();
wasmInstance.exports.main();
var b = new Date();
log(b-a);
The result I'm getting in browser (e.g. in WasmFiddle or on a custom website) in Google Chrome is the following (for multiple consecutive executions) in milliseconds:
3687
1757
1837
1753
1726
1731
1774
1741
1771
1727
3549
1742
1731
1847
1734
1745
3515
1731
1772
Note the outliers performing at half the speed of the rest. How and why are there outliers with still such consistent performance? As much care as possible has been taken to ensure that no other processes are using up CPU time.
For the embedded version, the monolithic V8 library has been built from source using the following build config:
is_component_build = false
is_debug = false
target_cpu = "x64"
use_custom_libcxx = false
v8_monolithic = true
v8_use_external_startup_data = false
v8_enable_pointer_compression = false
The C++ code embedding the V8 library and executing the Wasm script (The Wasm code is the exact code produced by the WasmFiddle compiler):
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "include/libplatform/libplatform.h"
#include "include/v8.h"
int main(int argc, char* argv[]) {
// Initialize V8.
v8::V8::InitializeICUDefaultLocation(argv[0]);
v8::V8::InitializeExternalStartupData(argv[0]);
std::unique_ptr<v8::Platform> platform = v8::platform::NewDefaultPlatform();
v8::V8::InitializePlatform(platform.get());
v8::V8::Initialize();
// Create a new Isolate and make it the current one.
v8::Isolate::CreateParams create_params;
create_params.array_buffer_allocator = v8::ArrayBuffer::Allocator::NewDefaultAllocator();
v8::Isolate* isolate = v8::Isolate::New(create_params);
{
v8::Isolate::Scope isolate_scope(isolate);
// Create a stack-allocated handle scope.
v8::HandleScope handle_scope(isolate);
// Create a new context.
v8::Local<v8::Context> context = v8::Context::New(isolate);
v8::Context::Scope context_scope(context);
{
const char csource[] = R"(
let bytes = new Uint8Array([
0x0, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00, 0x01, 0x85, 0x80, 0x80, 0x80, 0x00, 0x01, 0x60,
0x00, 0x01, 0x7F, 0x03, 0x82, 0x80, 0x80, 0x80, 0x00, 0x01, 0x00, 0x04, 0x84, 0x80, 0x80, 0x80,
0x00, 0x01, 0x70, 0x00, 0x00, 0x05, 0x83, 0x80, 0x80, 0x80, 0x00, 0x01, 0x00, 0x3E, 0x06, 0x81,
0x80, 0x80, 0x80, 0x00, 0x00, 0x07, 0x91, 0x80, 0x80, 0x80, 0x00, 0x02, 0x06, 0x6D, 0x65, 0x6D,
0x6F, 0x72, 0x79, 0x02, 0x00, 0x04, 0x6D, 0x61, 0x69, 0x6E, 0x00, 0x00, 0x0A, 0x8F, 0x82, 0x80,
0x80, 0x00, 0x01, 0x89, 0x82, 0x80, 0x80, 0x00, 0x01, 0x08, 0x7F, 0x41, 0x00, 0x21, 0x02, 0x41,
0x10, 0x21, 0x05, 0x03, 0x40, 0x20, 0x05, 0x21, 0x07, 0x41, 0x00, 0x21, 0x04, 0x41, 0x00, 0x21,
0x03, 0x03, 0x40, 0x20, 0x07, 0x20, 0x04, 0x41, 0x07, 0x6F, 0x41, 0x01, 0x6A, 0x41, 0xE7, 0x07,
0x20, 0x02, 0x20, 0x03, 0x6A, 0x22, 0x00, 0x41, 0x07, 0x6F, 0x1B, 0x41, 0xE7, 0x07, 0x20, 0x00,
0x41, 0x0D, 0x6F, 0x1B, 0x41, 0xE7, 0x07, 0x20, 0x00, 0x41, 0x0B, 0x6F, 0x1B, 0x36, 0x02, 0x00,
0x20, 0x07, 0x41, 0x04, 0x6A, 0x21, 0x07, 0x20, 0x04, 0x20, 0x02, 0x6A, 0x21, 0x04, 0x20, 0x03,
0x41, 0x01, 0x6A, 0x22, 0x03, 0x41, 0xE8, 0x07, 0x47, 0x0D, 0x00, 0x0B, 0x20, 0x05, 0x41, 0xA0,
0x1F, 0x6A, 0x21, 0x05, 0x20, 0x02, 0x41, 0x01, 0x6A, 0x22, 0x02, 0x41, 0xE8, 0x07, 0x47, 0x0D,
0x00, 0x0B, 0x41, 0x00, 0x21, 0x06, 0x41, 0x10, 0x21, 0x05, 0x03, 0x40, 0x41, 0x10, 0x21, 0x00,
0x41, 0x00, 0x21, 0x01, 0x03, 0x40, 0x20, 0x01, 0x41, 0xA0, 0x1F, 0x6C, 0x20, 0x06, 0x41, 0x02,
0x74, 0x6A, 0x41, 0x10, 0x6A, 0x21, 0x02, 0x41, 0x00, 0x21, 0x07, 0x03, 0x40, 0x20, 0x00, 0x20,
0x07, 0x6A, 0x22, 0x04, 0x20, 0x04, 0x28, 0x02, 0x00, 0x22, 0x04, 0x20, 0x05, 0x20, 0x07, 0x6A,
0x28, 0x02, 0x00, 0x20, 0x02, 0x28, 0x02, 0x00, 0x6A, 0x22, 0x03, 0x20, 0x04, 0x20, 0x03, 0x48,
0x1B, 0x36, 0x02, 0x00, 0x20, 0x07, 0x41, 0x04, 0x6A, 0x22, 0x07, 0x41, 0xA0, 0x1F, 0x47, 0x0D,
0x00, 0x0B, 0x20, 0x00, 0x41, 0xA0, 0x1F, 0x6A, 0x21, 0x00, 0x20, 0x01, 0x41, 0x01, 0x6A, 0x22,
0x01, 0x41, 0xE8, 0x07, 0x47, 0x0D, 0x00, 0x0B, 0x20, 0x05, 0x41, 0xA0, 0x1F, 0x6A, 0x21, 0x05,
0x20, 0x06, 0x41, 0x01, 0x6A, 0x22, 0x06, 0x41, 0xE8, 0x07, 0x47, 0x0D, 0x00, 0x0B, 0x41, 0x00,
0x0B
]);
let module = new WebAssembly.Module(bytes);
let instance = new WebAssembly.Instance(module);
instance.exports.main();
)";
// Create a string containing the JavaScript source code.
v8::Local<v8::String> source = v8::String::NewFromUtf8Literal(isolate, csource);
// Compile the source code.
v8::Local<v8::Script> script = v8::Script::Compile(context, source).ToLocalChecked();
// Run the script to get the result.
v8::Local<v8::Value> result = script->Run(context).ToLocalChecked();
}
}
// Dispose the isolate and tear down V8.
isolate->Dispose();
v8::V8::Dispose();
v8::V8::ShutdownPlatform();
delete create_params.array_buffer_allocator;
return 0;
}
I compile it as follows:
g++ -I. -O2 -Iinclude samples/wasm.cc -o wasm -lv8_monolith -Lout.gn/x64.release.sample/obj/ -pthread -std=c++17
On execution with time ./wasm, I get execution times between 4.9s and 5.1s - almost triple that of in-Chrome/WasmFiddle execution! Did I miss anything? Maybe some optimization switches? This result is perfectly reproducible and I have even tested various different versions of the V8 library - still the same result.
Ah, the joys of microbenchmarking :-)
V8 has two compilers for Wasm: a non-optimizing baseline compiler that produces code really fast, and an optimizing compiler that takes quite a bit longer to produce code, but that code is typically about twice as fast. When a module is loaded, current versions first compile all functions with the baseline compiler. Once that's done, execution can start, and optimized compilation jobs are scheduled to run in the background. When an optimized compilation job is complete, the respective function's code is swapped, and the next invocation of the function will use it. (The details here will very likely change in the future, but the general principle will remain.) That way, typical applications get both good startup latency, and good peak performance.
But, as with any heuristic or strategy, you can craft a case where it gets it wrong...
In your benchmark, each function is called only once. In the fast cases, optimizing kernel finishes before init returns. In the slow cases, kernel is called before its optimized compilation job is done, so its baseline version runs. Apparently when embedding V8 directly, you reliably get the latter scenario, whereas when running via WasmFiddle in Chrome, you get the former most of the time, but not always.
I can't explain why your custom embedding runs are even slower than the slow case in Chrome; I'm not seeing that on my machine (OTOH, in Chrome, I'm seeing an even bigger delta: about 1100ms for a fast run and 4400ms for a slow run); however I used the d8 shell instead of compiling my own embedding. One thing that's different is that when measuring with time on the command line, you include process startup and initialization, which the Date.now() calls around main() don't include. But that should only account for 10-50 milliseconds or so, not for a 3.6s → 5.0s difference.
While this situation might look quite unfortunate for your microbenchmark, it is generally working as intended, i.e. not a bug, and hence unlikely to change on V8's side. There are several things you can do to make the benchmark more reflective of real-world behavior (assuming this one doesn't exactly represent some real application you have):
execute functions multiple times; you'll see that the first run will be slower (or, depending on function size and module size and number of available CPU cores and scheduling luck, the first few runs)
wait a bit before calling the hottest functions, e.g. by doing
var wasmModule = new WebAssembly.Module(wasmCode);
var wasmInstance = new WebAssembly.Instance(wasmModule, wasmImports);
window.setTimeout(() => {
var a = Date.now();
wasmInstance.exports.main();
var b = Date.now();
log(b-a);
}, 10);
In my tests with d8 I've found that even a silly busy-wait did the trick:
let wait = Date.now() + 10;
while (Date.now() < wait) {}
instance.exports.main();
generally make the benchmark bigger and more complex: have and execute more different functions, don't just spend 99% of the time in a single line.
(FWIW, the earliest V8 versions that supported WebAssembly had no tiering, only optimized compilation. So modules always had to wait for that to finish. It was not a good user experience; for large modules the wait time could be tens of seconds. Having a baseline compiler is quite clearly the better solution overall, even if it comes at the cost of not having maximum performance available immediately. Looking good on artificial one-liners is not what matters in practice; providing a good user experience for large real-world applications matters.)
I'm trying to rotate a h264 video 90 degrees anticlockwise. The syntax for the Display Orientation Supplementary EnhancementInformation (SEI) is given as:
Which I first tried encoding as follows:
val prefix = byteArrayOf(0, 0, 0, 1)
val nalHeader = byteArrayOf(6) // 0 forbidden_zero_bit = 0, nal_ref_idc = 0, nal_unit_type = 6
val display = byteArrayOf(47 /* Display orientation type*/, 3 /*payload size*/)
val displayOrientationCancelFlag = "0" // u(1); Rotation information follows
val horFlip = "0" // hor_flip; u(1); Do not flip horizontally
val verFlip = "0" // ver_flip; u(1); Do not flip vertically
val anticlockwiseRotation = "0100000000000000" // u(16); value / 2^16 -> 90 degrees
val displayOrientationRepetitionPeriod = "010" // ue(v); Persistent till next video sequence
val displayOrientationExtensionFlag = "0" // u(1); No other value is permitted by the spec atm
val byteAlignment = "1"
The above is my kotlin code for generating the SEI. This is a static variable so I went for human readable version.
Dropping the Annex B start codes, the hex SEI is
06 2f 03 08 00 09
ffmpeg complained about this however, saying that my payload size was specified as 3 bytes (24 bits) but it only read 23 bits before reading the NAL stop bit. To fix this I padded my SEI payload by one zero bit to allow ffmpeg parse it successfully, and added the stop bit again with more byte alignment bits:
val byteAlignment = "010000000"
This in hex is
06 2f 03 08 00 08 80
When I add this before the first IDR NAL unit in my h264 rbsp, ffmpeg will accept it to convert to whatever format I ask, jpeg or mp4. The output is not rotated however. Playing it with ffplay also does not rotate it. I'm note quite sure what I'm doing wrong. ffprobe yields the following output:
[NULL # 0x7fca03808a00] Opening 'sei2.264' for reading
[file # 0x7fca01c289c0] Setting default whitelist 'file,crypto'
Probing h264 score:51 size:1502
Probing mp3 score:1 size:1502
[h264 # 0x7fca03808a00] Format h264 probed with size=2048 and score=51
[h264 # 0x7fca03808a00] Before avformat_find_stream_info() pos: 0 bytes read:1502 seeks:0 nb_streams:1
[AVBSFContext # 0x7fca02808680] nal_unit_type: 7(SPS), nal_ref_idc: 3
[AVBSFContext # 0x7fca02808680] nal_unit_type: 8(PPS), nal_ref_idc: 3
[AVBSFContext # 0x7fca02808680] nal_unit_type: 6(SEI), nal_ref_idc: 0
[AVBSFContext # 0x7fca02808680] nal_unit_type: 5(IDR), nal_ref_idc: 3
[h264 # 0x7fca0381d400] nal_unit_type: 7(SPS), nal_ref_idc: 3
[h264 # 0x7fca0381d400] nal_unit_type: 8(PPS), nal_ref_idc: 3
[h264 # 0x7fca0381d400] nal_unit_type: 6(SEI), nal_ref_idc: 0
[h264 # 0x7fca0381d400] nal_unit_type: 5(IDR), nal_ref_idc: 3
[h264 # 0x7fca0381d400] Format yuv420p chosen by get_format().
[h264 # 0x7fca0381d400] Reinit context to 176x144, pix_fmt: yuv420p
[h264 # 0x7fca03808a00] decoding for stream 0 failed
[h264 # 0x7fca03808a00] stream 0: start_time: -7686143364045.646 duration: -7686143364045.646
[h264 # 0x7fca03808a00] format: start_time: -9223372036854.775 duration: -9223372036854.775 bitrate=0 kb/s
[h264 # 0x7fca03808a00] After avformat_find_stream_info() pos: 1502 bytes read:1502 seeks:0 frames:1
Input #0, h264, from 'sei2.264':
Duration: N/A, bitrate: N/A
Stream #0:0, 1, 1/1200000: Video: h264 (Constrained Baseline), 1 reference frame, yuv420p(progressive, left), 176x144, 0/1, 25 tbr, 1200k tbn, 50 tbc
[h264 # 0x7fca0380dc00] nal_unit_type: 7(SPS), nal_ref_idc: 3
[h264 # 0x7fca0380dc00] nal_unit_type: 8(PPS), nal_ref_idc: 3
[AVIOContext # 0x7fca02801b00] Statistics: 1502 bytes read, 0 seeks
My full RBSP with SPS, PPS, SEI and IDR follow:
unsigned char rbsp[1502] = {
// Offset 0x00000000 to 0x00001501
0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x29, 0x8d, 0x68, 0x2c, 0x4e,
0x80, 0x78, 0x44, 0x23, 0x50, 0x00, 0x00, 0x00, 0x01, 0x68, 0xce, 0x01,
0xa8, 0x35, 0xc8, 0x00, 0x00, 0x00, 0x01, 0x06, 0x2f, 0x03, 0x08, 0x00,
0x08, 0x80, 0x00, 0x00, 0x00, 0x01, 0x65, 0xb8, 0x00, 0x04, 0x05, 0x9f,
0xff, 0xff, 0x04, 0x51, 0x40, 0x00, 0x41, 0x63, 0xc7, 0x31, 0xcf, 0xff,
0xff, 0x27, 0xff, 0xfe, 0x4f, 0xff, 0xfc, 0x9f, 0xff, 0xf9, 0x3f, 0xff,
0xf2, 0x7f, 0xff, 0xe4, 0xff, 0xff, 0xc9, 0xff, 0xff, 0x93, 0xff, 0xff,
0x27, 0xff, 0xfe, 0x4f, 0xff, 0xfc, 0x9f, 0xff, 0xf9, 0x3f, 0xff, 0xf2,
0x7f, 0xff, 0xe4, 0xff, 0xfc, 0x43, 0x05, 0xd0, 0x03, 0x2b, 0x2d, 0x11,
0x03, 0x16, 0x84, 0xb4, 0x5e, 0xc2, 0x00, 0x03, 0xa0, 0x20, 0xc7, 0x42,
0xb6, 0xb8, 0x00, 0x57, 0x66, 0x66, 0x09, 0xdc, 0xc1, 0x92, 0x50, 0x86,
0x38, 0x04, 0x8f, 0x6d, 0x83, 0xff, 0xff, 0x41, 0x5e, 0xef, 0xbb, 0xff,
0xff, 0xd0, 0x57, 0xbb, 0xee, 0xff, 0xff, 0xf4, 0x15, 0xee, 0xfb, 0xbf,
0xff, 0xfd, 0x05, 0x7b, 0xbe, 0xef, 0xff, 0xff, 0x41, 0x5e, 0xef, 0xbb,
0xff, 0xff, 0xd0, 0x57, 0xbb, 0xc0, 0x25, 0x55, 0x9a, 0xdf, 0x9f, 0x22,
0x63, 0xff, 0xff, 0x27, 0xff, 0xfe, 0x4f, 0xff, 0xfc, 0x9f, 0xff, 0xf9,
0x3f, 0xff, 0xec, 0x86, 0xfc, 0x7d, 0x2d, 0x2f, 0xf8, 0x07, 0xfa, 0xf8,
0x00, 0x37, 0x66, 0x35, 0x11, 0x29, 0x61, 0x8b, 0x30, 0x53, 0x41, 0x3c,
0xb3, 0xee, 0x59, 0xa7, 0xb7, 0xc2, 0x00, 0x03, 0x00, 0x07, 0x05, 0x86,
0x02, 0x10, 0x88, 0x38, 0x00, 0x53, 0x68, 0x93, 0x1b, 0xf4, 0x22, 0x3b,
0x90, 0x00, 0x08, 0x30, 0xd9, 0x64, 0x00, 0x02, 0x0c, 0x36, 0xe6, 0x10,
0xef, 0xa0, 0xf6, 0x52, 0xc3, 0xd9, 0x5c, 0xf8, 0x1f, 0xff, 0xb0, 0xdf,
0x80, 0x06, 0xc6, 0xaa, 0x46, 0x19, 0xd8, 0x55, 0x96, 0x5f, 0xfb, 0x38,
0xc2, 0xf8, 0x40, 0x00, 0x64, 0x0a, 0x0a, 0x30, 0x11, 0x20, 0xd4, 0x00,
0x8c, 0x99, 0xb0, 0x6c, 0xd9, 0x4a, 0x10, 0xde, 0x0c, 0x80, 0x00, 0x83,
0x8c, 0xb9, 0x88, 0xed, 0x77, 0x3c, 0x85, 0x1d, 0xcd, 0x2f, 0xff, 0xc7,
0x9c, 0x3b, 0xc0, 0xe8, 0x22, 0xb2, 0xbe, 0x04, 0xdf, 0xdf, 0xfe, 0x42,
0x80, 0x0b, 0x83, 0x88, 0xa5, 0x81, 0xc0, 0x02, 0x05, 0x08, 0x96, 0x00,
0x43, 0x64, 0x44, 0x09, 0xdf, 0xa1, 0x03, 0x88, 0xbe, 0x0d, 0x9c, 0xb1,
0x23, 0x3a, 0xe7, 0xff, 0xfd, 0x93, 0xff, 0xff, 0x27, 0xe2, 0x1f, 0xf8,
0x2c, 0x80, 0xf2, 0x10, 0xf2, 0xdf, 0x08, 0x00, 0x44, 0x00, 0x18, 0x0e,
0x81, 0x20, 0x02, 0x24, 0x0e, 0x00, 0x10, 0x20, 0x44, 0xbf, 0x06, 0xa7,
0x2f, 0xfc, 0xbf, 0x0c, 0x01, 0x02, 0xf8, 0xc4, 0xcf, 0x07, 0xc0, 0xe0,
0x00, 0x80, 0x08, 0x0f, 0x65, 0x84, 0x04, 0x0a, 0x8a, 0x03, 0x83, 0x44,
0xc0, 0x02, 0x92, 0x22, 0x20, 0x6c, 0xd9, 0x08, 0x1c, 0xfe, 0x03, 0x80,
0x8c, 0x5c, 0xb2, 0x00, 0x46, 0x2e, 0x5e, 0x04, 0x6c, 0xbb, 0x80, 0x71,
0x08, 0xf2, 0xc8, 0x10, 0x8f, 0x2c, 0x5a, 0xf2, 0x74, 0x0d, 0xb7, 0xf8,
0xd3, 0xed, 0x6d, 0x7f, 0xfc, 0x38, 0x03, 0x0e, 0xf8, 0x00, 0xee, 0x12,
0x89, 0xef, 0x22, 0x09, 0x97, 0xd3, 0xdf, 0x30, 0xc1, 0x48, 0x01, 0x80,
0x06, 0x9b, 0x80, 0x05, 0x64, 0xcd, 0x98, 0x9b, 0x29, 0x48, 0x74, 0x22,
0x32, 0x4b, 0xe0, 0x0f, 0x06, 0xd8, 0x61, 0x77, 0xb8, 0x70, 0x11, 0x8b,
0x97, 0xff, 0xc1, 0x30, 0x06, 0x1d, 0xf0, 0x00, 0xdc, 0xa1, 0x16, 0x88,
0xd5, 0xcc, 0x49, 0x17, 0xd9, 0xe8, 0x36, 0x61, 0x05, 0x55, 0x80, 0xe0,
0x0c, 0x37, 0x00, 0x21, 0xb4, 0x4a, 0x1b, 0xf4, 0x25, 0x8a, 0x40, 0x08,
0xc5, 0xcb, 0xe0, 0x70, 0x64, 0xa2, 0x13, 0xbd, 0xc3, 0x88, 0x47, 0x97,
0xf9, 0xf8, 0x07, 0xb0, 0x5f, 0x8e, 0xcc, 0xe0, 0xf8, 0x00, 0xac, 0x2f,
0x8e, 0xe6, 0x11, 0x2f, 0x51, 0xea, 0xf0, 0x40, 0xbb, 0x30, 0x4b, 0x4b,
0x0e, 0x02, 0x31, 0x72, 0xfc, 0x38, 0x84, 0x79, 0x7f, 0xe1, 0xff, 0xb0,
0x59, 0xea, 0xcf, 0xdc, 0x66, 0x12, 0xc3, 0xd5, 0xf2, 0xbd, 0x3c, 0x76,
0x02, 0x47, 0xae, 0x3d, 0xff, 0xf0, 0x24, 0x5c, 0xb3, 0xd2, 0x26, 0x2b,
0x9f, 0xf1, 0xfe, 0xc1, 0x6c, 0xf4, 0x1e, 0xe1, 0x00, 0x00, 0x80, 0x30,
0x03, 0x07, 0x97, 0x08, 0x4c, 0x85, 0x81, 0xc0, 0x02, 0x04, 0x08, 0x97,
0xe0, 0xd4, 0xe5, 0xff, 0xff, 0xf6, 0x16, 0xdb, 0xee, 0xfd, 0x1f, 0x6b,
0x6b, 0x6b, 0x6b, 0x6b, 0xc3, 0xff, 0xf6, 0x08, 0xbe, 0x0b, 0x44, 0xaf,
0xf9, 0xff, 0xb0, 0x59, 0xe0, 0x0d, 0x5f, 0x6a, 0x38, 0xde, 0x82, 0x01,
0x20, 0xc4, 0x80, 0x78, 0x00, 0x56, 0x46, 0x66, 0x84, 0xd9, 0x4a, 0xac,
0xd2, 0x00, 0x02, 0x05, 0x08, 0xb9, 0x04, 0xdf, 0xf6, 0x95, 0x9d, 0xce,
0x01, 0x82, 0xfc, 0x34, 0x3b, 0xee, 0x47, 0xc0, 0x4f, 0xaf, 0xad, 0x6f,
0x5e, 0x10, 0x17, 0x1d, 0x82, 0xc6, 0x35, 0xa0, 0x01, 0x4d, 0x91, 0x13,
0x1b, 0xf4, 0x26, 0x57, 0xc8, 0x00, 0x04, 0x08, 0x13, 0x71, 0xc6, 0x81,
0x23, 0x5b, 0xd6, 0x51, 0x2b, 0x9f, 0xff, 0xf6, 0x1f, 0xef, 0x3b, 0xcf,
0x53, 0xd7, 0x5d, 0x75, 0xd7, 0x4f, 0xff, 0xff, 0xc1, 0x07, 0x01, 0xc0,
0x21, 0xce, 0x2e, 0x5b, 0xff, 0xff, 0xb0, 0x43, 0xe0, 0x03, 0x6d, 0x10,
0xe1, 0x0a, 0x27, 0xa4, 0xdd, 0xa1, 0xce, 0x03, 0xda, 0x72, 0xdf, 0xfe,
0x29, 0x6a, 0xa5, 0xea, 0xa5, 0x80, 0x06, 0x0b, 0x68, 0x8a, 0x66, 0x62,
0xdf, 0xfc, 0x7e, 0xc1, 0x07, 0x01, 0xe6, 0x3a, 0x88, 0x7f, 0xd0, 0xe8,
0x00, 0xfc, 0x02, 0x67, 0xb1, 0xe0, 0x73, 0xd2, 0x2e, 0x03, 0xf8, 0x00,
0xc4, 0x00, 0x6f, 0x9b, 0x15, 0x6b, 0x2d, 0xcf, 0x07, 0xc0, 0x02, 0x09,
0x6a, 0x32, 0xd8, 0xba, 0x3f, 0x0c, 0x28, 0x20, 0x28, 0x20, 0x20, 0x42,
0x00, 0x83, 0xe0, 0x70, 0x23, 0x15, 0x2c, 0x01, 0x03, 0x2a, 0x63, 0xcc,
0x42, 0x99, 0x02, 0x19, 0xee, 0x01, 0xc8, 0x46, 0x96, 0x00, 0x47, 0x6c,
0xd8, 0x04, 0xaa, 0x41, 0xbc, 0x41, 0x05, 0x32, 0x00, 0x08, 0x20, 0xab,
0x9f, 0xff, 0xf4, 0x0a, 0xb4, 0xab, 0x8d, 0x3e, 0x09, 0x25, 0xff, 0xff,
0xe8, 0x9f, 0xff, 0xf4, 0x4f, 0xff, 0xfa, 0x27, 0xff, 0xfd, 0x07, 0x30,
0x04, 0x2d, 0x20, 0x47, 0x38, 0x95, 0xab, 0xfc, 0x00, 0x6b, 0x19, 0x05,
0xce, 0x53, 0x10, 0xa6, 0xed, 0x08, 0x12, 0xc0, 0x90, 0x00, 0x26, 0x1a,
0x00, 0x21, 0xfb, 0x90, 0xfc, 0x84, 0x24, 0x10, 0xcd, 0x72, 0x00, 0x14,
0xd2, 0x35, 0x91, 0x86, 0x37, 0x5e, 0x9b, 0x20, 0x00, 0x41, 0x05, 0xdc,
0xff, 0xe0, 0x18, 0x57, 0xe0, 0x02, 0x98, 0xf7, 0x33, 0x98, 0x55, 0x3d,
0x47, 0xab, 0xdf, 0xe8, 0x30, 0x01, 0xab, 0x58, 0x48, 0x53, 0xb1, 0x8a,
0x7a, 0xbd, 0xd5, 0x8f, 0x87, 0xe0, 0x01, 0x0c, 0xa4, 0x12, 0xad, 0xcc,
0x29, 0x1a, 0xd0, 0x83, 0xd9, 0xc2, 0x9a, 0xf4, 0xd0, 0x38, 0x08, 0xa4,
0xcb, 0xf2, 0xfc, 0xb0, 0xe0, 0x9c, 0x79, 0x7f, 0xff, 0xfe, 0x10, 0x61,
0xef, 0xc0, 0x01, 0x93, 0x46, 0x83, 0x76, 0xea, 0x41, 0x8d, 0xf7, 0x4b,
0x14, 0x1c, 0x00, 0x04, 0x10, 0x20, 0x18, 0x00, 0x08, 0x14, 0xf0, 0x91,
0xcd, 0xb8, 0x08, 0xd4, 0xb7, 0xcf, 0x7f, 0xff, 0xd0, 0x43, 0xdf, 0xd0,
0x81, 0x83, 0xca, 0xc0, 0x05, 0x34, 0x63, 0x27, 0x15, 0x4d, 0x52, 0x0d,
0xda, 0x5e, 0x30, 0x5e, 0x05, 0x2a, 0x0d, 0xbb, 0x90, 0x22, 0x03, 0x7a,
0x68, 0x1d, 0x21, 0xdc, 0xd4, 0xc8, 0x91, 0x4d, 0xf3, 0xcf, 0xff, 0xfa,
0x05, 0x9c, 0x53, 0xfa, 0x45, 0xff, 0xe3, 0x4f, 0xff, 0xfe, 0x82, 0xdb,
0x7d, 0xdf, 0x1c, 0x00, 0x0d, 0xc7, 0x00, 0x01, 0x01, 0x5f, 0xff, 0xfd,
0x06, 0xae, 0xf9, 0x7d, 0x31, 0xf1, 0x08, 0xe2, 0x00, 0x00, 0x80, 0x30,
0x03, 0xea, 0x2e, 0x30, 0x92, 0x6b, 0xc8, 0x70, 0x98, 0x6b, 0x90, 0xe1,
0x30, 0xd7, 0x3f, 0xf0, 0xc0, 0x34, 0x0b, 0x60, 0x00, 0x99, 0x90, 0xc8,
0xe1, 0x98, 0xdc, 0x41, 0x23, 0xcc, 0x71, 0xf0, 0x40, 0xc6, 0x17, 0xc2,
0x80, 0x07, 0xc0, 0x81, 0x40, 0x00, 0xf8, 0x09, 0xc0, 0x27, 0xb1, 0x37,
0x72, 0x02, 0xa7, 0xd5, 0xe2, 0xc0, 0x80, 0xa6, 0x78, 0x04, 0xe9, 0x89,
0x95, 0xc8, 0x0e, 0xbf, 0x57, 0x8b, 0x00, 0x80, 0x53, 0x3f, 0xf5, 0xf0,
0xd8, 0x76, 0x00, 0x09, 0xe4, 0x38, 0xcc, 0x7e, 0x90, 0x66, 0x90, 0xb5,
0x86, 0x08, 0x50, 0x00, 0x6c, 0x99, 0xb4, 0x44, 0x8c, 0x6a, 0x89, 0xc4,
0xf8, 0x27, 0xf8, 0x41, 0x1f, 0xdc, 0x10, 0x47, 0xf7, 0x60, 0x04, 0x9d,
0x0f, 0x31, 0x5c, 0x9e, 0xaf, 0x0e, 0x09, 0x89, 0x81, 0x0d, 0x34, 0xdb,
0x56, 0x00, 0x48, 0xd4, 0x34, 0x62, 0xb9, 0xbd, 0x5e, 0x1c, 0x05, 0xd9,
0x81, 0x0d, 0x34, 0xdb, 0x57, 0xfc, 0x03, 0x0e, 0x81, 0x54, 0x20, 0x04,
0x86, 0x28, 0x40, 0x09, 0x0c, 0x5f, 0xc0, 0x15, 0x24, 0x63, 0x05, 0x38,
0xc3, 0x6d, 0x5f, 0x58, 0x02, 0xa1, 0x67, 0x70, 0x53, 0x0c, 0x36, 0xd5,
0xff, 0xf0, 0x0d, 0x87, 0x70, 0x00, 0x4c, 0xc8, 0x64, 0x70, 0xcc, 0x6e,
0x20, 0x91, 0xe6, 0x38, 0xfc, 0x61, 0x70, 0x40, 0xfc, 0x20, 0x08, 0x75,
0x94, 0x10, 0x04, 0x15, 0x66, 0x61, 0x35, 0xc3, 0xec, 0x47, 0x2f, 0xab,
0xc5, 0x81, 0x01, 0x4c, 0xe1, 0x32, 0x70, 0xd5, 0x88, 0xe7, 0xf5, 0x78,
0xb0, 0x08, 0x05, 0x33, 0xff, 0x05, 0xfd, 0x82, 0xff, 0xaf, 0x80, 0x02,
0x79, 0x0e, 0x33, 0x1f, 0xa4, 0x19, 0xa4, 0x2d, 0x70, 0x42, 0x08, 0x01,
0x3c, 0x06, 0x08, 0x01, 0x38, 0x25, 0x43, 0x82, 0x61, 0x6e, 0x00, 0x12,
0xf0, 0x9b, 0x39, 0x41, 0x51, 0xea, 0xf1, 0x00, 0x98, 0x5b, 0x90, 0xe4,
0x64, 0xb8, 0x00, 0x4a, 0xd0, 0x99, 0x1c, 0xa0, 0x55, 0x7a, 0xbc, 0x41,
0x19, 0x2e, 0x7f, 0xff, 0xd0, 0x63, 0x2e, 0xa6, 0x3c, 0x00, 0x19, 0xb4,
0x49, 0x8d, 0xfa, 0x11, 0x1d, 0xd2, 0x33, 0xe8, 0xef, 0xff, 0xfe, 0xfc,
0x07, 0x00, 0x10, 0x41, 0xaa, 0x4b, 0x7e, 0x00, 0x0d, 0xd3, 0x26, 0xc4,
0xf3, 0x15, 0x1d, 0x13, 0x9f, 0x04, 0x2c, 0x0e, 0x00, 0xc1, 0x02, 0x40,
0x07, 0x81, 0xb4, 0x70, 0x75, 0x43, 0xf0, 0x3c, 0xc5, 0x70, 0x8f, 0xff,
0xff, 0xb0, 0xc7, 0x80, 0x03, 0x36, 0x89, 0x31, 0xbf, 0xc4, 0x47, 0x7d,
0x04, 0xfb, 0xf8, 0x07, 0xff, 0xb0, 0x45, 0xcb, 0x9e, 0xc2, 0x05, 0xd9,
0x82, 0x00, 0x11, 0x58, 0xaf, 0xff, 0xfb, 0x08, 0xdf, 0xff, 0xff, 0x61,
0x1b, 0x7f, 0xff, 0xec, 0x15, 0x76, 0xff, 0xff, 0xdd, 0xff, 0xff, 0xec,
0x15, 0x44, 0x7b, 0xff, 0xcd, 0x7f, 0xff, 0xec, 0x23, 0x77, 0xff, 0xfe,
0xc8, 0x9f
}
The payloadSize is only the the size of the sei_message() The stop bit is not included in this size. So there must be a 0x80 at the end of the SEI,
I have an input byte array:
60614e680e705d0fefcf7ac8102c4452
ecb0c85768f2f2dc52415c43a36712f0
31c9037dafd31f01ecb0c85768f2f2dc
7b00be7e5a15fee1e78c63c58c2c6861
fef9a1c4130a354c846448512e6a97ce
4a9005690d1e3808f065c957538e1bac
87e7228322ab39a6900146786840dc0b
c536ad6afb6e4e3267fb045dd9c7e670
f1c2d2ac1fcc71ad06b7b194de4031f4
046744610aafa7b92fd3f392c3a5eeb1
474ffa60c0587e68ecb0c85768f2f2dc
2a88827461b41c99b2539b6bfdcd4325
be3ced59be7b594addb3366e076f6e47
0cc41df1eb3a8d93c99eb7bdad5a474c
33659653762910d0ecb0c85768f2f2dc
ecb0c85768f2f2dc82e715e7952a79c4
660074ccc50741cab5eabb873ae706b4
c8b008128df0af80fece91741fc5f641
1145aab35ac9f6e0f8a937baed012d00
c3be705a5e8c3440ddc1cd4e0051cccc
The algorithm by which they were encrypted XTEA ECB. I try to use the library for decryption “golang.org/x/crypto/xtea”.
package main
import (
"golang.org/x/crypto/xtea"
"fmt"
)
func main() {
// encripte data
test :=[]byte{
0x60, 0x61, 0x4e, 0x68, 0x0e, 0x70, 0x5d, 0x0f, 0xef, 0xcf, 0x7a, 0xc8, 0x10, 0x2c, 0x44, 0x52,
0xec, 0xb0, 0xc8, 0x57, 0x68, 0xf2, 0xf2, 0xdc, 0x52, 0x41, 0x5c, 0x43, 0xa3, 0x67, 0x12, 0xf0,
0x31, 0xc9, 0x03, 0x7d, 0xaf, 0xd3, 0x1f, 0x01, 0xec, 0xb0, 0xc8, 0x57, 0x68, 0xf2, 0xf2, 0xdc,
0x7b, 0x00, 0xbe, 0x7e, 0x5a, 0x15, 0xfe, 0xe1, 0xe7, 0x8c, 0x63, 0xc5, 0x8c, 0x2c, 0x68, 0x61,
0xfe, 0xf9, 0xa1, 0xc4, 0x13, 0x0a, 0x35, 0x4c, 0x84, 0x64, 0x48, 0x51, 0x2e, 0x6a, 0x97, 0xce,
0x4a, 0x90, 0x05, 0x69, 0x0d, 0x1e, 0x38, 0x08, 0xf0, 0x65, 0xc9, 0x57, 0x53, 0x8e, 0x1b, 0xac,
0x87, 0xe7, 0x22, 0x83, 0x22, 0xab, 0x39, 0xa6, 0x90, 0x01, 0x46, 0x78, 0x68, 0x40, 0xdc, 0x0b,
0xc5, 0x36, 0xad, 0x6a, 0xfb, 0x6e, 0x4e, 0x32, 0x67, 0xfb, 0x04, 0x5d, 0xd9, 0xc7, 0xe6, 0x70,
0xf1, 0xc2, 0xd2, 0xac, 0x1f, 0xcc, 0x71, 0xad, 0x06, 0xb7, 0xb1, 0x94, 0xde, 0x40, 0x31, 0xf4,
0x04, 0x67, 0x44, 0x61, 0x0a, 0xaf, 0xa7, 0xb9, 0x2f, 0xd3, 0xf3, 0x92, 0xc3, 0xa5, 0xee, 0xb1,
0x47, 0x4f, 0xfa, 0x60, 0xc0, 0x58, 0x7e, 0x68, 0xec, 0xb0, 0xc8, 0x57, 0x68, 0xf2, 0xf2, 0xdc,
0x2a, 0x88, 0x82, 0x74, 0x61, 0xb4, 0x1c, 0x99, 0xb2, 0x53, 0x9b, 0x6b, 0xfd, 0xcd, 0x43, 0x25,
0xbe, 0x3c, 0xed, 0x59, 0xbe, 0x7b, 0x59, 0x4a, 0xdd, 0xb3, 0x36, 0x6e, 0x07, 0x6f, 0x6e, 0x47,
0x0c, 0xc4, 0x1d, 0xf1, 0xeb, 0x3a, 0x8d, 0x93, 0xc9, 0x9e, 0xb7, 0xbd, 0xad, 0x5a, 0x47, 0x4c,
0x33, 0x65, 0x96, 0x53, 0x76, 0x29, 0x10, 0xd0, 0xec, 0xb0, 0xc8, 0x57, 0x68, 0xf2, 0xf2, 0xdc,
0xec, 0xb0, 0xc8, 0x57, 0x68, 0xf2, 0xf2, 0xdc, 0x82, 0xe7, 0x15, 0xe7, 0x95, 0x2a, 0x79, 0xc4,
0x66, 0x00, 0x74, 0xcc, 0xc5, 0x07, 0x41, 0xca, 0xb5, 0xea, 0xbb, 0x87, 0x3a, 0xe7, 0x06, 0xb4,
0xc8, 0xb0, 0x08, 0x12, 0x8d, 0xf0, 0xaf, 0x80, 0xfe, 0xce, 0x91, 0x74, 0x1f, 0xc5, 0xf6, 0x41,
0x11, 0x45, 0xaa, 0xb3, 0x5a, 0xc9, 0xf6, 0xe0, 0xf8, 0xa9, 0x37, 0xba, 0xed, 0x01, 0x2d, 0x00,
0xc3, 0xbe, 0x70, 0x5a, 0x5e, 0x8c, 0x34, 0x40, 0xdd, 0xc1, 0xcd, 0x4e, 0x00, 0x51, 0xcc, 0xcc,
}
// key
key := []byte("yuyuyuyuopopopop")
c, _ := xtea.NewCipher(key)
myout := make([]byte, len(test))
// decrypte
c.Decrypt(myout, test)
fmt.Println(myout)
}
My results do not coincide with what should be the result. Maybe I'm not using the library correctly? Or was it wrong with the data type? I'm new to the golang. The array that should be output:
09300004100e00000104f47795590210
0000000000000000616161615d5d5d5d
09150000000000000000000000000000
000000000000000d1052545530322e30
312e3030303200000012040000000013
040000000014046161616115045d5d5d
5d1604010e00001704010e0000180401
0e00001904010e00001a04010e00001b
04010e00001c04010e00001d04010e00
001e01001f0100200100210100240100
25113235303032000000000000000000
0000002604ef1400002704930d00002d
01022e02e0012f05ffffffff00300103
3101003301003404050100003d20342e
3132382e323400000000000000000000
00000000000000000000000000004401
184f04960d0000500400000000570460
ea0000580422060000590460ea00005a
04e01500005b01005c01025d01005e01
005f010360010361010262010400011b
Maybe someone will come in handy. The library “golang.org/x/crypto/xtea” does not include the XTEA algorithm with the ECB model because of its insecurity, therefore I wrote my own version:
package main
import (
"fmt"
"encoding/hex"
"encoding/binary"
)
func main() {
original := []byte{
0x60, 0x61, 0x4e, 0x68, 0x0e, 0x70, 0x5d, 0x0f, 0xef, 0xcf, 0x7a, 0xc8, 0x10, 0x2c, 0x44, 0x52,
0xec, 0xb0, 0xc8, 0x57, 0x68, 0xf2, 0xf2, 0xdc, 0x52, 0x41, 0x5c, 0x43, 0xa3, 0x67, 0x12, 0xf0,
0x31, 0xc9, 0x03, 0x7d, 0xaf, 0xd3, 0x1f, 0x01, 0xec, 0xb0, 0xc8, 0x57, 0x68, 0xf2, 0xf2, 0xdc,
0x7b, 0x00, 0xbe, 0x7e, 0x5a, 0x15, 0xfe, 0xe1, 0xe7, 0x8c, 0x63, 0xc5, 0x8c, 0x2c, 0x68, 0x61,
0xfe, 0xf9, 0xa1, 0xc4, 0x13, 0x0a, 0x35, 0x4c, 0x84, 0x64, 0x48, 0x51, 0x2e, 0x6a, 0x97, 0xce,
0x4a, 0x90, 0x05, 0x69, 0x0d, 0x1e, 0x38, 0x08, 0xf0, 0x65, 0xc9, 0x57, 0x53, 0x8e, 0x1b, 0xac,
0x87, 0xe7, 0x22, 0x83, 0x22, 0xab, 0x39, 0xa6, 0x90, 0x01, 0x46, 0x78, 0x68, 0x40, 0xdc, 0x0b,
0xc5, 0x36, 0xad, 0x6a, 0xfb, 0x6e, 0x4e, 0x32, 0x67, 0xfb, 0x04, 0x5d, 0xd9, 0xc7, 0xe6, 0x70,
0xf1, 0xc2, 0xd2, 0xac, 0x1f, 0xcc, 0x71, 0xad, 0x06, 0xb7, 0xb1, 0x94, 0xde, 0x40, 0x31, 0xf4,
0x04, 0x67, 0x44, 0x61, 0x0a, 0xaf, 0xa7, 0xb9, 0x2f, 0xd3, 0xf3, 0x92, 0xc3, 0xa5, 0xee, 0xb1,
0x47, 0x4f, 0xfa, 0x60, 0xc0, 0x58, 0x7e, 0x68, 0xec, 0xb0, 0xc8, 0x57, 0x68, 0xf2, 0xf2, 0xdc,
0x2a, 0x88, 0x82, 0x74, 0x61, 0xb4, 0x1c, 0x99, 0xb2, 0x53, 0x9b, 0x6b, 0xfd, 0xcd, 0x43, 0x25,
0xbe, 0x3c, 0xed, 0x59, 0xbe, 0x7b, 0x59, 0x4a, 0xdd, 0xb3, 0x36, 0x6e, 0x07, 0x6f, 0x6e, 0x47,
0x0c, 0xc4, 0x1d, 0xf1, 0xeb, 0x3a, 0x8d, 0x93, 0xc9, 0x9e, 0xb7, 0xbd, 0xad, 0x5a, 0x47, 0x4c,
0x33, 0x65, 0x96, 0x53, 0x76, 0x29, 0x10, 0xd0, 0xec, 0xb0, 0xc8, 0x57, 0x68, 0xf2, 0xf2, 0xdc,
0xec, 0xb0, 0xc8, 0x57, 0x68, 0xf2, 0xf2, 0xdc, 0x82, 0xe7, 0x15, 0xe7, 0x95, 0x2a, 0x79, 0xc4,
0x66, 0x00, 0x74, 0xcc, 0xc5, 0x07, 0x41, 0xca, 0xb5, 0xea, 0xbb, 0x87, 0x3a, 0xe7, 0x06, 0xb4,
0xc8, 0xb0, 0x08, 0x12, 0x8d, 0xf0, 0xaf, 0x80, 0xfe, 0xce, 0x91, 0x74, 0x1f, 0xc5, 0xf6, 0x41,
0x11, 0x45, 0xaa, 0xb3, 0x5a, 0xc9, 0xf6, 0xe0, 0xf8, 0xa9, 0x37, 0xba, 0xed, 0x01, 0x2d, 0x00,
0xc3, 0xbe, 0x70, 0x5a, 0x5e, 0x8c, 0x34, 0x40, 0xdd, 0xc1, 0xcd, 0x4e, 0x00, 0x51, 0xcc, 0xcc}
testKey := []byte("yuyuyuyuopopopop")
block := make([]byte, 8)
for i := 0; i< len(original)/8; i++ {
block = decrypt(testKey , original[(i*8):((i+1)*8)], 32)
fmt.Println(hex.Dump(block))
}
}
func decrypt(key []byte, block []byte, rounds uint32) ([]byte){
var k [4]uint32
var i uint32
end := make([]byte, 8)
v0 := binary.LittleEndian.Uint32(block[:4])
v1 := binary.LittleEndian.Uint32(block[4:])
k[0] = binary.LittleEndian.Uint32(key[:4])
k[1] = binary.LittleEndian.Uint32(key[4:8])
k[2] = binary.LittleEndian.Uint32(key[8:12])
k[3] = binary.LittleEndian.Uint32(key[12:])
delta := binary.LittleEndian.Uint32([]byte{0xb9, 0x79, 0x37, 0x9e})
mask := binary.LittleEndian.Uint32([]byte{0xff, 0xff, 0xff, 0xff})
sum := (delta * rounds) & mask
for i=0; i<rounds; i++ {
v1 = (v1 - (((v0<<4 ^ v0>>5) + v0) ^ (sum + k[sum>>11 & 3]))) & mask
sum = (sum - delta) & mask
v0 = (v0 - (((v1<<4 ^ v1>>5) + v1) ^ (sum + k[sum & 3]))) & mask
}
binary.LittleEndian.PutUint32(end[:4], v0)
binary.LittleEndian.PutUint32(end[4:], v1)
return end
}