Process data in pipe in asynchronous way - rxjs

Question

I have process function which works in async way - it takes line of input text and process it (in snippet below - for simplification - it extract number from line). The lines came form input large file (many GB) and are read and send to pipe one-by-one (this is 'simulated' by of operator below). The process function is slow - but reading lines is fast - so critical point is to read and process lines one-by-one to avoid memory "stack-overflow".
const { of, Subject } = rxjs;
const { take, bufferCount, map, finalize } = rxjs.operators;
let source = of("line 11 of text", "line 22 of text", "line 33 of text") // large file with lines
.pipe(map(line => process(line, ()=> {
// ???? how te callback/pipe shoud look ????
}))
);
source.subscribe(x=> console.log(x));
// expected result shoud be:
// 11
// 22
// 33
// This is third-party library - I cannot change it
function process(line, callback) {
setTimeout(_=>{
let result = line.match(/\d/)[0];
callback(result);
}, (1+Math.random()*9)*100); // random processing time
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/rxjs/6.5.5/rxjs.umd.min.js" integrity="sha256-85uCh8dPb35sH3WK435rtYUKALbcvEQFC65eg+raeuc=" crossorigin="anonymous"></script>
I don't know how to do it - any idea?

Answer 1

You need to convert your callback to Promise as Promises can work with rxjs streams.
import { of } from "rxjs";
import { concatMap } from "rxjs/operators";
const promiseWrapper = line => new Promise(resolve => process(line, resolve));
let source = of("line 11 of text", "line 22 of text", "line 33 of text").pipe(
concatMap(item => promiseWrapper(item))
);
source.subscribe(x => console.log(x));
// expected result shoud be:
// 11
// 22
// 33
// This is third-party library - I cannot change it
function process(line, callback) {
setTimeout(_ => {
let result = line.match(/\d+/)[0];
callback(result);
}, 1000);
}
also, I've prepared stackblitz (or below snippet)
const { of, Subject } = rxjs;
const { concatMap } = rxjs.operators;
const promiseWrapper = line => new Promise(resolve => process(line, resolve));
let source = of(
"line 11 of text",
"line 22 of text",
"line 33 of text") // large file with lines
.pipe(concatMap(item => promiseWrapper(item))
);
source.subscribe(x=> console.log(x));
// expected result shoud be:
// 11
// 22
// 33
// This is third-party library - I cannot change it
function process(line, callback) {
setTimeout(_=>{
let result = line.match(/\d+/)[0];
callback(result);
}, (1+Math.random()*9)*100);
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/rxjs/6.5.5/rxjs.umd.min.js" integrity="sha256-85uCh8dPb35sH3WK435rtYUKALbcvEQFC65eg+raeuc=" crossorigin="anonymous"></script>

Answer 2

You can use bindCallback to convert a function of type f(x, callback) to a function g(x) that returns a cold observable that emits the result passed to the callback.
import { bindCallback, of } from "rxjs";
import { concatMap } from "rxjs/operators";
const boundProcess = bindCallback(process);
let source = of("line 11 of text", "line 22 of text", "line 33 of text").pipe(
concatMap(line => boundProcess(line))
);
source.subscribe(x => console.log(x));
// expected result shoud be:
// 11
// 22
// 33
// This is third-party library - I cannot change it
function process(line, callback) {
setTimeout(_ => {
let result = line.match(/\d+/)[0];
callback(result);
}, 1000);
}
https://stackblitz.com/edit/typescript-2rai4c?file=index.ts