How to list folders and files in a directory using ReactiveX - rxjs

Question

When using Observables for certain tasks that involve a lot of chaining and a lot of asynchronous operations, such as listing all the items in a folder and checking all of the folders in it for a specific file, I often end up either needing to build the complex chain for each task (return Observable.of(folder)...) or having some kind of special value that gets forwarded to the end to signal the end of a batch (every operator starts with if(res === false) return Observable.of(false)).
Sort of like that stick that you put between your groceries and those of the person in front of you at the checkout.
It seems like there should be a better way that doesn't involve forwarding a stop value through all kinds of callbacks and operators.
So what is a good way to call a function that takes a folder path string and returns a list of all the files and folders in it. It also specifies whether the files are HTML files or not, and whether or not the folders contain a file called tiddlywiki.json.
The only requirement is that it can't return anything like Observable.of(...).... It should probably have a subject at the top of the chain, but that is not a requirement.
function listFolders(folder) {
return [
{ type: 'folder', name: 'folder1' },
{ type: 'datafolder', name: 'folder2' }, //contains "tiddlywiki.json" file
{ type: 'folder', name: 'folder3' },
{ type: 'htmlfile', name: 'test.html' },
{ type: 'other', name: 'mytest.txt' }
]
}

Answer 1

Here is one that does not follow the rules I layed out (see below for one that does), but it took about ten minutes, using the first one as a guide.
export function statFolder(subscriber, input: Observable<any>) {
return input.mergeMap(([folder, tag]) => {
return obs_readdir({ folder, tag })(folder);
}).mergeMap(([err, files, { folder, tag }]) => {
if (err) { return Observable.of({ error: err }) as any; }
else return Observable.from(files).mergeMap(file => {
return obs_stat([file,folder])(path.join(folder, file as string));
}).map(statFolderEntryCB).mergeMap<any, any>((res) => {
let [entry, [name, folder]] = res as [any, [string, string, number, any]];
if (entry.type === 'folder')
return obs_readdir([entry])(path.join(entry.folder, entry.name));
else return Observable.of([true, entry]);
}, 20).map((res) => {
if (res[0] === true) return (res);
let [err, files, [entry]] = res as [any, string[], [FolderEntry, number, any]];
if (err) {
entry.type = "error";
} else if (files.indexOf('tiddlywiki.json') > -1)
entry.type = 'datafolder';
return ([true, entry]);
}).reduce((n, [dud, entry]) => {
n.push(entry);
return n;
}, []).map(entries => {
return { entries, folder, tag };
}) as Observable<{ entries: any, folder: any, tag: any }>;
}).subscribe(subscriber);
}
Original: This took a few hours to write...and it works...but...it uses concatMap, so it can only take one request at a time. It uses a custom operator that I wrote for the purpose.
export function statFileBatch(subscriber, input: Observable<any>) {
const signal = new Subject<number>();
var count = 0;
//use set timeout to fire after the buffer recieves this item
const sendSignal = (item) => setTimeout(() => { count = 0; signal.next(item); });
return input.concatMap(([folder, tag]) => {
return obs_readdir({ folder, tag })(folder);
}).lift({
call: (subs: Subscriber<any>, source: Observable<any>) => {
const signalFunction = (count) => signal.mapTo(1), forwardWhenEmpty = true;
const waiting = [];
const _output = new Subject();
var _count = new Subject<number>()
const countFactory = Observable.defer(() => {
return Observable.create(subscriber => {
_count.subscribe(subscriber);
})
});
var isEmpty = true;
const sourceSubs = source.subscribe(item => {
if (isEmpty && forwardWhenEmpty) {
_output.next(item);
} else {
waiting.push(item)
}
isEmpty = false;
})
const pulse = new Subject<any>();
const signalSubs = pulse.switchMap(() => {
return signalFunction(countFactory)
}).subscribe(count => {
//act on the closing observable value
var i = 0;
while (waiting.length > 0 && i++ < count)
_output.next(waiting.shift());
//if nothing was output, then we are empty
//if something was output then we are not
//this is meant to be used with bufferWhen
if (i === 0) isEmpty = true;
_count.next(i);
_count.complete();
_count = new Subject<number>();
pulse.next();
})
pulse.next(); //prime the pump
const outputSubs = Observable.create((subscriber) => {
return _output.subscribe(subscriber);
}).subscribe(subs) as Subscription;
return function () {
outputSubs.unsubscribe();
signalSubs.unsubscribe();
sourceSubs.unsubscribe();
}
}
}).mergeMap(([err, files, { folder, tag }]) => {
if (err) { sendSignal(err); return Observable.empty(); }
return Observable.from(files.map(a => [a, folder, files.length, tag])) as any;
}).mergeMap((res: any) => {
let [file, folder, fileCount, tag] = res as [string, string, number, any];
return obs_stat([file, folder, fileCount, tag])(path.join(folder, file))
}, 20).map(statFolderEntryCB).mergeMap<any, any>((res) => {
let [entry, [name, folder, fileCount, tag]] = res as [any, [string, string, number, any]];
if (entry.type === 'folder')
return obs_readdir([entry, fileCount, tag])(path.join(entry.folder, entry.name));
else return Observable.of([true, entry, fileCount, tag]);
}, 20).map((res) => {
//if (res === false) return (false);
if (res[0] === true) return (res);
let [err, files, [entry, fileCount, tag]] = res as [any, string[], [FolderEntry, number, any]];
if (err) {
entry.type = "error";
} else if (files.indexOf('tiddlywiki.json') > -1)
entry.type = 'datafolder';
return ([true, entry, fileCount, tag]);
}).map(([dud, entry, fileCount, tag]) => {
count++;
if (count === fileCount) {
sendSignal([count, tag]);
}
return entry;
}).bufferWhen(() => signal).withLatestFrom(signal).map(([files, [sigResult, tag]]: any) => {
return [
typeof sigResult !== 'number' ? sigResult : null, //error object
files, //file list
typeof sigResult === 'number' ? sigResult : null, //file count
tag //tag
];
}).subscribe(subscriber);
}