flow like the river

This commit is contained in:
root 2025-11-07 00:06:12 +01:00
commit 013fe673f3
42435 changed files with 5764238 additions and 0 deletions

View file

@ -0,0 +1,176 @@
'use strict';
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
const errorUtils = require('./errorUtils');
class RPCManager {
constructor(options) {
this.callQueue = [];
this.callId = 0;
this.calls = new Map();
this.maxConcurrentCalls = options.maxConcurrentCalls || Infinity;
this.send = options.send;
}
// Keep in mind to make sure responses to these calls are JSON.Stringify safe
addCall(request, awaitResponse = true) {
var _this = this;
return _asyncToGenerator(function* () {
// console.log('add call', request)
let call = request;
call.type = 'request';
call.child = _this.childId;
call.awaitResponse = awaitResponse;
let promise;
if (awaitResponse) {
promise = new Promise(function (resolve, reject) {
call.resolve = resolve;
call.reject = reject;
});
}
_this.callQueue.push(call);
_this.processQueue();
return promise;
})();
}
sendRequest(call) {
var _this2 = this;
return _asyncToGenerator(function* () {
let idx;
if (call.awaitResponse) {
idx = _this2.callId++;
_this2.calls.set(idx, call);
}
return _this2.send({
idx: idx,
child: call.child,
type: call.type,
location: call.location,
method: call.method,
args: call.args,
awaitResponse: call.awaitResponse
});
})();
}
processQueue() {
var _this3 = this;
return _asyncToGenerator(function* () {
if (!_this3.callQueue.length) {
return;
}
if (_this3.calls.size < _this3.maxConcurrentCalls) {
let call = _this3.callQueue[0];
if (_this3.sendRequest(call) !== false) {
_this3.callQueue.shift();
}
}
})();
}
handleMessage(data, child) {
var _this4 = this;
return _asyncToGenerator(function* () {
// console.log('message', data)
if (data.type === 'module' && data.module && !_this4.module) {
_this4.module = require(data.module);
_this4.childId = data.child;
if (_this4.module.setChildReference) {
_this4.module.setChildReference(_this4);
}
return;
}
if (data.type === 'response') {
return _this4.handleResponse(data);
} else if (data.type === 'request') {
return _this4.handleRequest(data, child);
}
})();
}
handleRequest(data, child) {
var _this5 = this;
return _asyncToGenerator(function* () {
let result = {
type: 'response',
idx: data.idx,
child: data.child
};
let module = _this5.module;
if (data.location) {
module = require(data.location);
}
if (!module) {
throw new Error('No module was specified');
}
let func = data.method ? module[data.method] : module;
if (typeof func !== 'function') {
throw new Error('Module or method is not a function');
}
try {
result.contentType = 'data';
result.content = yield func(...data.args);
} catch (e) {
result.contentType = 'error';
result.content = errorUtils.errorToJson(e);
}
if (data.awaitResponse) {
if (child) {
child.send(result);
} else {
_this5.send(result);
}
}
})();
}
handleResponse(data, child) {
var _this6 = this;
return _asyncToGenerator(function* () {
let idx = data.idx;
let contentType = data.contentType;
let content = data.content;
let call = _this6.calls.get(idx);
if (!call) {
throw new Error(`Worker Farm: Received message for unknown call index. This should not happen!`);
}
if (contentType === 'error') {
call.reject(errorUtils.jsonToError(content));
} else {
call.resolve(content);
}
_this6.calls.delete(idx);
if (child && child.calls) {
child.calls.delete(idx);
}
// Process the next call
_this6.processQueue();
})();
}
}
module.exports = RPCManager;

View file

@ -0,0 +1,142 @@
'use strict';
const childProcess = require('child_process');
var _require = require('events');
const EventEmitter = _require.EventEmitter;
const errorUtils = require('./errorUtils');
const childModule = parseInt(process.versions.node, 10) < 8 ? require.resolve('../../lib/workerfarm/child') : require.resolve('../../src/workerfarm/child');
let WORKER_ID = 0;
class Worker extends EventEmitter {
constructor(forkModule, options) {
super();
this.options = options;
this.id = WORKER_ID++;
this.sendQueue = [];
this.processQueue = true;
this.calls = new Map();
this.exitCode = null;
this.callId = 0;
this.stopped = false;
this.fork(forkModule);
}
fork(forkModule) {
let filteredArgs = process.execArgv.filter(v => !/^--(debug|inspect)/.test(v));
let options = {
execArgv: filteredArgs,
env: process.env,
cwd: process.cwd()
};
this.child = childProcess.fork(childModule, process.argv, options);
this.send({
type: 'module',
module: forkModule,
child: this.id
});
this.child.on('message', this.receive.bind(this));
this.child.once('exit', code => {
this.exitCode = code;
this.emit('exit', code);
});
this.child.on('error', err => {
this.emit('error', err);
});
}
send(data) {
if (!this.processQueue) {
return this.sendQueue.push(data);
}
let result = this.child.send(data, error => {
if (error && error instanceof Error) {
// Ignore this, the workerfarm handles child errors
return;
}
this.processQueue = true;
if (this.sendQueue.length > 0) {
let queueCopy = this.sendQueue.slice(0);
this.sendQueue = [];
queueCopy.forEach(entry => this.send(entry));
}
});
if (!result || /^win/.test(process.platform)) {
// Queue is handling too much messages throttle it
this.processQueue = false;
}
}
call(call) {
let idx = this.callId++;
this.calls.set(idx, call);
this.send({
type: 'request',
idx: idx,
child: this.id,
method: call.method,
args: call.args
});
}
receive(data) {
if (this.stopped) {
return;
}
let idx = data.idx;
let type = data.type;
let content = data.content;
let contentType = data.contentType;
if (type === 'request') {
this.emit('request', data);
} else if (type === 'response') {
let call = this.calls.get(idx);
if (!call) {
// Return for unknown calls, these might accur if a third party process uses workers
return;
}
if (contentType === 'error') {
call.reject(errorUtils.jsonToError(content));
} else {
call.resolve(content);
}
this.calls.delete(idx);
this.emit('response', data);
}
}
stop() {
this.stopped = true;
this.send('die');
setTimeout(() => {
if (this.exitCode === null) {
this.child.kill('SIGKILL');
}
}, this.options.forcedKillTime);
}
}
module.exports = Worker;

View file

@ -0,0 +1,372 @@
'use strict';
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
var _require = require('events');
const EventEmitter = _require.EventEmitter;
const os = require('os');
const errorUtils = require('./errorUtils');
const Worker = require('./Worker');
let shared = null;
class WorkerFarm extends EventEmitter {
constructor(options, farmOptions = {}) {
super();
this.options = Object.assign({
maxConcurrentWorkers: WorkerFarm.getNumWorkers(),
maxConcurrentCallsPerWorker: 10,
forcedKillTime: 100,
warmWorkers: true,
useLocalWorker: true,
workerPath: '../worker'
}, farmOptions);
this.started = false;
this.warmWorkers = 0;
this.children = new Map();
this.callQueue = [];
this.localWorker = require(this.options.workerPath);
this.run = this.mkhandle('run');
this.init(options);
}
warmupWorker(method, args) {
// Workers have started, but are not warmed up yet.
// Send the job to a remote worker in the background,
// but use the result from the local worker - it will be faster.
if (this.started) {
let promise = this.addCall(method, [...args, true]);
if (promise) {
promise.then(() => {
this.warmWorkers++;
if (this.warmWorkers >= this.children.size) {
this.emit('warmedup');
}
}).catch(() => {});
}
}
}
mkhandle(method) {
return function (...args) {
// Child process workers are slow to start (~600ms).
// While we're waiting, just run on the main thread.
// This significantly speeds up startup time.
if (this.shouldUseRemoteWorkers()) {
return this.addCall(method, [...args, false]);
} else {
if (this.options.warmWorkers) {
this.warmupWorker(method, args);
}
return this.localWorker[method](...args, false);
}
}.bind(this);
}
onError(error, childId) {
// Handle ipc errors
if (error.code === 'ERR_IPC_CHANNEL_CLOSED') {
return this.stopChild(childId);
}
}
onExit(childId) {
// delay this to give any sends a chance to finish
setTimeout(() => {
let doQueue = false;
let child = this.children.get(childId);
if (child && child.calls.size) {
var _iteratorNormalCompletion = true;
var _didIteratorError = false;
var _iteratorError = undefined;
try {
for (var _iterator = child.calls.values()[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true) {
let call = _step.value;
call.retries++;
this.callQueue.unshift(call);
doQueue = true;
}
} catch (err) {
_didIteratorError = true;
_iteratorError = err;
} finally {
try {
if (!_iteratorNormalCompletion && _iterator.return) {
_iterator.return();
}
} finally {
if (_didIteratorError) {
throw _iteratorError;
}
}
}
}
this.stopChild(childId);
if (doQueue) {
this.processQueue();
}
}, 10);
}
startChild() {
let worker = new Worker(this.options.workerPath, this.options);
worker.on('request', data => {
this.processRequest(data, worker);
});
worker.on('response', () => {
// allow any outstanding calls to be processed
this.processQueue();
});
worker.once('exit', () => {
this.onExit(worker.id);
});
worker.on('error', err => {
this.onError(err, worker.id);
});
this.children.set(worker.id, worker);
}
stopChild(childId) {
let child = this.children.get(childId);
if (child) {
child.stop();
this.children.delete(childId);
}
}
processQueue() {
var _this = this;
return _asyncToGenerator(function* () {
if (_this.ending || !_this.callQueue.length) return;
if (_this.children.size < _this.options.maxConcurrentWorkers) {
_this.startChild();
}
var _iteratorNormalCompletion2 = true;
var _didIteratorError2 = false;
var _iteratorError2 = undefined;
try {
for (var _iterator2 = _this.children.values()[Symbol.iterator](), _step2; !(_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done); _iteratorNormalCompletion2 = true) {
let child = _step2.value;
if (!_this.callQueue.length) {
break;
}
if (child.calls.size < _this.options.maxConcurrentCallsPerWorker) {
child.call(_this.callQueue.shift());
}
}
} catch (err) {
_didIteratorError2 = true;
_iteratorError2 = err;
} finally {
try {
if (!_iteratorNormalCompletion2 && _iterator2.return) {
_iterator2.return();
}
} finally {
if (_didIteratorError2) {
throw _iteratorError2;
}
}
}
})();
}
processRequest(data, child = false) {
return _asyncToGenerator(function* () {
let result = {
idx: data.idx,
type: 'response'
};
let method = data.method;
let args = data.args;
let location = data.location;
let awaitResponse = data.awaitResponse;
if (!location) {
throw new Error('Unknown request');
}
const mod = require(location);
try {
let func;
if (method) {
func = mod[method];
} else {
func = mod;
}
result.contentType = 'data';
result.content = yield func(...args);
} catch (e) {
result.contentType = 'error';
result.content = errorUtils.errorToJson(e);
}
if (awaitResponse) {
if (child) {
child.send(result);
} else {
return result;
}
}
})();
}
addCall(method, args) {
if (this.ending) return; // don't add anything new to the queue
return new Promise((resolve, reject) => {
this.callQueue.push({
method,
args: args,
retries: 0,
resolve,
reject
});
this.processQueue();
});
}
end() {
var _this2 = this;
return _asyncToGenerator(function* () {
_this2.ending = true;
var _iteratorNormalCompletion3 = true;
var _didIteratorError3 = false;
var _iteratorError3 = undefined;
try {
for (var _iterator3 = _this2.children.keys()[Symbol.iterator](), _step3; !(_iteratorNormalCompletion3 = (_step3 = _iterator3.next()).done); _iteratorNormalCompletion3 = true) {
let childId = _step3.value;
_this2.stopChild(childId);
}
} catch (err) {
_didIteratorError3 = true;
_iteratorError3 = err;
} finally {
try {
if (!_iteratorNormalCompletion3 && _iterator3.return) {
_iterator3.return();
}
} finally {
if (_didIteratorError3) {
throw _iteratorError3;
}
}
}
_this2.ending = false;
shared = null;
})();
}
init(options) {
this.localWorker.init(options, true);
this.initRemoteWorkers(options);
}
initRemoteWorkers(options) {
var _this3 = this;
return _asyncToGenerator(function* () {
_this3.started = false;
_this3.warmWorkers = 0;
// Start workers if there isn't enough workers already
for (let i = _this3.children.size; i < _this3.options.maxConcurrentWorkers; i++) {
_this3.startChild();
}
// Reliable way of initialising workers
let promises = [];
var _iteratorNormalCompletion4 = true;
var _didIteratorError4 = false;
var _iteratorError4 = undefined;
try {
for (var _iterator4 = _this3.children.values()[Symbol.iterator](), _step4; !(_iteratorNormalCompletion4 = (_step4 = _iterator4.next()).done); _iteratorNormalCompletion4 = true) {
let child = _step4.value;
promises.push(new Promise(function (resolve, reject) {
child.call({
method: 'init',
args: [options],
retries: 0,
resolve,
reject
});
}));
}
} catch (err) {
_didIteratorError4 = true;
_iteratorError4 = err;
} finally {
try {
if (!_iteratorNormalCompletion4 && _iterator4.return) {
_iterator4.return();
}
} finally {
if (_didIteratorError4) {
throw _iteratorError4;
}
}
}
yield Promise.all(promises);
if (_this3.options.maxConcurrentWorkers > 0) {
_this3.started = true;
_this3.emit('started');
}
})();
}
shouldUseRemoteWorkers() {
return !this.options.useLocalWorker || this.started && (this.warmWorkers >= this.children.size || !this.options.warmWorkers);
}
static getShared(options) {
if (!shared) {
shared = new WorkerFarm(options);
} else if (options) {
shared.init(options);
}
return shared;
}
static getNumWorkers() {
if (process.env.PARCEL_WORKERS) {
return parseInt(process.env.PARCEL_WORKERS, 10);
}
let cores;
try {
cores = require('physical-cpu-count');
} catch (err) {
cores = os.cpus().length;
}
return cores || 1;
}
}
module.exports = WorkerFarm;

View file

@ -0,0 +1,168 @@
'use strict';
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
const errorUtils = require('./errorUtils');
class Child {
constructor() {
this.module = undefined;
this.childId = undefined;
this.callQueue = [];
this.responseQueue = new Map();
this.responseId = 0;
this.maxConcurrentCalls = 10;
}
messageListener(data) {
if (data === 'die') {
return this.end();
}
if (data.type === 'module' && data.module && !this.module) {
this.module = require(data.module);
this.childId = data.child;
if (this.module.setChildReference) {
this.module.setChildReference(this);
}
return;
}
let type = data.type;
if (type === 'response') {
return this.handleResponse(data);
} else if (type === 'request') {
return this.handleRequest(data);
}
}
send(data) {
var _this = this;
return _asyncToGenerator(function* () {
process.send(data, function (err) {
if (err && err instanceof Error) {
if (err.code === 'ERR_IPC_CHANNEL_CLOSED') {
// IPC connection closed
// no need to keep the worker running if it can't send or receive data
return _this.end();
}
}
});
})();
}
handleRequest(data) {
var _this2 = this;
return _asyncToGenerator(function* () {
let idx = data.idx;
let child = data.child;
let method = data.method;
let args = data.args;
let result = { idx, child, type: 'response' };
try {
result.contentType = 'data';
result.content = yield _this2.module[method](...args);
} catch (e) {
result.contentType = 'error';
result.content = errorUtils.errorToJson(e);
}
_this2.send(result);
})();
}
handleResponse(data) {
var _this3 = this;
return _asyncToGenerator(function* () {
let idx = data.idx;
let contentType = data.contentType;
let content = data.content;
let call = _this3.responseQueue.get(idx);
if (contentType === 'error') {
call.reject(errorUtils.jsonToError(content));
} else {
call.resolve(content);
}
_this3.responseQueue.delete(idx);
// Process the next call
_this3.processQueue();
})();
}
// Keep in mind to make sure responses to these calls are JSON.Stringify safe
addCall(request, awaitResponse = true) {
var _this4 = this;
return _asyncToGenerator(function* () {
let call = request;
call.type = 'request';
call.child = _this4.childId;
call.awaitResponse = awaitResponse;
let promise;
if (awaitResponse) {
promise = new Promise(function (resolve, reject) {
call.resolve = resolve;
call.reject = reject;
});
}
_this4.callQueue.push(call);
_this4.processQueue();
return promise;
})();
}
sendRequest(call) {
var _this5 = this;
return _asyncToGenerator(function* () {
let idx;
if (call.awaitResponse) {
idx = _this5.responseId++;
_this5.responseQueue.set(idx, call);
}
_this5.send({
idx: idx,
child: call.child,
type: call.type,
location: call.location,
method: call.method,
args: call.args,
awaitResponse: call.awaitResponse
});
})();
}
processQueue() {
var _this6 = this;
return _asyncToGenerator(function* () {
if (!_this6.callQueue.length) {
return;
}
if (_this6.responseQueue.size < _this6.maxConcurrentCalls) {
_this6.sendRequest(_this6.callQueue.shift());
}
})();
}
end() {
return process.exit(0);
}
}
let child = new Child();
process.on('message', child.messageListener.bind(child));
module.exports = child;

View file

@ -0,0 +1,25 @@
"use strict";
function errorToJson(error) {
let jsonError = {
message: error.message,
stack: error.stack,
name: error.name
};
// Add all custom codeFrame properties
Object.keys(error).forEach(key => {
jsonError[key] = error[key];
});
return jsonError;
}
function jsonToError(json) {
let error = new Error(json.message);
Object.keys(json).forEach(key => {
error[key] = json[key];
});
return error;
}
exports.errorToJson = errorToJson;
exports.jsonToError = jsonToError;

View file

@ -0,0 +1,51 @@
'use strict';
const childProcess = require('child_process');
const childModule = parseInt(process.versions.node, 10) < 8 ? require.resolve('../../lib/workerfarm/child') : require.resolve('../../src/workerfarm/child');
function fork(forkModule, childId) {
// suppress --debug / --inspect flags while preserving others (like --harmony)
let filteredArgs = process.execArgv.filter(v => !/^--(debug|inspect)/.test(v));
let options = {
execArgv: filteredArgs,
env: process.env,
cwd: process.cwd()
};
let child = childProcess.fork(childModule, process.argv, options);
let sendQueue = [];
let processQueue = true;
function send(data) {
if (!processQueue) {
return sendQueue.push(data);
}
let result = child.send(data, error => {
if (error && error instanceof Error) {
// Ignore this, the workerfarm handles child errors
return;
}
processQueue = true;
if (sendQueue.length > 0) {
let queueCopy = sendQueue.slice(0);
sendQueue = [];
queueCopy.forEach(entry => send(entry));
}
});
if (!result || /^win/.test(process.platform)) {
// Queue is handling too much messages throttle it
processQueue = false;
}
}
send({ module: forkModule, child: childId });
// return a send function for this child
return { send, child };
}
module.exports = fork;