larry babby and threejs for glsl

This commit is contained in:
Sam
2024-06-24 21:24:00 +12:00
parent 87d5dc634d
commit 907ebae4c0
6474 changed files with 1279596 additions and 8 deletions

View File

@@ -0,0 +1,46 @@
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.default = void 0;
function _core() {
const data = require("@parcel/core");
_core = function () {
return data;
};
return data;
}
var _package = _interopRequireDefault(require("../package.json"));
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
// $FlowFixMe
let HANDLE_ID = 0;
// $FlowFixMe
const handleById = new Map();
class Handle {
constructor(opts) {
var _opts$id;
this.id = (_opts$id = opts.id) !== null && _opts$id !== void 0 ? _opts$id : ++HANDLE_ID;
this.fn = opts.fn;
this.childId = opts.childId;
handleById.set(this.id, this);
}
dispose() {
handleById.delete(this.id);
}
serialize() {
return {
id: this.id,
childId: this.childId
};
}
static deserialize(opts) {
return new Handle(opts);
}
}
// Register the Handle as a serializable class so that it will properly be deserialized
// by anything that uses WorkerFarm.
exports.default = Handle;
(0, _core().registerSerializableClass)(`${_package.default.version}:Handle`, Handle);

View File

@@ -0,0 +1,188 @@
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.default = void 0;
function _nullthrows() {
const data = _interopRequireDefault(require("nullthrows"));
_nullthrows = function () {
return data;
};
return data;
}
function _events() {
const data = _interopRequireDefault(require("events"));
_events = function () {
return data;
};
return data;
}
function _diagnostic() {
const data = _interopRequireDefault(require("@parcel/diagnostic"));
_diagnostic = function () {
return data;
};
return data;
}
var _backend = require("./backend");
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
let WORKER_ID = 0;
class Worker extends _events().default {
id = WORKER_ID++;
sentSharedReferences = new Set();
calls = new Map();
exitCode = null;
callId = 0;
ready = false;
stopped = false;
isStopping = false;
constructor(options) {
super();
this.options = options;
}
async fork(forkModule) {
let filteredArgs = [];
if (process.execArgv) {
filteredArgs = process.execArgv.filter(v => !/^--(debug|inspect|no-opt|max-old-space-size=|max-semi-space-size=|expose-gc)/.test(v));
for (let i = 0; i < filteredArgs.length; i++) {
let arg = filteredArgs[i];
let isArgWithParam = (arg === '-r' || arg === '--require') && filteredArgs[i + 1] === '@parcel/register' || arg === '--title';
if (isArgWithParam) {
filteredArgs.splice(i, 2);
i--;
}
}
}
// Workaround for https://github.com/nodejs/node/issues/29117
if (process.env.NODE_OPTIONS) {
// arg parsing logic adapted from https://stackoverflow.com/a/46946420/2352201
let opts = [''];
let quote = false;
for (let c of (0, _nullthrows().default)(process.env.NODE_OPTIONS.match(/.|^$/g))) {
if (c === '"') {
quote = !quote;
} else if (!quote && c === ' ') {
opts.push('');
} else {
opts[opts.length - 1] += c.replace(/\\(.)/, '$1');
}
}
for (let i = 0; i < opts.length; i++) {
let opt = opts[i];
if (opt === '-r' || opt === '--require') {
filteredArgs.push(opt, opts[i + 1]);
i++;
}
}
}
let onMessage = data => this.receive(data);
let onExit = code => {
this.exitCode = code;
this.emit('exit', code);
};
let onError = err => {
this.emit('error', err);
};
let WorkerBackend = (0, _backend.getWorkerBackend)(this.options.backend);
this.worker = new WorkerBackend(filteredArgs, onMessage, onError, onExit);
await this.worker.start();
await new Promise((resolve, reject) => {
this.call({
method: 'childInit',
args: [forkModule, {
shouldPatchConsole: !!this.options.shouldPatchConsole,
shouldTrace: !!this.options.shouldTrace
}],
retries: 0,
skipReadyCheck: true,
resolve,
reject
});
});
let sharedRefs = this.options.sharedReferences;
let refsShared = new Set();
// in case more refs are created while initial refs are sending
while (refsShared.size < sharedRefs.size) {
await Promise.all([...sharedRefs].filter(([ref]) => !refsShared.has(ref)).map(async ([ref, value]) => {
await this.sendSharedReference(ref, value);
refsShared.add(ref);
}));
}
this.ready = true;
this.emit('ready');
}
sendSharedReference(ref, value) {
this.sentSharedReferences.add(ref);
return new Promise((resolve, reject) => {
this.call({
method: 'createSharedReference',
args: [ref, value],
resolve,
reject,
retries: 0,
skipReadyCheck: true
});
});
}
send(data) {
this.worker.send(data);
}
call(call) {
if (this.stopped || this.isStopping) {
return;
}
let idx = this.callId++;
this.calls.set(idx, call);
let msg = {
type: 'request',
idx: idx,
child: this.id,
handle: call.handle,
method: call.method,
args: call.args
};
if (this.ready || call.skipReadyCheck === true) {
this.send(msg);
} else {
this.once('ready', () => this.send(msg));
}
}
receive(message) {
if (this.stopped || this.isStopping) {
return;
}
if (message.type === 'request') {
this.emit('request', message);
} else if (message.type === 'response') {
let idx = message.idx;
if (idx == null) {
return;
}
let call = this.calls.get(idx);
if (!call) {
// Return for unknown calls, these might accur if a third party process uses workers
return;
}
if (message.contentType === 'error') {
call.reject(new (_diagnostic().default)({
diagnostic: message.content
}));
} else {
call.resolve(message.content);
}
this.calls.delete(idx);
this.emit('response', message);
}
}
async stop() {
if (!this.stopped) {
this.stopped = true;
if (this.worker) {
await this.worker.stop();
}
}
}
}
exports.default = Worker;

View File

@@ -0,0 +1,563 @@
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
Object.defineProperty(exports, "Handle", {
enumerable: true,
get: function () {
return _Handle.default;
}
});
exports.default = void 0;
var coreWorker = _interopRequireWildcard(require("./core-worker"));
var bus = _interopRequireWildcard(require("./bus"));
function _assert() {
const data = _interopRequireDefault(require("assert"));
_assert = function () {
return data;
};
return data;
}
function _nullthrows() {
const data = _interopRequireDefault(require("nullthrows"));
_nullthrows = function () {
return data;
};
return data;
}
function _events() {
const data = _interopRequireDefault(require("events"));
_events = function () {
return data;
};
return data;
}
function _core() {
const data = require("@parcel/core");
_core = function () {
return data;
};
return data;
}
function _diagnostic() {
const data = _interopRequireWildcard(require("@parcel/diagnostic"));
_diagnostic = function () {
return data;
};
return data;
}
var _Worker = _interopRequireDefault(require("./Worker"));
var _cpuCount = _interopRequireDefault(require("./cpuCount"));
var _Handle = _interopRequireDefault(require("./Handle"));
var _childState = require("./childState");
var _backend = require("./backend");
function _profiler() {
const data = require("@parcel/profiler");
_profiler = function () {
return data;
};
return data;
}
function _fs() {
const data = _interopRequireDefault(require("fs"));
_fs = function () {
return data;
};
return data;
}
function _logger() {
const data = _interopRequireDefault(require("@parcel/logger"));
_logger = function () {
return data;
};
return data;
}
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
let referenceId = 1;
const DEFAULT_MAX_CONCURRENT_CALLS = 30;
/**
* workerPath should always be defined inside farmOptions
*/
class WorkerFarm extends _events().default {
callQueue = [];
ending = false;
warmWorkers = 0;
readyWorkers = 0;
workers = new Map();
handles = new Map();
sharedReferences = new Map();
sharedReferencesByValue = new Map();
serializedSharedReferences = new Map();
constructor(farmOptions = {}) {
var _process$stdout;
super();
this.options = {
maxConcurrentWorkers: WorkerFarm.getNumWorkers(),
maxConcurrentCallsPerWorker: WorkerFarm.getConcurrentCallsPerWorker(farmOptions.shouldTrace ? 1 : DEFAULT_MAX_CONCURRENT_CALLS),
forcedKillTime: 500,
warmWorkers: false,
useLocalWorker: true,
// TODO: setting this to false makes some tests fail, figure out why
backend: (0, _backend.detectBackend)(),
...farmOptions
};
if (!this.options.workerPath) {
throw new Error('Please provide a worker path!');
}
// $FlowFixMe
if (process.browser) {
if (this.options.workerPath === '@parcel/core/src/worker.js') {
this.localWorker = coreWorker;
} else {
throw new Error('No dynamic require possible: ' + this.options.workerPath);
}
} else {
// $FlowFixMe this must be dynamic
this.localWorker = require(this.options.workerPath);
}
this.localWorkerInit = this.localWorker.childInit != null ? this.localWorker.childInit() : null;
this.run = this.createHandle('run');
// Worker thread stdout is by default piped into the process stdout, if there are enough worker
// threads to exceed the default listener limit, then anything else piping into stdout will trigger
// the `MaxListenersExceededWarning`, so we should ensure the max listeners is at least equal to the
// number of workers + 1 for the main thread.
//
// Note this can't be fixed easily where other things pipe into stdout - even after starting > 10 worker
// threads `process.stdout.getMaxListeners()` will still return 10, however adding another pipe into `stdout`
// will give the warning with `<worker count + 1>` as the number of listeners.
(_process$stdout = process.stdout) === null || _process$stdout === void 0 || _process$stdout.setMaxListeners(Math.max(process.stdout.getMaxListeners(), WorkerFarm.getNumWorkers() + 1));
this.startMaxWorkers();
}
workerApi = {
callMaster: async (request, awaitResponse = true) => {
// $FlowFixMe
let result = await this.processRequest({
...request,
awaitResponse
});
return (0, _core().deserialize)((0, _core().serialize)(result));
},
createReverseHandle: fn => this.createReverseHandle(fn),
callChild: (childId, request) => new Promise((resolve, reject) => {
(0, _nullthrows().default)(this.workers.get(childId)).call({
...request,
resolve,
reject,
retries: 0
});
}),
runHandle: (handle, args) => this.workerApi.callChild((0, _nullthrows().default)(handle.childId), {
handle: handle.id,
args
}),
getSharedReference: ref => this.sharedReferences.get(ref),
resolveSharedReference: value => this.sharedReferencesByValue.get(value)
};
warmupWorker(method, args) {
// Workers are already stopping
if (this.ending) {
return;
}
// Workers are not warmed up yet.
// Send the job to a remote worker in the background,
// but use the result from the local worker - it will be faster.
let promise = this.addCall(method, [...args, true]);
if (promise) {
promise.then(() => {
this.warmWorkers++;
if (this.warmWorkers >= this.workers.size) {
this.emit('warmedup');
}
}).catch(() => {});
}
}
shouldStartRemoteWorkers() {
return this.options.maxConcurrentWorkers > 0 || !this.options.useLocalWorker;
}
createHandle(method, useMainThread = false) {
if (!this.options.useLocalWorker) {
useMainThread = false;
}
return async (...args) => {
// Child process workers are slow to start (~600ms).
// While we're waiting, just run on the main thread.
// This significantly speeds up startup time.
if (this.shouldUseRemoteWorkers() && !useMainThread) {
return this.addCall(method, [...args, false]);
} else {
if (this.options.warmWorkers && this.shouldStartRemoteWorkers()) {
this.warmupWorker(method, args);
}
let processedArgs;
if (!useMainThread) {
processedArgs = (0, _core().restoreDeserializedObject)((0, _core().prepareForSerialization)([...args, false]));
} else {
processedArgs = args;
}
if (this.localWorkerInit != null) {
await this.localWorkerInit;
this.localWorkerInit = null;
}
return this.localWorker[method](this.workerApi, ...processedArgs);
}
};
}
onError(error, worker) {
// Handle ipc errors
if (error.code === 'ERR_IPC_CHANNEL_CLOSED') {
return this.stopWorker(worker);
} else {
_logger().default.error(error, '@parcel/workers');
}
}
startChild() {
let worker = new _Worker.default({
forcedKillTime: this.options.forcedKillTime,
backend: this.options.backend,
shouldPatchConsole: this.options.shouldPatchConsole,
shouldTrace: this.options.shouldTrace,
sharedReferences: this.sharedReferences
});
worker.fork((0, _nullthrows().default)(this.options.workerPath));
worker.on('request', data => this.processRequest(data, worker));
worker.on('ready', () => {
this.readyWorkers++;
if (this.readyWorkers === this.options.maxConcurrentWorkers) {
this.emit('ready');
}
this.processQueue();
});
worker.on('response', () => this.processQueue());
worker.on('error', err => this.onError(err, worker));
worker.once('exit', () => this.stopWorker(worker));
this.workers.set(worker.id, worker);
}
async stopWorker(worker) {
if (!worker.stopped) {
this.workers.delete(worker.id);
worker.isStopping = true;
if (worker.calls.size) {
for (let call of worker.calls.values()) {
call.retries++;
this.callQueue.unshift(call);
}
}
worker.calls.clear();
await worker.stop();
// Process any requests that failed and start a new worker
this.processQueue();
}
}
processQueue() {
if (this.ending || !this.callQueue.length) return;
if (this.workers.size < this.options.maxConcurrentWorkers) {
this.startChild();
}
let workers = [...this.workers.values()].sort((a, b) => a.calls.size - b.calls.size);
for (let worker of workers) {
if (!this.callQueue.length) {
break;
}
if (!worker.ready || worker.stopped || worker.isStopping) {
continue;
}
if (worker.calls.size < this.options.maxConcurrentCallsPerWorker) {
this.callWorker(worker, this.callQueue.shift());
}
}
}
async callWorker(worker, call) {
for (let ref of this.sharedReferences.keys()) {
if (!worker.sentSharedReferences.has(ref)) {
await worker.sendSharedReference(ref, this.getSerializedSharedReference(ref));
}
}
worker.call(call);
}
async processRequest(data, worker) {
let {
method,
args,
location,
awaitResponse,
idx,
handle: handleId
} = data;
let mod;
if (handleId != null) {
var _this$handles$get;
mod = (0, _nullthrows().default)((_this$handles$get = this.handles.get(handleId)) === null || _this$handles$get === void 0 ? void 0 : _this$handles$get.fn);
} else if (location) {
// $FlowFixMe
if (process.browser) {
if (location === '@parcel/workers/src/bus.js') {
mod = bus;
} else {
throw new Error('No dynamic require possible: ' + location);
}
} else {
// $FlowFixMe this must be dynamic
mod = require(location);
}
} else {
throw new Error('Unknown request');
}
const responseFromContent = content => ({
idx,
type: 'response',
contentType: 'data',
content
});
const errorResponseFromError = e => ({
idx,
type: 'response',
contentType: 'error',
content: (0, _diagnostic().anyToDiagnostic)(e)
});
let result;
if (method == null) {
try {
result = responseFromContent(await mod(...args));
} catch (e) {
result = errorResponseFromError(e);
}
} else {
// ESModule default interop
if (mod.__esModule && !mod[method] && mod.default) {
mod = mod.default;
}
try {
// $FlowFixMe
result = responseFromContent(await mod[method](...args));
} catch (e) {
result = errorResponseFromError(e);
}
}
if (awaitResponse) {
if (worker) {
worker.send(result);
} else {
if (result.contentType === 'error') {
throw new (_diagnostic().default)({
diagnostic: result.content
});
}
return result.content;
}
}
}
addCall(method, args) {
if (this.ending) {
throw new Error('Cannot add a worker call if workerfarm is ending.');
}
return new Promise((resolve, reject) => {
this.callQueue.push({
method,
args: args,
retries: 0,
resolve,
reject
});
this.processQueue();
});
}
async end() {
this.ending = true;
await Promise.all(Array.from(this.workers.values()).map(worker => this.stopWorker(worker)));
for (let handle of this.handles.values()) {
handle.dispose();
}
this.handles = new Map();
this.sharedReferences = new Map();
this.sharedReferencesByValue = new Map();
this.ending = false;
}
startMaxWorkers() {
// Starts workers until the maximum is reached
if (this.workers.size < this.options.maxConcurrentWorkers) {
let toStart = this.options.maxConcurrentWorkers - this.workers.size;
while (toStart--) {
this.startChild();
}
}
}
shouldUseRemoteWorkers() {
return !this.options.useLocalWorker || (this.warmWorkers >= this.workers.size || !this.options.warmWorkers) && this.options.maxConcurrentWorkers > 0;
}
createReverseHandle(fn) {
let handle = new _Handle.default({
fn
});
this.handles.set(handle.id, handle);
return handle;
}
createSharedReference(value, isCacheable = true) {
let ref = referenceId++;
this.sharedReferences.set(ref, value);
this.sharedReferencesByValue.set(value, ref);
if (!isCacheable) {
this.serializedSharedReferences.set(ref, null);
}
return {
ref,
dispose: () => {
this.sharedReferences.delete(ref);
this.sharedReferencesByValue.delete(value);
this.serializedSharedReferences.delete(ref);
let promises = [];
for (let worker of this.workers.values()) {
if (!worker.sentSharedReferences.has(ref)) {
continue;
}
worker.sentSharedReferences.delete(ref);
promises.push(new Promise((resolve, reject) => {
worker.call({
method: 'deleteSharedReference',
args: [ref],
resolve,
reject,
skipReadyCheck: true,
retries: 0
});
}));
}
return Promise.all(promises);
}
};
}
getSerializedSharedReference(ref) {
let cached = this.serializedSharedReferences.get(ref);
if (cached) {
return cached;
}
let value = this.sharedReferences.get(ref);
let buf = (0, _core().serialize)(value).buffer;
// If the reference was created with the isCacheable option set to false,
// serializedSharedReferences will contain `null` as the value.
if (cached !== null) {
this.serializedSharedReferences.set(ref, buf);
}
return buf;
}
async startProfile() {
let promises = [];
for (let worker of this.workers.values()) {
promises.push(new Promise((resolve, reject) => {
worker.call({
method: 'startProfile',
args: [],
resolve,
reject,
retries: 0,
skipReadyCheck: true
});
}));
}
this.profiler = new (_profiler().SamplingProfiler)();
promises.push(this.profiler.startProfiling());
await Promise.all(promises);
}
async endProfile() {
if (!this.profiler) {
return;
}
let promises = [this.profiler.stopProfiling()];
let names = ['Master'];
for (let worker of this.workers.values()) {
names.push('Worker ' + worker.id);
promises.push(new Promise((resolve, reject) => {
worker.call({
method: 'endProfile',
args: [],
resolve,
reject,
retries: 0,
skipReadyCheck: true
});
}));
}
var profiles = await Promise.all(promises);
let trace = new (_profiler().Trace)();
let filename = `profile-${getTimeId()}.trace`;
let stream = trace.pipe(_fs().default.createWriteStream(filename));
for (let profile of profiles) {
trace.addCPUProfile(names.shift(), profile);
}
trace.flush();
await new Promise(resolve => {
stream.once('finish', resolve);
});
_logger().default.info({
origin: '@parcel/workers',
message: (0, _diagnostic().md)`Wrote profile to ${filename}`
});
}
async callAllWorkers(method, args) {
let promises = [];
for (let worker of this.workers.values()) {
promises.push(new Promise((resolve, reject) => {
worker.call({
method,
args,
resolve,
reject,
retries: 0
});
}));
}
promises.push(this.localWorker[method](this.workerApi, ...args));
await Promise.all(promises);
}
async takeHeapSnapshot() {
let snapshotId = getTimeId();
try {
let snapshotPaths = await Promise.all([...this.workers.values()].map(worker => new Promise((resolve, reject) => {
worker.call({
method: 'takeHeapSnapshot',
args: [snapshotId],
resolve,
reject,
retries: 0,
skipReadyCheck: true
});
})));
_logger().default.info({
origin: '@parcel/workers',
message: (0, _diagnostic().md)`Wrote heap snapshots to the following paths:\n${snapshotPaths.join('\n')}`
});
} catch {
_logger().default.error({
origin: '@parcel/workers',
message: 'Unable to take heap snapshots. Note: requires Node 11.13.0+'
});
}
}
static getNumWorkers() {
return process.env.PARCEL_WORKERS ? parseInt(process.env.PARCEL_WORKERS, 10) : Math.min(4, Math.ceil((0, _cpuCount.default)() / 2));
}
static isWorker() {
return !!_childState.child;
}
static getWorkerApi() {
(0, _assert().default)(_childState.child != null, 'WorkerFarm.getWorkerApi can only be called within workers');
return _childState.child.workerApi;
}
static getConcurrentCallsPerWorker(defaultValue = DEFAULT_MAX_CONCURRENT_CALLS) {
return parseInt(process.env.PARCEL_MAX_CONCURRENT_CALLS, 10) || defaultValue;
}
}
exports.default = WorkerFarm;
function getTimeId() {
let now = new Date();
return String(now.getFullYear()) + String(now.getMonth() + 1).padStart(2, '0') + String(now.getDate()).padStart(2, '0') + '-' + String(now.getHours()).padStart(2, '0') + String(now.getMinutes()).padStart(2, '0') + String(now.getSeconds()).padStart(2, '0');
}

View File

@@ -0,0 +1,34 @@
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.detectBackend = detectBackend;
exports.getWorkerBackend = getWorkerBackend;
function detectBackend() {
// $FlowFixMe
if (process.browser) return 'web';
switch (process.env.PARCEL_WORKER_BACKEND) {
case 'threads':
case 'process':
return process.env.PARCEL_WORKER_BACKEND;
}
try {
require('worker_threads');
return 'threads';
} catch (err) {
return 'process';
}
}
function getWorkerBackend(backend) {
switch (backend) {
case 'threads':
return require('./threads/ThreadsWorker').default;
case 'process':
return require('./process/ProcessWorker').default;
case 'web':
return require('./web/WebWorker').default;
default:
throw new Error(`Invalid backend: ${backend}`);
}
}

View File

@@ -0,0 +1,31 @@
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.default = void 0;
function _events() {
const data = _interopRequireDefault(require("events"));
_events = function () {
return data;
};
return data;
}
var _childState = require("./childState");
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
class Bus extends _events().default {
emit(event, ...args) {
if (_childState.child) {
_childState.child.workerApi.callMaster({
// $FlowFixMe
location: process.browser ? '@parcel/workers/src/bus.js' : __filename,
method: 'emit',
args: [event, ...args]
}, false);
return true;
} else {
return super.emit(event, ...args);
}
}
}
var _default = exports.default = new Bus();

View File

@@ -0,0 +1,295 @@
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.Child = void 0;
var coreWorker = _interopRequireWildcard(require("./core-worker"));
function _assert() {
const data = _interopRequireDefault(require("assert"));
_assert = function () {
return data;
};
return data;
}
function _nullthrows() {
const data = _interopRequireDefault(require("nullthrows"));
_nullthrows = function () {
return data;
};
return data;
}
function _logger() {
const data = _interopRequireWildcard(require("@parcel/logger"));
_logger = function () {
return data;
};
return data;
}
function _diagnostic() {
const data = _interopRequireWildcard(require("@parcel/diagnostic"));
_diagnostic = function () {
return data;
};
return data;
}
function _core() {
const data = require("@parcel/core");
_core = function () {
return data;
};
return data;
}
var _bus = _interopRequireDefault(require("./bus"));
function _profiler() {
const data = require("@parcel/profiler");
_profiler = function () {
return data;
};
return data;
}
var _Handle2 = _interopRequireDefault(require("./Handle"));
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
// The import of './Handle' should really be imported eagerly (with @babel/plugin-transform-modules-commonjs's lazy mode).
const Handle = _Handle2.default;
class Child {
callQueue = [];
maxConcurrentCalls = 10;
responseId = 0;
responseQueue = new Map();
handles = new Map();
sharedReferences = new Map();
sharedReferencesByValue = new Map();
constructor(ChildBackend) {
this.child = new ChildBackend(m => {
this.messageListener(m);
}, () => this.handleEnd());
// Monitior all logging events inside this child process and forward to
// the main process via the bus.
this.loggerDisposable = _logger().default.onLog(event => {
_bus.default.emit('logEvent', event);
});
// .. and do the same for trace events
this.tracerDisposable = _profiler().tracer.onTrace(event => {
_bus.default.emit('traceEvent', event);
});
}
workerApi = {
callMaster: (request, awaitResponse = true) => this.addCall(request, awaitResponse),
createReverseHandle: fn => this.createReverseHandle(fn),
runHandle: (handle, args) => this.workerApi.callMaster({
handle: handle.id,
args
}, true),
getSharedReference: ref => this.sharedReferences.get(ref),
resolveSharedReference: value => this.sharedReferencesByValue.get(value)
};
messageListener(message) {
if (message.type === 'response') {
return this.handleResponse(message);
} else if (message.type === 'request') {
return this.handleRequest(message);
}
}
send(data) {
this.child.send(data);
}
async childInit(module, childId) {
// $FlowFixMe
if (process.browser) {
if (module === '@parcel/core/src/worker.js') {
this.module = coreWorker;
} else {
throw new Error('No dynamic require possible: ' + module);
}
} else {
// $FlowFixMe this must be dynamic
this.module = require(module);
}
this.childId = childId;
if (this.module.childInit != null) {
await this.module.childInit();
}
}
async handleRequest(data) {
let {
idx,
method,
args,
handle: handleId
} = data;
let child = (0, _nullthrows().default)(data.child);
const responseFromContent = content => ({
idx,
child,
type: 'response',
contentType: 'data',
content
});
const errorResponseFromError = e => ({
idx,
child,
type: 'response',
contentType: 'error',
content: (0, _diagnostic().anyToDiagnostic)(e)
});
let result;
if (handleId != null) {
try {
var _this$handles$get;
let fn = (0, _nullthrows().default)((_this$handles$get = this.handles.get(handleId)) === null || _this$handles$get === void 0 ? void 0 : _this$handles$get.fn);
result = responseFromContent(fn(...args));
} catch (e) {
result = errorResponseFromError(e);
}
} else if (method === 'childInit') {
try {
let [moduleName, childOptions] = args;
if (childOptions.shouldPatchConsole) {
(0, _logger().patchConsole)();
} else {
(0, _logger().unpatchConsole)();
}
if (childOptions.shouldTrace) {
_profiler().tracer.enable();
}
result = responseFromContent(await this.childInit(moduleName, child));
} catch (e) {
result = errorResponseFromError(e);
}
} else if (method === 'startProfile') {
this.profiler = new (_profiler().SamplingProfiler)();
try {
result = responseFromContent(await this.profiler.startProfiling());
} catch (e) {
result = errorResponseFromError(e);
}
} else if (method === 'endProfile') {
try {
let res = this.profiler ? await this.profiler.stopProfiling() : null;
result = responseFromContent(res);
} catch (e) {
result = errorResponseFromError(e);
}
} else if (method === 'takeHeapSnapshot') {
try {
let v8 = require('v8');
result = responseFromContent(v8.writeHeapSnapshot('heap-' + args[0] + '-' + (this.childId ? 'worker' + this.childId : 'main') + '.heapsnapshot'));
} catch (e) {
result = errorResponseFromError(e);
}
} else if (method === 'createSharedReference') {
let [ref, _value] = args;
let value = _value instanceof ArrayBuffer ?
// In the case the value is pre-serialized as a buffer,
// deserialize it.
(0, _core().deserialize)(Buffer.from(_value)) : _value;
this.sharedReferences.set(ref, value);
this.sharedReferencesByValue.set(value, ref);
result = responseFromContent(null);
} else if (method === 'deleteSharedReference') {
let ref = args[0];
let value = this.sharedReferences.get(ref);
this.sharedReferencesByValue.delete(value);
this.sharedReferences.delete(ref);
result = responseFromContent(null);
} else {
try {
result = responseFromContent(
// $FlowFixMe
await this.module[method](this.workerApi, ...args));
} catch (e) {
result = errorResponseFromError(e);
}
}
try {
this.send(result);
} catch (e) {
result = this.send(errorResponseFromError(e));
}
}
handleResponse(data) {
let idx = (0, _nullthrows().default)(data.idx);
let contentType = data.contentType;
let content = data.content;
let call = (0, _nullthrows().default)(this.responseQueue.get(idx));
if (contentType === 'error') {
(0, _assert().default)(typeof content !== 'string');
call.reject(new (_diagnostic().default)({
diagnostic: content
}));
} else {
call.resolve(content);
}
this.responseQueue.delete(idx);
// Process the next call
this.processQueue();
}
// Keep in mind to make sure responses to these calls are JSON.Stringify safe
addCall(request, awaitResponse = true) {
var _promise;
// $FlowFixMe
let call = {
...request,
type: 'request',
child: this.childId,
// $FlowFixMe Added in Flow 0.121.0 upgrade in #4381
awaitResponse,
resolve: () => {},
reject: () => {}
};
let promise;
if (awaitResponse) {
promise = new Promise((resolve, reject) => {
call.resolve = resolve;
call.reject = reject;
});
}
this.callQueue.push(call);
this.processQueue();
return (_promise = promise) !== null && _promise !== void 0 ? _promise : Promise.resolve();
}
sendRequest(call) {
let idx;
if (call.awaitResponse) {
idx = this.responseId++;
this.responseQueue.set(idx, call);
}
this.send({
idx,
child: call.child,
type: call.type,
location: call.location,
handle: call.handle,
method: call.method,
args: call.args,
awaitResponse: call.awaitResponse
});
}
processQueue() {
if (!this.callQueue.length) {
return;
}
if (this.responseQueue.size < this.maxConcurrentCalls) {
this.sendRequest(this.callQueue.shift());
}
}
handleEnd() {
this.loggerDisposable.dispose();
this.tracerDisposable.dispose();
}
createReverseHandle(fn) {
let handle = new Handle({
fn,
childId: this.childId
});
this.handles.set(handle.id, handle);
return handle;
}
}
exports.Child = Child;

View File

@@ -0,0 +1,14 @@
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.child = void 0;
exports.setChild = setChild;
// This file is imported by both the WorkerFarm and child implementation.
// When a worker is inited, it sets the state in this file.
// This way, WorkerFarm can access the state without directly importing the child code.
let child = exports.child = null;
function setChild(c) {
exports.child = child = c;
}

View File

@@ -0,0 +1,4 @@
"use strict";
// eslint-disable-next-line monorepo/no-internal-import
module.exports = require('@parcel/core/src/worker.js');

View File

@@ -0,0 +1,4 @@
"use strict";
// This is used only in browser builds
module.exports = {};

View File

@@ -0,0 +1,79 @@
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.default = getCores;
exports.detectRealCores = detectRealCores;
function _os() {
const data = _interopRequireDefault(require("os"));
_os = function () {
return data;
};
return data;
}
function _child_process() {
const data = require("child_process");
_child_process = function () {
return data;
};
return data;
}
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
const exec = command => {
try {
let stdout = (0, _child_process().execSync)(command, {
encoding: 'utf8',
// This prevents the command from outputting to the console
stdio: [null, null, null]
});
return stdout.trim();
} catch (e) {
return '';
}
};
function detectRealCores() {
let platform = _os().default.platform();
let amount = 0;
if (platform === 'linux') {
amount = parseInt(exec('lscpu -p | egrep -v "^#" | sort -u -t, -k 2,4 | wc -l'), 10);
} else if (platform === 'darwin') {
amount = parseInt(exec('sysctl -n hw.physicalcpu_max'), 10);
} else if (platform === 'win32') {
const str = exec('wmic cpu get NumberOfCores').match(/\d+/g);
if (str !== null) {
amount = parseInt(str.filter(n => n !== '')[0], 10);
}
}
if (!amount || amount <= 0) {
throw new Error('Could not detect cpu count!');
}
return amount;
}
let cores;
function getCores(bypassCache = false) {
// Do not re-run commands if we already have the count...
if (cores && !bypassCache) {
return cores;
}
// $FlowFixMe
if (process.browser) {
// eslint-disable-next-line no-undef
cores = navigator.hardwareConcurrency / 2;
}
if (!cores) {
try {
cores = detectRealCores();
} catch (e) {
// Guess the amount of real cores
cores = _os().default.cpus().filter((cpu, index) => !cpu.model.includes('Intel') || index % 2 === 1).length;
}
}
// Another fallback
if (!cores) {
cores = 1;
}
return cores;
}

View File

@@ -0,0 +1,75 @@
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
Object.defineProperty(exports, "Handle", {
enumerable: true,
get: function () {
return _WorkerFarm.Handle;
}
});
Object.defineProperty(exports, "bus", {
enumerable: true,
get: function () {
return _bus.default;
}
});
exports.default = void 0;
function _assert() {
const data = _interopRequireDefault(require("assert"));
_assert = function () {
return data;
};
return data;
}
var _WorkerFarm = _interopRequireWildcard(require("./WorkerFarm"));
function _logger() {
const data = _interopRequireDefault(require("@parcel/logger"));
_logger = function () {
return data;
};
return data;
}
var _bus = _interopRequireDefault(require("./bus"));
function _profiler() {
const data = require("@parcel/profiler");
_profiler = function () {
return data;
};
return data;
}
function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
if (!_WorkerFarm.default.isWorker()) {
// Forward all logger events originating from workers into the main process
_bus.default.on('logEvent', e => {
switch (e.level) {
case 'info':
_logger().default.info(e.diagnostics);
break;
case 'progress':
(0, _assert().default)(typeof e.message === 'string');
_logger().default.progress(e.message);
break;
case 'verbose':
_logger().default.verbose(e.diagnostics);
break;
case 'warn':
_logger().default.warn(e.diagnostics);
break;
case 'error':
_logger().default.error(e.diagnostics);
break;
default:
throw new Error('Unknown log level');
}
});
// Forward all trace events originating from workers into the main process
_bus.default.on('traceEvent', e => {
_profiler().tracer.trace(e);
});
}
var _default = exports.default = _WorkerFarm.default;

View File

@@ -0,0 +1,58 @@
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.default = void 0;
function _nullthrows() {
const data = _interopRequireDefault(require("nullthrows"));
_nullthrows = function () {
return data;
};
return data;
}
var _childState = require("../childState");
var _child = require("../child");
function _core() {
const data = require("@parcel/core");
_core = function () {
return data;
};
return data;
}
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
class ProcessChild {
constructor(onMessage, onExit) {
if (!process.send) {
throw new Error('Only create ProcessChild instances in a worker!');
}
this.onMessage = onMessage;
this.onExit = onExit;
process.on('message', data => this.handleMessage(data));
}
handleMessage(data) {
if (data === 'die') {
return this.stop();
}
this.onMessage((0, _core().deserialize)(Buffer.from(data, 'base64')));
}
send(data) {
let processSend = (0, _nullthrows().default)(process.send).bind(process);
processSend((0, _core().serialize)(data).toString('base64'), err => {
if (err && err instanceof Error) {
// $FlowFixMe[prop-missing]
if (err.code === 'ERR_IPC_CHANNEL_CLOSED') {
// IPC connection closed
// no need to keep the worker running if it can't send or receive data
return this.stop();
}
}
});
}
stop() {
this.onExit(0);
process.exit();
}
}
exports.default = ProcessChild;
(0, _childState.setChild)(new _child.Child(ProcessChild));

View File

@@ -0,0 +1,83 @@
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.default = void 0;
function _child_process() {
const data = _interopRequireDefault(require("child_process"));
_child_process = function () {
return data;
};
return data;
}
function _path() {
const data = _interopRequireDefault(require("path"));
_path = function () {
return data;
};
return data;
}
function _core() {
const data = require("@parcel/core");
_core = function () {
return data;
};
return data;
}
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
const WORKER_PATH = _path().default.join(__dirname, 'ProcessChild.js');
class ProcessWorker {
processQueue = true;
sendQueue = [];
constructor(execArgv, onMessage, onError, onExit) {
this.execArgv = execArgv;
this.onMessage = onMessage;
this.onError = onError;
this.onExit = onExit;
}
start() {
this.child = _child_process().default.fork(WORKER_PATH, process.argv, {
execArgv: this.execArgv,
env: process.env,
cwd: process.cwd()
});
this.child.on('message', data => {
this.onMessage((0, _core().deserialize)(Buffer.from(data, 'base64')));
});
this.child.once('exit', this.onExit);
this.child.on('error', this.onError);
return Promise.resolve();
}
async stop() {
this.child.send('die');
let forceKill = setTimeout(() => this.child.kill('SIGINT'), 500);
await new Promise(resolve => {
this.child.once('exit', resolve);
});
clearTimeout(forceKill);
}
send(data) {
if (!this.processQueue) {
this.sendQueue.push(data);
return;
}
let result = this.child.send((0, _core().serialize)(data).toString('base64'), error => {
if (error && error instanceof Error) {
// Ignore this, the workerfarm handles child errors
return;
}
this.processQueue = true;
if (this.sendQueue.length > 0) {
let queueCopy = this.sendQueue.slice(0);
this.sendQueue = [];
queueCopy.forEach(entry => this.send(entry));
}
});
if (!result || /^win/.test(process.platform)) {
// Queue is handling too much messages throttle it
this.processQueue = false;
}
}
}
exports.default = ProcessWorker;

View File

@@ -0,0 +1,49 @@
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.default = void 0;
function _worker_threads() {
const data = require("worker_threads");
_worker_threads = function () {
return data;
};
return data;
}
function _nullthrows() {
const data = _interopRequireDefault(require("nullthrows"));
_nullthrows = function () {
return data;
};
return data;
}
var _childState = require("../childState");
var _child = require("../child");
function _core() {
const data = require("@parcel/core");
_core = function () {
return data;
};
return data;
}
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
class ThreadsChild {
constructor(onMessage, onExit) {
if (_worker_threads().isMainThread || !_worker_threads().parentPort) {
throw new Error('Only create ThreadsChild instances in a worker!');
}
this.onMessage = onMessage;
this.onExit = onExit;
_worker_threads().parentPort.on('message', data => this.handleMessage(data));
_worker_threads().parentPort.on('close', this.onExit);
}
handleMessage(data) {
this.onMessage((0, _core().restoreDeserializedObject)(data));
}
send(data) {
(0, _nullthrows().default)(_worker_threads().parentPort).postMessage((0, _core().prepareForSerialization)(data));
}
}
exports.default = ThreadsChild;
(0, _childState.setChild)(new _child.Child(ThreadsChild));

View File

@@ -0,0 +1,61 @@
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.default = void 0;
function _worker_threads() {
const data = require("worker_threads");
_worker_threads = function () {
return data;
};
return data;
}
function _path() {
const data = _interopRequireDefault(require("path"));
_path = function () {
return data;
};
return data;
}
function _core() {
const data = require("@parcel/core");
_core = function () {
return data;
};
return data;
}
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
const WORKER_PATH = _path().default.join(__dirname, 'ThreadsChild.js');
class ThreadsWorker {
constructor(execArgv, onMessage, onError, onExit) {
this.execArgv = execArgv;
this.onMessage = onMessage;
this.onError = onError;
this.onExit = onExit;
}
start() {
this.worker = new (_worker_threads().Worker)(WORKER_PATH, {
execArgv: this.execArgv,
env: process.env
});
this.worker.on('message', data => this.handleMessage(data));
this.worker.on('error', this.onError);
this.worker.on('exit', this.onExit);
return new Promise(resolve => {
this.worker.on('online', resolve);
});
}
stop() {
// In node 12, this returns a promise, but previously it accepted a callback
// TODO: Pass a callback in earlier versions of Node
return Promise.resolve(this.worker.terminate());
}
handleMessage(data) {
this.onMessage((0, _core().restoreDeserializedObject)(data));
}
send(data) {
this.worker.postMessage((0, _core().prepareForSerialization)(data));
}
}
exports.default = ThreadsWorker;

View File

@@ -0,0 +1 @@
"use strict";

View File

@@ -0,0 +1,44 @@
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.default = void 0;
var _childState = require("../childState");
var _child = require("../child");
function _core() {
const data = require("@parcel/core");
_core = function () {
return data;
};
return data;
}
/* eslint-env worker*/
class WebChild {
constructor(onMessage, onExit) {
if (!(typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScope)) {
throw new Error('Only create WebChild instances in a worker!');
}
this.onMessage = onMessage;
this.onExit = onExit;
self.addEventListener('message', ({
data
}) => {
if (data === 'stop') {
this.onExit(0);
self.postMessage('stopped');
}
// $FlowFixMe assume WorkerMessage as data
this.handleMessage(data);
});
self.postMessage('online');
}
handleMessage(data) {
this.onMessage((0, _core().restoreDeserializedObject)(data));
}
send(data) {
self.postMessage((0, _core().prepareForSerialization)(data));
}
}
exports.default = WebChild;
(0, _childState.setChild)(new _child.Child(WebChild));

View File

@@ -0,0 +1,85 @@
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.default = void 0;
function _core() {
const data = require("@parcel/core");
_core = function () {
return data;
};
return data;
}
function _utils() {
const data = require("@parcel/utils");
_utils = function () {
return data;
};
return data;
}
let id = 0;
class WebWorker {
constructor(execArgv, onMessage, onError, onExit) {
this.execArgv = execArgv;
this.onMessage = onMessage;
this.onError = onError;
this.onExit = onExit;
}
start() {
// $FlowFixMe[incompatible-call]
this.worker = new Worker(new URL('./WebChild.js', import.meta.url), {
name: `Parcel Worker ${id++}`,
type: 'module'
});
let {
deferred,
promise
} = (0, _utils().makeDeferredWithPromise)();
this.worker.onmessage = ({
data
}) => {
if (data === 'online') {
deferred.resolve();
return;
}
// $FlowFixMe assume WorkerMessage as data
this.handleMessage(data);
};
this.worker.onerror = this.onError;
// Web workers can't crash or intentionally stop on their own, apart from stop() below
// this.worker.on('exit', this.onExit);
return promise;
}
stop() {
if (!this.stopping) {
this.stopping = (async () => {
this.worker.postMessage('stop');
let {
deferred,
promise
} = (0, _utils().makeDeferredWithPromise)();
this.worker.addEventListener('message', ({
data
}) => {
if (data === 'stopped') {
deferred.resolve();
}
});
await promise;
this.worker.terminate();
this.onExit(0);
})();
}
return this.stopping;
}
handleMessage(data) {
this.onMessage((0, _core().restoreDeserializedObject)(data));
}
send(data) {
this.worker.postMessage((0, _core().prepareForSerialization)(data));
}
}
exports.default = WebWorker;