larry babby and threejs for glsl

This commit is contained in:
Sam
2024-06-24 21:24:00 +12:00
parent 87d5dc634d
commit 907ebae4c0
6474 changed files with 1279596 additions and 8 deletions

View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020 Kris Zyp
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@@ -0,0 +1,5 @@
## Summary
This module is designed to do fast and efficient native/C-level extraction of strings from MessagePack binary data. This works by calling `extractStrings(buffer, start, end)`, and it will extract strings by doing partial MessagePack parsing, and scanning to find the string data in the range specified in the buffer. It will return an array of strings that it finds. When it finds strings that can be represented with latin-1/one-byte strings (and important V8 optimization), it will attempt return a continuous string of MessagePack data that contains multiple sub-strings, so the decoder can slice off strings by offset. When a string contains non-latin characters, and must be represented as a two-byte string, this will always be returned as the string alone without combination with any other strings. The extractor will return an array of a maximum of 256 strings. The decoder can call the extractStrings again, with a new offset to continue extracting more strings as necessary.
## License
MIT

View File

@@ -0,0 +1,13 @@
#!/usr/bin/env node
const { dirname } = require('path');
const { fileURLToPath } = require('url');
const { exec } = require('child_process');
process.chdir(dirname(__dirname));
exec('prebuildify-ci download', (error, stdout, stderr) => {
console.error(stderr);
console.log(stdout);
if (error?.code)
process.exit(error.code);
});

View File

@@ -0,0 +1,63 @@
{
"variables": {
"os_linux_compiler%": "gcc",
"enable_v8%": "true",
"enable_pointer_compression%": "false",
"build_v8_with_gn": "false"
},
"conditions": [
['OS=="win"', {
"variables": {
"enable_v8%": "<!(echo %ENABLE_V8_FUNCTIONS%)",
}
}],
['OS!="win"', {
"variables": {
"enable_v8%": "<!(echo $ENABLE_V8_FUNCTIONS)",
}
}]
],
"targets": [
{
"target_name": "extract",
"sources": [
"src/extract.cpp",
],
"defines": [ "NAPI_DISABLE_CPP_EXCEPTIONS" ],
"conditions": [
["OS=='linux'", {
"variables": {
"gcc_version" : "<!(<(os_linux_compiler) -dumpversion | cut -d '.' -f 1)",
},
"cflags_cc": [
"-fPIC",
"-fvisibility=hidden",
"-fvisibility-inlines-hidden",
],
"conditions": [
["gcc_version>=7", {
"cflags": [
"-Wimplicit-fallthrough=2",
],
}],
],
"ldflags": [
"-fPIC",
"-fvisibility=hidden"
],
"cflags": [
"-fPIC",
"-fvisibility=hidden",
"-O3"
],
}],
["enable_v8!='false'", {
"defines": ["ENABLE_V8_API=1"]
}],
["enable_pointer_compression=='true'", {
"defines": ["V8_COMPRESS_POINTERS", "V8_COMPRESS_POINTERS_IN_ISOLATE_CAGE"],
}],
],
}
]
}

View File

@@ -0,0 +1 @@
module.exports = require('node-gyp-build-optional-packages')(__dirname);

View File

@@ -0,0 +1,16 @@
#!/bin/sh
basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')")
case `uname` in
*CYGWIN*|*MINGW*|*MSYS*)
if command -v cygpath > /dev/null 2>&1; then
basedir=`cygpath -w "$basedir"`
fi
;;
esac
if [ -x "$basedir/node" ]; then
exec "$basedir/node" "$basedir/../node-gyp-build-optional-packages/bin.js" "$@"
else
exec node "$basedir/../node-gyp-build-optional-packages/bin.js" "$@"
fi

View File

@@ -0,0 +1,16 @@
#!/bin/sh
basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')")
case `uname` in
*CYGWIN*|*MINGW*|*MSYS*)
if command -v cygpath > /dev/null 2>&1; then
basedir=`cygpath -w "$basedir"`
fi
;;
esac
if [ -x "$basedir/node" ]; then
exec "$basedir/node" "$basedir/../node-gyp-build-optional-packages/optional.js" "$@"
else
exec node "$basedir/../node-gyp-build-optional-packages/optional.js" "$@"
fi

View File

@@ -0,0 +1,17 @@
@ECHO off
GOTO start
:find_dp0
SET dp0=%~dp0
EXIT /b
:start
SETLOCAL
CALL :find_dp0
IF EXIST "%dp0%\node.exe" (
SET "_prog=%dp0%\node.exe"
) ELSE (
SET "_prog=node"
SET PATHEXT=%PATHEXT:;.JS;=;%
)
endLocal & goto #_undefined_# 2>NUL || title %COMSPEC% & "%_prog%" "%dp0%\..\node-gyp-build-optional-packages\optional.js" %*

View File

@@ -0,0 +1,28 @@
#!/usr/bin/env pwsh
$basedir=Split-Path $MyInvocation.MyCommand.Definition -Parent
$exe=""
if ($PSVersionTable.PSVersion -lt "6.0" -or $IsWindows) {
# Fix case when both the Windows and Linux builds of Node
# are installed in the same directory
$exe=".exe"
}
$ret=0
if (Test-Path "$basedir/node$exe") {
# Support pipeline input
if ($MyInvocation.ExpectingInput) {
$input | & "$basedir/node$exe" "$basedir/../node-gyp-build-optional-packages/optional.js" $args
} else {
& "$basedir/node$exe" "$basedir/../node-gyp-build-optional-packages/optional.js" $args
}
$ret=$LASTEXITCODE
} else {
# Support pipeline input
if ($MyInvocation.ExpectingInput) {
$input | & "node$exe" "$basedir/../node-gyp-build-optional-packages/optional.js" $args
} else {
& "node$exe" "$basedir/../node-gyp-build-optional-packages/optional.js" $args
}
$ret=$LASTEXITCODE
}
exit $ret

View File

@@ -0,0 +1,16 @@
#!/bin/sh
basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')")
case `uname` in
*CYGWIN*|*MINGW*|*MSYS*)
if command -v cygpath > /dev/null 2>&1; then
basedir=`cygpath -w "$basedir"`
fi
;;
esac
if [ -x "$basedir/node" ]; then
exec "$basedir/node" "$basedir/../node-gyp-build-optional-packages/build-test.js" "$@"
else
exec node "$basedir/../node-gyp-build-optional-packages/build-test.js" "$@"
fi

View File

@@ -0,0 +1,17 @@
@ECHO off
GOTO start
:find_dp0
SET dp0=%~dp0
EXIT /b
:start
SETLOCAL
CALL :find_dp0
IF EXIST "%dp0%\node.exe" (
SET "_prog=%dp0%\node.exe"
) ELSE (
SET "_prog=node"
SET PATHEXT=%PATHEXT:;.JS;=;%
)
endLocal & goto #_undefined_# 2>NUL || title %COMSPEC% & "%_prog%" "%dp0%\..\node-gyp-build-optional-packages\build-test.js" %*

View File

@@ -0,0 +1,28 @@
#!/usr/bin/env pwsh
$basedir=Split-Path $MyInvocation.MyCommand.Definition -Parent
$exe=""
if ($PSVersionTable.PSVersion -lt "6.0" -or $IsWindows) {
# Fix case when both the Windows and Linux builds of Node
# are installed in the same directory
$exe=".exe"
}
$ret=0
if (Test-Path "$basedir/node$exe") {
# Support pipeline input
if ($MyInvocation.ExpectingInput) {
$input | & "$basedir/node$exe" "$basedir/../node-gyp-build-optional-packages/build-test.js" $args
} else {
& "$basedir/node$exe" "$basedir/../node-gyp-build-optional-packages/build-test.js" $args
}
$ret=$LASTEXITCODE
} else {
# Support pipeline input
if ($MyInvocation.ExpectingInput) {
$input | & "node$exe" "$basedir/../node-gyp-build-optional-packages/build-test.js" $args
} else {
& "node$exe" "$basedir/../node-gyp-build-optional-packages/build-test.js" $args
}
$ret=$LASTEXITCODE
}
exit $ret

View File

@@ -0,0 +1,17 @@
@ECHO off
GOTO start
:find_dp0
SET dp0=%~dp0
EXIT /b
:start
SETLOCAL
CALL :find_dp0
IF EXIST "%dp0%\node.exe" (
SET "_prog=%dp0%\node.exe"
) ELSE (
SET "_prog=node"
SET PATHEXT=%PATHEXT:;.JS;=;%
)
endLocal & goto #_undefined_# 2>NUL || title %COMSPEC% & "%_prog%" "%dp0%\..\node-gyp-build-optional-packages\bin.js" %*

View File

@@ -0,0 +1,28 @@
#!/usr/bin/env pwsh
$basedir=Split-Path $MyInvocation.MyCommand.Definition -Parent
$exe=""
if ($PSVersionTable.PSVersion -lt "6.0" -or $IsWindows) {
# Fix case when both the Windows and Linux builds of Node
# are installed in the same directory
$exe=".exe"
}
$ret=0
if (Test-Path "$basedir/node$exe") {
# Support pipeline input
if ($MyInvocation.ExpectingInput) {
$input | & "$basedir/node$exe" "$basedir/../node-gyp-build-optional-packages/bin.js" $args
} else {
& "$basedir/node$exe" "$basedir/../node-gyp-build-optional-packages/bin.js" $args
}
$ret=$LASTEXITCODE
} else {
# Support pipeline input
if ($MyInvocation.ExpectingInput) {
$input | & "node$exe" "$basedir/../node-gyp-build-optional-packages/bin.js" $args
} else {
& "node$exe" "$basedir/../node-gyp-build-optional-packages/bin.js" $args
}
$ret=$LASTEXITCODE
}
exit $ret

View File

@@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2017 Mathias Buus
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

View File

@@ -0,0 +1,58 @@
# node-gyp-build
> Build tool and bindings loader for [`node-gyp`][node-gyp] that supports prebuilds.
```
npm install node-gyp-build
```
[![Test](https://github.com/prebuild/node-gyp-build/actions/workflows/test.yml/badge.svg)](https://github.com/prebuild/node-gyp-build/actions/workflows/test.yml)
Use together with [`prebuildify`][prebuildify] to easily support prebuilds for your native modules.
## Usage
> **Note.** Prebuild names have changed in [`prebuildify@3`][prebuildify] and `node-gyp-build@4`. Please see the documentation below.
`node-gyp-build` works similar to [`node-gyp build`][node-gyp] except that it will check if a build or prebuild is present before rebuilding your project.
It's main intended use is as an npm install script and bindings loader for native modules that bundle prebuilds using [`prebuildify`][prebuildify].
First add `node-gyp-build` as an install script to your native project
``` js
{
...
"scripts": {
"install": "node-gyp-build"
}
}
```
Then in your `index.js`, instead of using the [`bindings`](https://www.npmjs.com/package/bindings) module use `node-gyp-build` to load your binding.
``` js
var binding = require('node-gyp-build')(__dirname)
```
If you do these two things and bundle prebuilds with [`prebuildify`][prebuildify] your native module will work for most platforms
without having to compile on install time AND will work in both node and electron without the need to recompile between usage.
Users can override `node-gyp-build` and force compiling by doing `npm install --build-from-source`.
Prebuilds will be attempted loaded from `MODULE_PATH/prebuilds/...` and then next `EXEC_PATH/prebuilds/...` (the latter allowing use with `zeit/pkg`)
## Supported prebuild names
If so desired you can bundle more specific flavors, for example `musl` builds to support Alpine, or targeting a numbered ARM architecture version.
These prebuilds can be bundled in addition to generic prebuilds; `node-gyp-build` will try to find the most specific flavor first. Prebuild filenames are composed of _tags_. The runtime tag takes precedence, as does an `abi` tag over `napi`. For more details on tags, please see [`prebuildify`][prebuildify].
Values for the `libc` and `armv` tags are auto-detected but can be overridden through the `LIBC` and `ARM_VERSION` environment variables, respectively.
## License
MIT
[prebuildify]: https://github.com/prebuild/prebuildify
[node-gyp]: https://www.npmjs.com/package/node-gyp

View File

@@ -0,0 +1,77 @@
#!/usr/bin/env node
var proc = require('child_process')
var os = require('os')
var path = require('path')
if (!buildFromSource()) {
proc.exec('node-gyp-build-optional-packages-test', function (err, stdout, stderr) {
if (err) {
if (verbose()) console.error(stderr)
preinstall()
}
})
} else {
preinstall()
}
function build () {
var args = [os.platform() === 'win32' ? 'node-gyp.cmd' : 'node-gyp', 'rebuild']
try {
args = [
process.execPath,
path.join(require.resolve('node-gyp/package.json'), '..', require('node-gyp/package.json').bin['node-gyp']),
'rebuild'
]
} catch (_) {}
proc.spawn(args[0], args.slice(1), { stdio: 'inherit' }).on('exit', function (code) {
if (code || !process.argv[3]) process.exit(code)
exec(process.argv[3]).on('exit', function (code) {
process.exit(code)
})
})
}
function preinstall () {
if (!process.argv[2]) return build()
exec(process.argv[2]).on('exit', function (code) {
if (code) process.exit(code)
build()
})
}
function exec (cmd) {
if (process.platform !== 'win32') {
var shell = os.platform() === 'android' ? 'sh' : '/bin/sh'
return proc.spawn(shell, ['-c', '--', cmd], {
stdio: 'inherit'
})
}
return proc.spawn(process.env.comspec || 'cmd.exe', ['/s', '/c', '"' + cmd + '"'], {
windowsVerbatimArguments: true,
stdio: 'inherit'
})
}
function buildFromSource () {
return hasFlag('--build-from-source') || process.env.npm_config_build_from_source === 'true'
}
function verbose () {
return hasFlag('--verbose') || process.env.npm_config_loglevel === 'verbose'
}
// TODO (next major): remove in favor of env.npm_config_* which works since npm
// 0.1.8 while npm_config_argv will stop working in npm 7. See npm/rfcs#90
function hasFlag (flag) {
if (!process.env.npm_config_argv) return false
try {
return JSON.parse(process.env.npm_config_argv).original.indexOf(flag) !== -1
} catch (_) {
return false
}
}

View File

@@ -0,0 +1,19 @@
#!/usr/bin/env node
process.env.NODE_ENV = 'test'
var path = require('path')
var test = null
try {
var pkg = require(path.join(process.cwd(), 'package.json'))
if (pkg.name && process.env[pkg.name.toUpperCase().replace(/-/g, '_')]) {
process.exit(0)
}
test = pkg.prebuild.test
} catch (err) {
// do nothing
}
if (test) require(path.join(process.cwd(), test))
else require('./')()

View File

@@ -0,0 +1,221 @@
var fs = require('fs')
var path = require('path')
var url = require('url')
var vars = (process.config && process.config.variables) || {}
var prebuildsOnly = !!process.env.PREBUILDS_ONLY
var versions = process.versions
var abi = versions.modules
if (versions.deno || process.isBun) {
// both Deno and Bun made the very poor decision to shoot themselves in the foot and lie about support for ABI
// (which they do not have)
abi = 'unsupported'
}
var runtime = isElectron() ? 'electron' : 'node'
var arch = process.arch
var platform = process.platform
var libc = process.env.LIBC || (isAlpine(platform) ? 'musl' : 'glibc')
var armv = process.env.ARM_VERSION || (arch === 'arm64' ? '8' : vars.arm_version) || ''
var uv = (versions.uv || '').split('.')[0]
module.exports = load
function load (dir) {
// Workaround to fix webpack's build warnings: 'the request of a dependency is an expression', but without
// reassigning require in a way that breaks Bun.
if (typeof __webpack_require__ === 'function')
return __non_webpack_require__(load.path(dir))
else
return require(load.path(dir))
}
load.path = function (dir) {
dir = path.resolve(dir || '.')
var packageName
try {
// explanation above
if (typeof __webpack_require__ === 'function')
packageName = __non_webpack_require__(path.join(dir, 'package.json')).name
else
packageName = require(path.join(dir, 'package.json')).name
var varName = packageName.toUpperCase().replace(/-/g, '_') + '_PREBUILD'
if (process.env[varName]) dir = process.env[varName]
} catch (err) {}
if (!prebuildsOnly) {
var release = getFirst(path.join(dir, 'build/Release'), matchBuild)
if (release) return release
var debug = getFirst(path.join(dir, 'build/Debug'), matchBuild)
if (debug) return debug
}
var prebuild = resolve(dir)
if (prebuild) return prebuild
var nearby = resolve(path.dirname(process.execPath))
if (nearby) return nearby
var platformPackage = (packageName[0] == '@' ? '' : '@' + packageName + '/') + packageName + '-' + platform + '-' + arch
try {
var prebuildPackage = path.dirname(require('module').createRequire(url.pathToFileURL(path.join(dir, 'package.json'))).resolve(platformPackage))
return resolveFile(prebuildPackage)
} catch(error) {}
var target = [
'platform=' + platform,
'arch=' + arch,
'runtime=' + runtime,
'abi=' + abi,
'uv=' + uv,
armv ? 'armv=' + armv : '',
'libc=' + libc,
'node=' + process.versions.node,
process.versions.electron ? 'electron=' + process.versions.electron : '',
typeof __webpack_require__ === 'function' ? 'webpack=true' : '' // eslint-disable-line
].filter(Boolean).join(' ')
throw new Error('No native build was found for ' + target + '\n loaded from: ' + dir + ' and package: ' + platformPackage + '\n')
function resolve (dir) {
// Find matching "prebuilds/<platform>-<arch>" directory
var tuples = readdirSync(path.join(dir, 'prebuilds')).map(parseTuple)
var tuple = tuples.filter(matchTuple(platform, arch)).sort(compareTuples)[0]
if (!tuple) return
return resolveFile(path.join(dir, 'prebuilds', tuple.name))
}
function resolveFile (prebuilds) {
// Find most specific flavor first
var parsed = readdirSync(prebuilds).map(parseTags)
var candidates = parsed.filter(matchTags(runtime, abi))
var winner = candidates.sort(compareTags(runtime))[0]
if (winner) return path.join(prebuilds, winner.file)
}
}
function readdirSync (dir) {
try {
return fs.readdirSync(dir)
} catch (err) {
return []
}
}
function getFirst (dir, filter) {
var files = readdirSync(dir).filter(filter)
return files[0] && path.join(dir, files[0])
}
function matchBuild (name) {
return /\.node$/.test(name)
}
function parseTuple (name) {
// Example: darwin-x64+arm64
var arr = name.split('-')
if (arr.length !== 2) return
var platform = arr[0]
var architectures = arr[1].split('+')
if (!platform) return
if (!architectures.length) return
if (!architectures.every(Boolean)) return
return { name, platform, architectures }
}
function matchTuple (platform, arch) {
return function (tuple) {
if (tuple == null) return false
if (tuple.platform !== platform) return false
return tuple.architectures.includes(arch)
}
}
function compareTuples (a, b) {
// Prefer single-arch prebuilds over multi-arch
return a.architectures.length - b.architectures.length
}
function parseTags (file) {
var arr = file.split('.')
var extension = arr.pop()
var tags = { file: file, specificity: 0 }
if (extension !== 'node') return
for (var i = 0; i < arr.length; i++) {
var tag = arr[i]
if (tag === 'node' || tag === 'electron' || tag === 'node-webkit') {
tags.runtime = tag
} else if (tag === 'napi') {
tags.napi = true
} else if (tag.slice(0, 3) === 'abi') {
tags.abi = tag.slice(3)
} else if (tag.slice(0, 2) === 'uv') {
tags.uv = tag.slice(2)
} else if (tag.slice(0, 4) === 'armv') {
tags.armv = tag.slice(4)
} else if (tag === 'glibc' || tag === 'musl') {
tags.libc = tag
} else {
continue
}
tags.specificity++
}
return tags
}
function matchTags (runtime, abi) {
return function (tags) {
if (tags == null) return false
if (tags.runtime !== runtime && !runtimeAgnostic(tags)) return false
if (tags.abi !== abi && !tags.napi) return false
if (tags.uv && tags.uv !== uv) return false
if (tags.armv && tags.armv !== armv) return false
if (tags.libc && tags.libc !== libc) return false
return true
}
}
function runtimeAgnostic (tags) {
return tags.runtime === 'node' && tags.napi
}
function compareTags (runtime) {
// Precedence: non-agnostic runtime, abi over napi, then by specificity.
return function (a, b) {
if (a.runtime !== b.runtime) {
return a.runtime === runtime ? -1 : 1
} else if (a.abi !== b.abi) {
return a.abi ? -1 : 1
} else if (a.specificity !== b.specificity) {
return a.specificity > b.specificity ? -1 : 1
} else {
return 0
}
}
}
function isElectron () {
if (process.versions && process.versions.electron) return true
if (process.env.ELECTRON_RUN_AS_NODE) return true
return typeof window !== 'undefined' && window.process && window.process.type === 'renderer'
}
function isAlpine (platform) {
return platform === 'linux' && fs.existsSync('/etc/alpine-release')
}
// Exposed for unit tests
// TODO: move to lib
load.parseTags = parseTags
load.matchTags = matchTags
load.compareTags = compareTags
load.parseTuple = parseTuple
load.matchTuple = matchTuple
load.compareTuples = compareTuples

View File

@@ -0,0 +1,7 @@
#!/usr/bin/env node
/*
I am only useful as an install script to make node-gyp not compile for purely optional native deps
*/
process.exit(0)

View File

@@ -0,0 +1,29 @@
{
"name": "node-gyp-build-optional-packages",
"version": "5.0.7",
"description": "Build tool and bindings loader for node-gyp that supports prebuilds",
"main": "index.js",
"devDependencies": {
"array-shuffle": "^1.0.1",
"standard": "^14.0.0",
"tape": "^5.0.0"
},
"scripts": {
"test": "standard && node test"
},
"bin": {
"node-gyp-build-optional-packages": "./bin.js",
"node-gyp-build-optional-packages-optional": "./optional.js",
"node-gyp-build-optional-packages-test": "./build-test.js"
},
"repository": {
"type": "git",
"url": "https://github.com/prebuild/node-gyp-build.git"
},
"author": "Mathias Buus (@mafintosh)",
"license": "MIT",
"bugs": {
"url": "https://github.com/prebuild/node-gyp-build/issues"
},
"homepage": "https://github.com/prebuild/node-gyp-build"
}

View File

@@ -0,0 +1,49 @@
{
"name": "msgpackr-extract",
"author": "Kris Zyp",
"version": "3.0.2",
"description": "Node addon for string extraction for msgpackr",
"license": "MIT",
"repository": {
"type": "git",
"url": "http://github.com/kriszyp/msgpackr-extract"
},
"scripts": {
"install": "node-gyp-build-optional-packages",
"recompile": "node-gyp rebuild",
"before-publish": "prebuildify-ci download && node set-optional-deps.cjs",
"prebuild": "prebuildify-platform-packages --target 18.12.0",
"prebuild-win32": "prebuildify-platform-packages --target 18.12.0 && set ENABLE_V8_FUNCTIONS=false&& prebuildify-platform-packages --platform-packages --napi --target 18.12.0",
"prebuild-libc": "prebuildify-platform-packages --tag-libc --target 18.12.0 && prebuildify-platform-packages --platform-packages --napi --tag-libc --target 16.14.2 && ENABLE_V8_FUNCTIONS=false prebuildify-platform-packages --platform-packages --napi --tag-libc --target 18.12.0",
"prebuild-libc-alpine": "prebuildify-cross --image alpine --tag-libc --target 18.12.0",
"publish-all": "cd prebuilds/win32-x64 && npm publish --access public && cd ../darwin-x64 && npm publish --access public && cd ../darwin-arm64 && npm publish --access public && cd ../linux-x64 && npm publish --access public && cd ../linux-arm64 && npm publish --access public && cd ../linux-arm && npm publish --access public && cd ../.. && npm publish --access public",
"test": "node ./index.js"
},
"main": "./index.js",
"gypfile": true,
"dependencies": {
"node-gyp-build-optional-packages": "5.0.7"
},
"files": [
"index.js",
"/src",
"/*.gyp",
"/bin"
],
"bin": {
"download-msgpackr-prebuilds": "./bin/download-prebuilds.js"
},
"optionalDependencies": {
"@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.2",
"@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.2",
"@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.2",
"@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.2",
"@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.2",
"@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.2"
},
"devDependencies": {
"prebuildify-platform-packages": "5.0.2",
"prebuildify-ci": "^1.0.5",
"prebuildify-cross": "5.0.0"
}
}

View File

@@ -0,0 +1,274 @@
/*
This is responsible for extracting the strings, in bulk, from a MessagePack buffer. Creating strings from buffers can
be one of the biggest performance bottlenecks of parsing, but creating an array of extracting strings all at once
provides much better performance. This will parse and produce up to 256 strings at once .The JS parser can call this multiple
times as necessary to get more strings. This must be partially capable of parsing MessagePack so it can know where to
find the string tokens and determine their position and length. All strings are decoded as UTF-8.
*/
#include <node_api.h>
#if ENABLE_V8_API
#include <v8.h>
#endif
#ifndef thread_local
#ifdef __GNUC__
# define thread_local __thread
#elif __STDC_VERSION__ >= 201112L
# define thread_local _Thread_local
#elif defined(_MSC_VER)
# define thread_local __declspec(thread)
#else
# define thread_local
#endif
#endif
const int MAX_TARGET_SIZE = 255;
const uint32_t UNEXPECTED_END = 0xffffffff;
typedef uint32_t (*token_handler)(uint8_t* source, uint32_t position, uint32_t size);
token_handler tokenTable[256] = {};
napi_value unexpectedEnd(napi_env env) {
napi_value returnValue;
napi_get_undefined(env, &returnValue);
napi_throw_type_error(env, NULL, "Unexpected end of buffer reading string");
return returnValue;
}
class Extractor {
public:
// napi_ref targetArray; // could consider reenabling this optimization for napi
bool hasTargetArray = false;
uint8_t* source;
uint32_t position = 0;
uint32_t writePosition = 0;
uint32_t stringStart = 0;
uint32_t lastStringEnd = 0;
void readString(napi_env env, uint32_t length, bool allowStringBlocks, napi_value* target) {
uint32_t start = position;
uint32_t end = position + length;
if (allowStringBlocks) { // for larger strings, we don't bother to check every character for being latin, and just go right to creating a new string
while(position < end) {
if (source[position] < 0x80) // ensure we character is latin and can be decoded as one byte
position++;
else {
break;
}
}
}
if (position < end) {
// non-latin character
if (lastStringEnd) {
napi_value value;
napi_create_string_latin1(env, (const char*) source + stringStart, lastStringEnd - stringStart, &value);
target[writePosition++] = value;
lastStringEnd = 0;
}
// use standard utf-8 conversion
napi_value value;
napi_create_string_utf8(env, (const char*) source + start, (int) length, &value);
target[writePosition++] = value;
position = end;
return;
}
if (lastStringEnd) {
if (start - lastStringEnd > 40 || end - stringStart > 6000) {
napi_value value;
napi_create_string_latin1(env, (const char*) source + stringStart, lastStringEnd - stringStart, &value);
target[writePosition++] = value;
stringStart = start;
}
} else {
stringStart = start;
}
lastStringEnd = end;
}
napi_value extractStrings(napi_env env, uint32_t startingPosition, uint32_t size, uint8_t* inputSource) {
napi_value target[MAX_TARGET_SIZE + 1]; // leave one for the queued string
writePosition = 0;
lastStringEnd = 0;
position = startingPosition;
source = inputSource;
while (position < size) {
uint8_t token = source[position++];
if (token < 0xa0) {
// all one byte tokens
} else if (token < 0xc0) {
// fixstr, we want to convert this
token -= 0xa0;
if (token + position > size) {
return unexpectedEnd(env);
}
readString(env, token, true, target);
if (writePosition >= MAX_TARGET_SIZE)
break;
} else if (token <= 0xdb && token >= 0xd9) {
if (token == 0xd9) { //str 8
if (position >= size) {
return unexpectedEnd(env);
}
uint32_t length = source[position++];
if (length + position > size) {
return unexpectedEnd(env);
}
readString(env,length, true, target);
} else if (token == 0xda) { //str 16
if (2 + position > size) {
return unexpectedEnd(env);
}
uint32_t length = source[position++] << 8;
length += source[position++];
if (length + position > size) {
return unexpectedEnd(env);
}
readString(env,length, false, target);
} else { //str 32
if (4 + position > size) {
return unexpectedEnd(env);
}
uint32_t length = source[position++] << 24;
length += source[position++] << 16;
length += source[position++] << 8;
length += source[position++];
if (length + position > size) {
return unexpectedEnd(env);
}
readString(env, length, false, target);
}
if (writePosition >= MAX_TARGET_SIZE)
break;
} else {
auto handle = tokenTable[token];
if ((size_t ) handle < 20) {
position += (size_t ) handle;
} else {
position = tokenTable[token](source, position, size);
if (position == UNEXPECTED_END) {
return unexpectedEnd(env);
}
}
}
}
if (lastStringEnd) {
napi_value value;
napi_create_string_latin1(env, (const char*) source + stringStart, lastStringEnd - stringStart, &value);
if (writePosition == 0) {
return value;
}
target[writePosition++] = value;
} else if (writePosition == 1) {
return target[0];
}
napi_value array;
#if ENABLE_V8_API
v8::Local<v8::Array> v8Array = v8::Array::New(v8::Isolate::GetCurrent(), (v8::Local<v8::Value>*) target, writePosition);
memcpy(&array, &v8Array, sizeof(array));
#else
napi_create_array_with_length(env, writePosition, &array);
for (uint32_t i = 0; i < writePosition; i++) {
napi_set_element(env, array, i, target[i]);
}
#endif
return array;
}
};
void setupTokenTable() {
for (int i = 0; i < 256; i++) {
tokenTable[i] = nullptr;
}
// uint8, int8
tokenTable[0xcc] = tokenTable[0xd0] = (token_handler) 1;
// uint16, int16, array 16, map 16, fixext 1
tokenTable[0xcd] = tokenTable[0xd1] = tokenTable[0xdc] = tokenTable[0xde] = tokenTable[0xd4] = (token_handler) 2;
// fixext 16
tokenTable[0xd5] = (token_handler) 3;
// uint32, int32, float32, array 32, map 32
tokenTable[0xce] = tokenTable[0xd2] = tokenTable[0xca] = tokenTable[0xdd] = tokenTable[0xdf] = (token_handler) 4;
// fixext 4
tokenTable[0xd6] = (token_handler) 5;
// uint64, int64, float64, fixext 8
tokenTable[0xcf] = tokenTable[0xd3] = tokenTable[0xcb] = (token_handler) 8;
// fixext 8
tokenTable[0xd7] = (token_handler) 9;
// fixext 16
tokenTable[0xd8] = (token_handler) 17;
// bin 8
tokenTable[0xc4] = ([](uint8_t* source, uint32_t position, uint32_t size) -> uint32_t {
if (position >= size) {
return UNEXPECTED_END;
}
uint32_t length = source[position++];
return position + length;
});
// bin 16
tokenTable[0xc5] = ([](uint8_t* source, uint32_t position, uint32_t size) -> uint32_t {
if (position + 2 > size) {
return UNEXPECTED_END;
}
uint32_t length = source[position++] << 8;
length += source[position++];
return position + length;
});
// bin 32
tokenTable[0xc6] = ([](uint8_t* source, uint32_t position, uint32_t size) -> uint32_t {
if (position + 4 > size)
return UNEXPECTED_END;
uint32_t length = source[position++] << 24;
length += source[position++] << 16;
length += source[position++] << 8;
length += source[position++];
return position + length;
});
// ext 8
tokenTable[0xc7] = ([](uint8_t* source, uint32_t position, uint32_t size) -> uint32_t {
if (position >= size)
return UNEXPECTED_END;
uint32_t length = source[position++];
position++;
return position + length;
});
// ext 16
tokenTable[0xc8] = ([](uint8_t* source, uint32_t position, uint32_t size) -> uint32_t {
if (position + 2 > size)
return UNEXPECTED_END;
uint32_t length = source[position++] << 8;
length += source[position++];
position++;
return position + length;
});
// ext 32
tokenTable[0xc9] = ([](uint8_t* source, uint32_t position, uint32_t size) -> uint32_t {
if (position + 4 > size)
return UNEXPECTED_END;
uint32_t length = source[position++] << 24;
length += source[position++] << 16;
length += source[position++] << 8;
length += source[position++];
position++;
return position + length;
});
}
static thread_local Extractor* extractor;
napi_value extractStrings(napi_env env, napi_callback_info info) {
size_t argc = 3;
napi_value args[3];
napi_get_cb_info(env, info, &argc, args, NULL, NULL);
uint32_t position;
uint32_t size;
napi_get_value_uint32(env, args[0], &position);
napi_get_value_uint32(env, args[1], &size);
uint8_t* source;
size_t buffer_size;
napi_get_arraybuffer_info(env, args[2], (void**) &source, &buffer_size);
return extractor->extractStrings(env, position, size, source);
}
#define EXPORT_NAPI_FUNCTION(name, func) { napi_property_descriptor desc = { name, 0, func, 0, 0, 0, (napi_property_attributes) (napi_writable | napi_configurable), 0 }; napi_define_properties(env, exports, 1, &desc); }
NAPI_MODULE_INIT() {
extractor = new Extractor(); // create our thread-local extractor
setupTokenTable();
EXPORT_NAPI_FUNCTION("extractStrings", extractStrings);
return exports;
}