diff --git a/test/test_jsi_external.py b/test/test_jsi_external.py
index 1d52e3fb3..450b7ca21 100644
--- a/test/test_jsi_external.py
+++ b/test/test_jsi_external.py
@@ -16,10 +16,14 @@
from test.helper import (
FakeYDL,
)
+from yt_dlp.utils import (
+ variadic,
+)
from yt_dlp.cookies import YoutubeDLCookieJar
-from yt_dlp.jsinterp.common import ExternalJSI
+from yt_dlp.jsinterp.common import ExternalJSI, _ALL_FEATURES
from yt_dlp.jsinterp._deno import DenoJSI, DenoJITlessJSI, DenoJSDomJSI
from yt_dlp.jsinterp._phantomjs import PhantomJSJSI
+from yt_dlp.jsinterp._helper import prepare_wasm_jsmodule
@dataclasses.dataclass
@@ -49,9 +53,26 @@ def __eq__(self, other: NetscapeFields | http.cookiejar.Cookie):
return all(getattr(self, attr) == getattr(other, attr) for attr in ['name', 'value', 'domain', 'path', 'secure', 'expires'])
+covered_features = set()
+
+
+def requires_feature(features):
+ covered_features.update(variadic(features))
+
+ def outer(func):
+ def wrapper(self, *args, **kwargs):
+ if not self.jsi._SUPPORTED_FEATURES.issuperset(variadic(features)):
+ print(f'{self._JSI_CLASS.__name__} does not support {features!r}, skipping')
+ self.skipTest(f'{"&".join(variadic(features))} not supported')
+ return func(self, *args, **kwargs)
+ return wrapper
+ return outer
+
+
class Base:
class TestExternalJSI(unittest.TestCase):
_JSI_CLASS: type[ExternalJSI] = None
+ _TESTDATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'testdata', 'jsi_external')
maxDiff = 2000
def setUp(self):
@@ -77,28 +98,21 @@ def test_user_agent(self):
jsi = self._JSI_CLASS(self.ydl, self.url_param, 10, {}, user_agent='test/ua')
self.assertEqual(jsi.execute('console.log(navigator.userAgent);'), 'test/ua')
+ @requires_feature('location')
def test_location(self):
- if 'location' not in self._JSI_CLASS._SUPPORTED_FEATURES:
- print(f'{self._JSI_CLASS.__name__} does not support location, skipping')
- self.skipTest('Location not supported')
self.url_param = 'https://example.com/123/456'
self.assertEqual(self.jsi.execute('console.log(JSON.stringify([location.href, location.hostname]));'),
'["https://example.com/123/456","example.com"]')
+ @requires_feature('dom')
def test_execute_dom_parse(self):
- if 'dom' not in self.jsi._SUPPORTED_FEATURES:
- print(f'{self._JSI_CLASS.__name__} does not support DOM, skipping')
- self.skipTest('DOM not supported')
self.assertEqual(self.jsi.execute(
'console.log(document.getElementById("test-div").innerHTML);',
html='Hello, world!
'),
'Hello, world!')
+ @requires_feature('dom')
def test_execute_dom_script(self):
- if 'dom' not in self.jsi._SUPPORTED_FEATURES:
- print(f'{self._JSI_CLASS.__name__} does not support DOM, skipping')
- self.skipTest('DOM not supported')
-
self.assertEqual(self.jsi.execute(
'console.log(document.getElementById("test-div").innerHTML);',
html='''Hello, world!
@@ -112,11 +126,8 @@ def test_execute_dom_script(self):
'''),
'Hello, world!')
+ @requires_feature(['dom', 'location'])
def test_dom_location(self):
- if not self._JSI_CLASS._SUPPORTED_FEATURES.issuperset({'dom', 'location'}):
- print(f'{self._JSI_CLASS.__name__} does not support both DOM and location, skipping')
- self.skipTest('DOM or location not supported')
-
self.url_param = 'https://example.com/123/456'
self.assertEqual(self.jsi.execute(
'console.log(document.getElementById("test-div").innerHTML);',
@@ -125,10 +136,8 @@ def test_dom_location(self):
Hello, world!
'''),
'example.com')
+ @requires_feature('cookies')
def test_execute_cookiejar(self):
- if 'cookies' not in self.jsi._SUPPORTED_FEATURES:
- print(f'{self._JSI_CLASS.__name__} does not support cookies, skipping')
- self.skipTest('Cookies not supported')
cookiejar = YoutubeDLCookieJar()
ref_cookiejar = YoutubeDLCookieJar()
@@ -176,6 +185,22 @@ def _assert_expected_execute(cookie_str, ref_cookie_str):
cookiejar=cookiejar),
'test1=new1; test2=new2; test3=test3; test5=test5')
+ @requires_feature('wasm')
+ def test_wasm(self):
+ with open(os.path.join(self._TESTDATA_DIR, 'hello_wasm.js')) as f:
+ js_mod = f.read()
+ with open(os.path.join(self._TESTDATA_DIR, 'hello_wasm_bg.wasm'), 'rb') as f:
+ wasm = f.read()
+
+ js_base = prepare_wasm_jsmodule(js_mod, wasm)
+
+ js_code = js_base + ''';
+ console.log(add(1, 2));
+ greet('world');
+ '''
+
+ self.assertEqual(self.jsi.execute(js_code), '3\nHello, world!')
+
class TestDeno(Base.TestExternalJSI):
_JSI_CLASS = DenoJSI
@@ -193,5 +218,8 @@ class TestPhantomJS(Base.TestExternalJSI):
_JSI_CLASS = PhantomJSJSI
+expect_covered_features = set(_ALL_FEATURES) - {'js'}
+assert covered_features.issuperset(expect_covered_features), f'Missing tests for features: {expect_covered_features - covered_features}'
+
if __name__ == '__main__':
unittest.main()
diff --git a/test/testdata/jsi_external/hello_wasm.js b/test/testdata/jsi_external/hello_wasm.js
new file mode 100644
index 000000000..1a3a31c46
--- /dev/null
+++ b/test/testdata/jsi_external/hello_wasm.js
@@ -0,0 +1,234 @@
+// wasm-pack build --target web
+/* lib.rs
+use wasm_bindgen::prelude::*;
+
+#[wasm_bindgen]
+extern "C" {
+ pub fn eval(s: &str);
+}
+
+#[wasm_bindgen]
+pub fn greet(name: &str) {
+ eval(&format!("console.log('Hello, {}!')", name));
+}
+
+#[wasm_bindgen]
+pub fn add(left: i32, right: i32) -> i32 {
+ left + right
+}
+*/
+
+let wasm;
+
+const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
+
+if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
+
+let cachedUint8ArrayMemory0 = null;
+
+function getUint8ArrayMemory0() {
+ if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
+ cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
+ }
+ return cachedUint8ArrayMemory0;
+}
+
+function getStringFromWasm0(ptr, len) {
+ ptr = ptr >>> 0;
+ return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
+}
+
+let WASM_VECTOR_LEN = 0;
+
+const cachedTextEncoder = (typeof TextEncoder !== 'undefined' ? new TextEncoder('utf-8') : { encode: () => { throw Error('TextEncoder not available') } } );
+
+const encodeString = (typeof cachedTextEncoder.encodeInto === 'function'
+ ? function (arg, view) {
+ return cachedTextEncoder.encodeInto(arg, view);
+}
+ : function (arg, view) {
+ const buf = cachedTextEncoder.encode(arg);
+ view.set(buf);
+ return {
+ read: arg.length,
+ written: buf.length
+ };
+});
+
+function passStringToWasm0(arg, malloc, realloc) {
+
+ if (realloc === undefined) {
+ const buf = cachedTextEncoder.encode(arg);
+ const ptr = malloc(buf.length, 1) >>> 0;
+ getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
+ WASM_VECTOR_LEN = buf.length;
+ return ptr;
+ }
+
+ let len = arg.length;
+ let ptr = malloc(len, 1) >>> 0;
+
+ const mem = getUint8ArrayMemory0();
+
+ let offset = 0;
+
+ for (; offset < len; offset++) {
+ const code = arg.charCodeAt(offset);
+ if (code > 0x7F) break;
+ mem[ptr + offset] = code;
+ }
+
+ if (offset !== len) {
+ if (offset !== 0) {
+ arg = arg.slice(offset);
+ }
+ ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
+ const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
+ const ret = encodeString(arg, view);
+
+ offset += ret.written;
+ ptr = realloc(ptr, len, offset, 1) >>> 0;
+ }
+
+ WASM_VECTOR_LEN = offset;
+ return ptr;
+}
+/**
+ * @param {string} name
+ */
+export function greet(name) {
+ const ptr0 = passStringToWasm0(name, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+ const len0 = WASM_VECTOR_LEN;
+ wasm.greet(ptr0, len0);
+}
+
+/**
+ * @param {number} left
+ * @param {number} right
+ * @returns {number}
+ */
+export function add(left, right) {
+ const ret = wasm.add(left, right);
+ return ret;
+}
+
+async function __wbg_load(module, imports) {
+ if (typeof Response === 'function' && module instanceof Response) {
+ if (typeof WebAssembly.instantiateStreaming === 'function') {
+ try {
+ return await WebAssembly.instantiateStreaming(module, imports);
+
+ } catch (e) {
+ if (module.headers.get('Content-Type') != 'application/wasm') {
+ console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e);
+
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ const bytes = await module.arrayBuffer();
+ return await WebAssembly.instantiate(bytes, imports);
+
+ } else {
+ const instance = await WebAssembly.instantiate(module, imports);
+
+ if (instance instanceof WebAssembly.Instance) {
+ return { instance, module };
+
+ } else {
+ return instance;
+ }
+ }
+}
+
+function __wbg_get_imports() {
+ const imports = {};
+ imports.wbg = {};
+ imports.wbg.__wbg_eval_d1c6d8ede79fdfce = function(arg0, arg1) {
+ eval(getStringFromWasm0(arg0, arg1));
+ };
+ imports.wbg.__wbindgen_init_externref_table = function() {
+ const table = wasm.__wbindgen_export_0;
+ const offset = table.grow(4);
+ table.set(0, undefined);
+ table.set(offset + 0, undefined);
+ table.set(offset + 1, null);
+ table.set(offset + 2, true);
+ table.set(offset + 3, false);
+ ;
+ };
+
+ return imports;
+}
+
+function __wbg_init_memory(imports, memory) {
+
+}
+
+function __wbg_finalize_init(instance, module) {
+ wasm = instance.exports;
+ __wbg_init.__wbindgen_wasm_module = module;
+ cachedUint8ArrayMemory0 = null;
+
+
+ wasm.__wbindgen_start();
+ return wasm;
+}
+
+function initSync(module) {
+ if (wasm !== undefined) return wasm;
+
+
+ if (typeof module !== 'undefined') {
+ if (Object.getPrototypeOf(module) === Object.prototype) {
+ ({module} = module)
+ } else {
+ console.warn('using deprecated parameters for `initSync()`; pass a single object instead')
+ }
+ }
+
+ const imports = __wbg_get_imports();
+
+ __wbg_init_memory(imports);
+
+ if (!(module instanceof WebAssembly.Module)) {
+ module = new WebAssembly.Module(module);
+ }
+
+ const instance = new WebAssembly.Instance(module, imports);
+
+ return __wbg_finalize_init(instance, module);
+}
+
+async function __wbg_init(module_or_path) {
+ if (wasm !== undefined) return wasm;
+
+
+ if (typeof module_or_path !== 'undefined') {
+ if (Object.getPrototypeOf(module_or_path) === Object.prototype) {
+ ({module_or_path} = module_or_path)
+ } else {
+ console.warn('using deprecated parameters for the initialization function; pass a single object instead')
+ }
+ }
+
+ if (typeof module_or_path === 'undefined') {
+ module_or_path = new URL('hello_wasm_bg.wasm', import.meta.url);
+ }
+ const imports = __wbg_get_imports();
+
+ if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) {
+ module_or_path = fetch(module_or_path);
+ }
+
+ __wbg_init_memory(imports);
+
+ const { instance, module } = await __wbg_load(await module_or_path, imports);
+
+ return __wbg_finalize_init(instance, module);
+}
+
+export { initSync };
+export default __wbg_init;
diff --git a/test/testdata/jsi_external/hello_wasm_bg.wasm b/test/testdata/jsi_external/hello_wasm_bg.wasm
new file mode 100644
index 000000000..d8f32c44c
Binary files /dev/null and b/test/testdata/jsi_external/hello_wasm_bg.wasm differ
diff --git a/yt_dlp/jsinterp/_helper.py b/yt_dlp/jsinterp/_helper.py
index 9a86c73ad..389204f9e 100644
--- a/yt_dlp/jsinterp/_helper.py
+++ b/yt_dlp/jsinterp/_helper.py
@@ -109,3 +109,26 @@ def extract_script_tags(html: str) -> tuple[str, list[str]]:
html = html[:start] + html[end:]
return html, inline_scripts
+
+
+def prepare_wasm_jsmodule(js_mod: str, wasm: bytes) -> str:
+ """
+ Prepare wasm init for js wrapper module generated by rust wasm-pack
+ removes export and import.meta and inlines wasm binary as Uint8Array
+ See test/test_data/jsi_external/hello_wasm.js for example
+
+ @param {str} js_mod: js wrapper module generated by rust wasm-pack
+ @param {bytes} wasm: wasm binary
+ """
+
+ js_mod = re.sub(r'export(?:\s+default)?([\s{])', r'\1', js_mod)
+ js_mod = js_mod.replace('import.meta', '{}')
+
+ return js_mod + ''';
+ await (async () => {
+ const t = __wbg_get_imports();
+ __wbg_init_memory(t);
+ const {module, instance} = await WebAssembly.instantiate(Uint8Array.from(%s), t);
+ __wbg_finalize_init(instance, module);
+ })();
+ ''' % list(wasm)