mirror/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2025-03-09 12:50:23 -05:00
Code Activity

test wasm

Browse source
This commit is contained in:
c-basalt 2025-01-02 00:17:10 -05:00
parent e8eb983583
commit e0697299b6
4 changed files with 303 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

64
test/test_jsi_external.py
View file

@ -16,10 +16,14 @@
from test.helper import ( from test.helper import (
FakeYDL, FakeYDL,
) )
from yt_dlp.utils import (
variadic,
)
from yt_dlp.cookies import YoutubeDLCookieJar from yt_dlp.cookies import YoutubeDLCookieJar
from yt_dlp.jsinterp.common import ExternalJSI from yt_dlp.jsinterp.common import ExternalJSI, _ALL_FEATURES
from yt_dlp.jsinterp._deno import DenoJSI, DenoJITlessJSI, DenoJSDomJSI from yt_dlp.jsinterp._deno import DenoJSI, DenoJITlessJSI, DenoJSDomJSI
from yt_dlp.jsinterp._phantomjs import PhantomJSJSI from yt_dlp.jsinterp._phantomjs import PhantomJSJSI
from yt_dlp.jsinterp._helper import prepare_wasm_jsmodule
@dataclasses.dataclass @dataclasses.dataclass
@ -49,9 +53,26 @@ def __eq__(self, other: NetscapeFields | http.cookiejar.Cookie):
return all(getattr(self, attr) == getattr(other, attr) for attr in ['name', 'value', 'domain', 'path', 'secure', 'expires']) return all(getattr(self, attr) == getattr(other, attr) for attr in ['name', 'value', 'domain', 'path', 'secure', 'expires'])
covered_features = set()
def requires_feature(features):
covered_features.update(variadic(features))
def outer(func):
def wrapper(self, *args, **kwargs):
if not self.jsi._SUPPORTED_FEATURES.issuperset(variadic(features)):
print(f'{self._JSI_CLASS.__name__} does not support {features!r}, skipping')
self.skipTest(f'{"&".join(variadic(features))} not supported')
return func(self, *args, **kwargs)
return wrapper
return outer
class Base: class Base:
class TestExternalJSI(unittest.TestCase): class TestExternalJSI(unittest.TestCase):
_JSI_CLASS: type[ExternalJSI] = None _JSI_CLASS: type[ExternalJSI] = None
_TESTDATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'testdata', 'jsi_external')
maxDiff = 2000 maxDiff = 2000
def setUp(self): def setUp(self):
@ -77,28 +98,21 @@ def test_user_agent(self):
jsi = self._JSI_CLASS(self.ydl, self.url_param, 10, {}, user_agent='test/ua') jsi = self._JSI_CLASS(self.ydl, self.url_param, 10, {}, user_agent='test/ua')
self.assertEqual(jsi.execute('console.log(navigator.userAgent);'), 'test/ua') self.assertEqual(jsi.execute('console.log(navigator.userAgent);'), 'test/ua')
@requires_feature('location')
def test_location(self): def test_location(self):
if 'location' not in self._JSI_CLASS._SUPPORTED_FEATURES:
print(f'{self._JSI_CLASS.__name__} does not support location, skipping')
self.skipTest('Location not supported')
self.url_param = 'https://example.com/123/456' self.url_param = 'https://example.com/123/456'
self.assertEqual(self.jsi.execute('console.log(JSON.stringify([location.href, location.hostname]));'), self.assertEqual(self.jsi.execute('console.log(JSON.stringify([location.href, location.hostname]));'),
'["https://example.com/123/456","example.com"]') '["https://example.com/123/456","example.com"]')
@requires_feature('dom')
def test_execute_dom_parse(self): def test_execute_dom_parse(self):
if 'dom' not in self.jsi._SUPPORTED_FEATURES:
print(f'{self._JSI_CLASS.__name__} does not support DOM, skipping')
self.skipTest('DOM not supported')
self.assertEqual(self.jsi.execute( self.assertEqual(self.jsi.execute(
'console.log(document.getElementById("test-div").innerHTML);', 'console.log(document.getElementById("test-div").innerHTML);',
html='<html><body><div id="test-div">Hello, world!</div></body></html>'), html='<html><body><div id="test-div">Hello, world!</div></body></html>'),
'Hello, world!') 'Hello, world!')
@requires_feature('dom')
def test_execute_dom_script(self): def test_execute_dom_script(self):
if 'dom' not in self.jsi._SUPPORTED_FEATURES:
print(f'{self._JSI_CLASS.__name__} does not support DOM, skipping')
self.skipTest('DOM not supported')
self.assertEqual(self.jsi.execute( self.assertEqual(self.jsi.execute(
'console.log(document.getElementById("test-div").innerHTML);', 'console.log(document.getElementById("test-div").innerHTML);',
html='''<html><head><title>Hello, world!</title><body> html='''<html><head><title>Hello, world!</title><body>
@ -112,11 +126,8 @@ def test_execute_dom_script(self):
</body></html>'''), </body></html>'''),
'Hello, world!') 'Hello, world!')
@requires_feature(['dom', 'location'])
def test_dom_location(self): def test_dom_location(self):
if not self._JSI_CLASS._SUPPORTED_FEATURES.issuperset({'dom', 'location'}):
print(f'{self._JSI_CLASS.__name__} does not support both DOM and location, skipping')
self.skipTest('DOM or location not supported')
self.url_param = 'https://example.com/123/456' self.url_param = 'https://example.com/123/456'
self.assertEqual(self.jsi.execute( self.assertEqual(self.jsi.execute(
'console.log(document.getElementById("test-div").innerHTML);', 'console.log(document.getElementById("test-div").innerHTML);',
@ -125,10 +136,8 @@ def test_dom_location(self):
<body><div id="test-div">Hello, world!</div></body></html>'''), <body><div id="test-div">Hello, world!</div></body></html>'''),
'example.com') 'example.com')
@requires_feature('cookies')
def test_execute_cookiejar(self): def test_execute_cookiejar(self):
if 'cookies' not in self.jsi._SUPPORTED_FEATURES:
print(f'{self._JSI_CLASS.__name__} does not support cookies, skipping')
self.skipTest('Cookies not supported')
cookiejar = YoutubeDLCookieJar() cookiejar = YoutubeDLCookieJar()
ref_cookiejar = YoutubeDLCookieJar() ref_cookiejar = YoutubeDLCookieJar()
@ -176,6 +185,22 @@ def _assert_expected_execute(cookie_str, ref_cookie_str):
cookiejar=cookiejar), cookiejar=cookiejar),
'test1=new1; test2=new2; test3=test3; test5=test5') 'test1=new1; test2=new2; test3=test3; test5=test5')
@requires_feature('wasm')
def test_wasm(self):
with open(os.path.join(self._TESTDATA_DIR, 'hello_wasm.js')) as f:
js_mod = f.read()
with open(os.path.join(self._TESTDATA_DIR, 'hello_wasm_bg.wasm'), 'rb') as f:
wasm = f.read()
js_base = prepare_wasm_jsmodule(js_mod, wasm)
js_code = js_base + ''';
console.log(add(1, 2));
greet('world');
'''
self.assertEqual(self.jsi.execute(js_code), '3\nHello, world!')
class TestDeno(Base.TestExternalJSI): class TestDeno(Base.TestExternalJSI):
_JSI_CLASS = DenoJSI _JSI_CLASS = DenoJSI
@ -193,5 +218,8 @@ class TestPhantomJS(Base.TestExternalJSI):
_JSI_CLASS = PhantomJSJSI _JSI_CLASS = PhantomJSJSI
expect_covered_features = set(_ALL_FEATURES) - {'js'}
assert covered_features.issuperset(expect_covered_features), f'Missing tests for features: {expect_covered_features - covered_features}'
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

234
test/testdata/jsi_external/hello_wasm.js vendored Normal file
View file

@ -0,0 +1,234 @@
// wasm-pack build --target web
/* lib.rs
use wasm_bindgen::prelude::*;
#[wasm_bindgen]
extern "C" {
pub fn eval(s: &str);
}
#[wasm_bindgen]
pub fn greet(name: &str) {
eval(&format!("console.log('Hello, {}!')", name));
}
#[wasm_bindgen]
pub fn add(left: i32, right: i32) -> i32 {
left + right
}
*/
let wasm;
const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
let cachedUint8ArrayMemory0 = null;
function getUint8ArrayMemory0() {
if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
}
return cachedUint8ArrayMemory0;
}
function getStringFromWasm0(ptr, len) {
ptr = ptr >>> 0;
return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
}
let WASM_VECTOR_LEN = 0;
const cachedTextEncoder = (typeof TextEncoder !== 'undefined' ? new TextEncoder('utf-8') : { encode: () => { throw Error('TextEncoder not available') } } );
const encodeString = (typeof cachedTextEncoder.encodeInto === 'function'
? function (arg, view) {
return cachedTextEncoder.encodeInto(arg, view);
}
: function (arg, view) {
const buf = cachedTextEncoder.encode(arg);
view.set(buf);
return {
read: arg.length,
written: buf.length
};
});
function passStringToWasm0(arg, malloc, realloc) {
if (realloc === undefined) {
const buf = cachedTextEncoder.encode(arg);
const ptr = malloc(buf.length, 1) >>> 0;
getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
WASM_VECTOR_LEN = buf.length;
return ptr;
}
let len = arg.length;
let ptr = malloc(len, 1) >>> 0;
const mem = getUint8ArrayMemory0();
let offset = 0;
for (; offset < len; offset++) {
const code = arg.charCodeAt(offset);
if (code > 0x7F) break;
mem[ptr + offset] = code;
}
if (offset !== len) {
if (offset !== 0) {
arg = arg.slice(offset);
}
ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
const ret = encodeString(arg, view);
offset += ret.written;
ptr = realloc(ptr, len, offset, 1) >>> 0;
}
WASM_VECTOR_LEN = offset;
return ptr;
}
/**
* @param {string} name
*/
export function greet(name) {
const ptr0 = passStringToWasm0(name, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.greet(ptr0, len0);
}
/**
* @param {number} left
* @param {number} right
* @returns {number}
*/
export function add(left, right) {
const ret = wasm.add(left, right);
return ret;
}
async function __wbg_load(module, imports) {
if (typeof Response === 'function' && module instanceof Response) {
if (typeof WebAssembly.instantiateStreaming === 'function') {
try {
return await WebAssembly.instantiateStreaming(module, imports);
} catch (e) {
if (module.headers.get('Content-Type') != 'application/wasm') {
console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e);
} else {
throw e;
}
}
}
const bytes = await module.arrayBuffer();
return await WebAssembly.instantiate(bytes, imports);
} else {
const instance = await WebAssembly.instantiate(module, imports);
if (instance instanceof WebAssembly.Instance) {
return { instance, module };
} else {
return instance;
}
}
}
function __wbg_get_imports() {
const imports = {};
imports.wbg = {};
imports.wbg.__wbg_eval_d1c6d8ede79fdfce = function(arg0, arg1) {
eval(getStringFromWasm0(arg0, arg1));
};
imports.wbg.__wbindgen_init_externref_table = function() {
const table = wasm.__wbindgen_export_0;
const offset = table.grow(4);
table.set(0, undefined);
table.set(offset + 0, undefined);
table.set(offset + 1, null);
table.set(offset + 2, true);
table.set(offset + 3, false);
;
};
return imports;
}
function __wbg_init_memory(imports, memory) {
}
function __wbg_finalize_init(instance, module) {
wasm = instance.exports;
__wbg_init.__wbindgen_wasm_module = module;
cachedUint8ArrayMemory0 = null;
wasm.__wbindgen_start();
return wasm;
}
function initSync(module) {
if (wasm !== undefined) return wasm;
if (typeof module !== 'undefined') {
if (Object.getPrototypeOf(module) === Object.prototype) {
({module} = module)
} else {
console.warn('using deprecated parameters for `initSync()`; pass a single object instead')
}
}
const imports = __wbg_get_imports();
__wbg_init_memory(imports);
if (!(module instanceof WebAssembly.Module)) {
module = new WebAssembly.Module(module);
}
const instance = new WebAssembly.Instance(module, imports);
return __wbg_finalize_init(instance, module);
}
async function __wbg_init(module_or_path) {
if (wasm !== undefined) return wasm;
if (typeof module_or_path !== 'undefined') {
if (Object.getPrototypeOf(module_or_path) === Object.prototype) {
({module_or_path} = module_or_path)
} else {
console.warn('using deprecated parameters for the initialization function; pass a single object instead')
}
}
if (typeof module_or_path === 'undefined') {
module_or_path = new URL('hello_wasm_bg.wasm', import.meta.url);
}
const imports = __wbg_get_imports();
if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) {
module_or_path = fetch(module_or_path);
}
__wbg_init_memory(imports);
const { instance, module } = await __wbg_load(await module_or_path, imports);
return __wbg_finalize_init(instance, module);
}
export { initSync };
export default __wbg_init;

BIN
test/testdata/jsi_external/hello_wasm_bg.wasm vendored Normal file
View file

Binary file not shown.

23
yt_dlp/jsinterp/_helper.py
View file

@ -109,3 +109,26 @@ def extract_script_tags(html: str) -> tuple[str, list[str]]:
html = html[:start] + html[end:] html = html[:start] + html[end:]
return html, inline_scripts return html, inline_scripts
def prepare_wasm_jsmodule(js_mod: str, wasm: bytes) -> str:
"""
Prepare wasm init for js wrapper module generated by rust wasm-pack
removes export and import.meta and inlines wasm binary as Uint8Array
See test/test_data/jsi_external/hello_wasm.js for example
@param {str} js_mod: js wrapper module generated by rust wasm-pack
@param {bytes} wasm: wasm binary
"""
js_mod = re.sub(r'export(?:\s+default)?([\s{])', r'\1', js_mod)
js_mod = js_mod.replace('import.meta', '{}')
return js_mod + ''';
await (async () => {
const t = __wbg_get_imports();
__wbg_init_memory(t);
const {module, instance} = await WebAssembly.instantiate(Uint8Array.from(%s), t);
__wbg_finalize_init(instance, module);
})();
''' % list(wasm)