docs for 3.10.1 macos

Author: youknowone

.gitignore

@@ -0,0 +1,7 @@
+/target
+__pycache__
+.vscode
+.idea/
+
+/.cargo/config
+

Cargo.lock

@@ -0,0 +1,9 @@
+[package]
+name = "rustpython-doc"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+once_cell = "1.10.0"

Cargo.toml

@@ -0,0 +1,191 @@
+import re
+import sys
+import warnings
+from pydoc import ModuleScanner
+
+
+def scan_modules():
+    """taken from the source code of help('modules')
+
+    https://github.com/python/cpython/blob/63298930fb531ba2bb4f23bc3b915dbf1e17e9e1/Lib/pydoc.py#L2178"""
+    modules = {}
+
+    def callback(path, modname, desc, modules=modules):
+        if modname and modname[-9:] == ".__init__":
+            modname = modname[:-9] + " (package)"
+        if modname.find(".") < 0:
+            modules[modname] = 1
+
+    def onerror(modname):
+        callback(None, modname, None)
+
+    with warnings.catch_warnings():
+        # ignore warnings from importing deprecated modules
+        warnings.simplefilter("ignore")
+        ModuleScanner().run(callback, onerror=onerror)
+    return list(modules.keys())
+
+
+def import_module(module_name):
+    import io
+    from contextlib import redirect_stdout
+
+    # Importing modules causes ('Constant String', 2, None, 4) and
+    # "Hello world!" to be printed to stdout.
+    f = io.StringIO()
+    with warnings.catch_warnings(), redirect_stdout(f):
+        # ignore warnings caused by importing deprecated modules
+        warnings.filterwarnings("ignore", category=DeprecationWarning)
+        try:
+            module = __import__(module_name)
+        except Exception as e:
+            return e
+    return module
+
+
+def is_child(module, item):
+    import inspect
+
+    item_mod = inspect.getmodule(item)
+    return item_mod is module
+
+
+def traverse(module, names, item):
+    import inspect
+
+    has_doc = inspect.ismodule(item) or inspect.isclass(item) or inspect.isbuiltin(item)
+    if has_doc and isinstance(item.__doc__, str):
+        yield names, item.__doc__
+    attr_names = dir(item)
+    for name in attr_names:
+        if name in [
+            "__class__",
+            "__dict__",
+            "__doc__",
+            "__objclass__",
+            "__name__",
+            "__qualname__",
+            "__annotations__",
+        ]:
+            continue
+        try:
+            attr = getattr(item, name)
+        except AttributeError:
+            assert name == "__abstractmethods__", name
+            continue
+
+        if module is item and not is_child(module, attr):
+            continue
+
+        is_type_or_module = (type(attr) is type) or (type(attr) is type(__builtins__))
+        new_names = names.copy()
+        new_names.append(name)
+
+        if item == attr:
+            pass
+        elif not inspect.ismodule(item) and inspect.ismodule(attr):
+            pass
+        elif is_type_or_module:
+            yield from traverse(module, new_names, attr)
+        elif (
+            callable(attr)
+            or not issubclass(type(attr), type)
+            or type(attr).__name__ in ("getset_descriptor", "member_descriptor")
+        ):
+            if inspect.isbuiltin(attr):
+                yield new_names, attr.__doc__
+        else:
+            assert False, (module, new_names, attr, type(attr).__name__)
+
+
+def traverse_all(root):
+    from glob import glob
+    import os.path
+
+    files = (
+        glob(f"{root}/Lib/*")
+        + glob(f"{root}/vm/src/stdlib/*")
+        + glob(f"{root}/stdlib/src/*")
+    )
+    allowlist = set(
+        [os.path.basename(file).lstrip("_").rsplit(".", 1)[0] for file in files]
+    )
+    for denied in ("this", "antigravity"):
+        allowlist.remove(denied)
+
+    for module_name in scan_modules():
+        if module_name.lstrip("_") not in allowlist:
+            print("skipping:", module_name, file=sys.stderr)
+            continue
+        module = import_module(module_name)
+        if hasattr(module, "__cached__"):  # python module
+            continue
+        yield from traverse(module, [module_name], module)
+
+    builtin_types = [
+        type(bytearray().__iter__()),
+        type(bytes().__iter__()),
+        type(dict().__iter__()),
+        type(dict().values().__iter__()),
+        type(dict().items().__iter__()),
+        type(dict().values()),
+        type(dict().items()),
+        type(set().__iter__()),
+        type(list().__iter__()),
+        type(range(0).__iter__()),
+        type(str().__iter__()),
+        type(tuple().__iter__()),
+        type(None),
+    ]
+    for typ in builtin_types:
+        names = ["builtins", typ.__name__]
+        yield names, typ.__doc__
+        yield from traverse(__builtins__, names, typ)
+
+
+def docs(rustpython_path):
+    return ((".".join(names), escape(doc)) for names, doc in traverse_all(rustpython_path))
+
+
+UNICODE_ESCAPE = re.compile(r"\\u([0-9]+)")
+
+
+def escape(doc):
+    if doc is None:
+        return None
+    return re.sub(UNICODE_ESCAPE, r"\\u{\1}", doc)
+
+
+def test_escape():
+    input = r"It provides access to APT\u2019s idea of the"
+    expected = r"It provides access to APT\u{2019}s idea of the"
+    output = escape(input)
+    assert output == expected
+
+
+if __name__ == "__main__":
+    import sys
+    import json
+
+    try:
+        rustpython_path = sys.argv[1]
+    except IndexError:
+        print("1st argument is rustpython source code path")
+        raise SystemExit
+
+    try:
+        out_path = sys.argv[2]
+    except IndexError:
+        out_path = "-"
+
+    def dump(docs):
+        yield "[\n"
+        for name, doc in docs:
+            if doc is None:
+                yield f"    ({json.dumps(name)}, None),\n"
+            else:
+                yield f"    ({json.dumps(name)}, Some({json.dumps(doc)})),\n"
+        yield "]\n"
+
+    out_file = open(out_path, "w") if out_path != "-" else sys.stdout
+    out_file.writelines(dump(docs(rustpython_path)))

docs.inc.rs

@@ -0,0 +1,43 @@
+use once_cell::sync::Lazy;
+use std::collections::HashMap;
+
+pub type Result<'a> = std::result::Result<Option<&'a str>, ()>;
+
+pub struct Database<'a> {
+    inner: HashMap<&'a str, Option<&'a str>>,
+}
+
+impl<'a> Database<'a> {
+    pub fn shared() -> &'static Self {
+        static DATABASE: Lazy<Database> = Lazy::new(|| {
+            let data = include!("../docs.inc.rs");
+            let mut map = HashMap::with_capacity(data.len());
+            for (item, doc) in data {
+                map.insert(item, doc);
+            }
+            Database { inner: map }
+        });
+        &DATABASE
+    }
+
+    pub fn try_path(&self, path: &str) -> Result {
+        self.inner.get(path).copied().ok_or(())
+    }
+
+    pub fn try_module_item(&self, module: &str, item: &str) -> Result {
+        self.try_path(&format!("{}.{}", module, item))
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_module_item() {
+        let doc = Database::shared()
+            .try_module_item("array", "_array_reconstructor")
+            .unwrap();
+        assert!(doc.is_some());
+    }
+}