diff --git a/CRNS/__init__.py b/CRNS/__init__.py index 49340625..de6c7474 100644 --- a/CRNS/__init__.py +++ b/CRNS/__init__.py @@ -1,12 +1,232 @@ compiling = False noticed = False notice_delay = 0.3 -import time +import os import sys +import time import threading from importlib.util import find_spec if find_spec("pyximport") and find_spec("cython"): - import pyximport; pyxloader = pyximport.install(pyimport=True, language_level=3)[1] + import json + from importlib.abc import MetaPathFinder + from importlib.machinery import ExtensionFileLoader, SourceFileLoader + from importlib.machinery import EXTENSION_SUFFIXES + from importlib.util import spec_from_file_location + from distutils.extension import Extension + + from pyximport import pyxbuild + + class _RNSImportFinder(MetaPathFinder): + """A cythonizing import hook for the RNS package tree. + + pyximport's own ``.py`` hook is unsuitable for this: it only + resolves top-level module names (so every sub-module import + misses the file), and it disables itself after compiling a + single module. + + This finder compiles RNS source modules into a mirror tree + under ``~/.pyxbld/rnslib/`` that exactly mirrors the source + package layout, so that: + + * ``os.path.dirname(__file__)`` of a compiled package + initialiser equals that package's own mirror directory, + keeping the ``__all__`` globs in package ``__init__`` + modules correct when the modules are compiled. + * sub-module resolution through ``__path__`` finds the + compiled extensions in the mirror. + * each loaded module is a C extension (``.so``/``.pyd``). + + The entire RNS source tree is compiled up front (on first use, + and incrementally on later uses), so the mirror is complete + before any RNS module is imported and imported modules can be + loaded straight from it. + + Third-party and standard-library modules are left untouched, + and the ``RNS.vendor`` subtree is deliberately imported from + source. Modules that cannot be compiled fall back to the + interpreted source module with a warning. + """ + + def __init__(self): + self.build_dir = os.path.join(os.path.expanduser("~"), ".pyxbld") + self.mirror_root = os.path.join(self.build_dir, "rnslib") + self.source_root = self._find_source_root() + self.abi_suffix = EXTENSION_SUFFIXES[0] + self.unbuildable = {} + self.unbuildable_path = os.path.join(self.build_dir, "unbuildable.json") + try: + with open(self.unbuildable_path, "r") as fh: + self.unbuildable = json.load(fh) + except Exception: + self.unbuildable = {} + + def _save_unbuildable(self): + try: + os.makedirs(os.path.dirname(self.unbuildable_path), exist_ok=True) + with open(self.unbuildable_path, "w") as fh: + json.dump(self.unbuildable, fh, indent=1) + except Exception: + pass + + def _mark_unbuildable(self, fullname, source): + try: + self.unbuildable[fullname] = int(os.path.getmtime(source)) + self._save_unbuildable() + except Exception: + pass + + def _unmark_buildable(self, fullname): + if fullname in self.unbuildable: + del self.unbuildable[fullname] + self._save_unbuildable() + + def _is_unbuildable(self, fullname, source): + if fullname not in self.unbuildable: + return False + try: + if int(os.path.getmtime(source)) == self.unbuildable[fullname]: + return True + except Exception: + pass + self.unbuildable.pop(fullname, None) + return False + + def _find_source_root(self): + here = os.path.dirname(os.path.abspath(__file__)) + candidate = os.path.abspath(os.path.join(here, "..", "RNS", "__init__.py")) + if os.path.exists(candidate): + return os.path.dirname(os.path.dirname(candidate)) + for entry in [os.getcwd()] + list(sys.path): + if not entry: + continue + candidate = os.path.join(entry, "RNS", "__init__.py") + if os.path.exists(candidate): + return os.path.dirname(candidate) + raise RuntimeError("CRNS could not locate the RNS source tree") + + def _all_source_modules(self): + rns_root = os.path.join(self.source_root, "RNS") + for root, dirs, files in os.walk(rns_root): + dirs[:] = [d for d in dirs if d not in ("__pycache__", "vendor")] + rel = os.path.relpath(root, self.source_root).replace(os.sep, ".") + for fn in sorted(files): + if not fn.endswith(".py"): + continue + if fn == "__init__.py": + fullname = rel + else: + fullname = rel + "." + fn[:-3] + yield fullname + + def _mirror_target(self, fullname, is_package): + rel = fullname.split(".") + basename = rel[-1] + if is_package: + return os.path.join(self.mirror_root, *rel, "__init__" + self.abi_suffix) + return os.path.join(self.mirror_root, *rel[:-1], basename + self.abi_suffix) + + def _source(self, fullname): + rel = fullname.split(".") + package_source = os.path.join(self.source_root, *rel, "__init__.py") + if os.path.exists(package_source): + return package_source, True + module_source = os.path.join(self.source_root, *rel) + ".py" + if os.path.exists(module_source): + return module_source, False + return None, None + + def _compile(self, fullname, source): + basename = fullname.rsplit(".", 1)[-1] + extension = Extension(name=basename, sources=[source]) + so_path = pyxbuild.pyx_to_dll( + source, + ext=extension, + build_in_temp=True, + pyxbuild_dir=self.build_dir, + ) + abi_suffix = os.path.basename(so_path)[len(basename):] + return so_path, abi_suffix + + def _ensure_compiled(self, fullname, source, is_package): + target = self._mirror_target(fullname, is_package) + if os.path.exists(target): + return target + so_path, abi_suffix = self._compile(fullname, source) + os.makedirs(os.path.dirname(target), exist_ok=True) + os.replace(so_path, target) + return target + + def precompile(self): + if not os.path.isdir(self.mirror_root): + os.makedirs(self.mirror_root, exist_ok=True) + for fullname in self._all_source_modules(): + source, is_package = self._source(fullname) + if source is None: + continue + if os.path.exists(self._mirror_target(fullname, is_package)): + self._unmark_buildable(fullname) + continue + if self._is_unbuildable(fullname, source): + continue + try: + self._ensure_compiled(fullname, source, is_package) + self._unmark_buildable(fullname) + except Exception as e: + self._mark_unbuildable(fullname, source) + print(f"CRNS: could not compile {fullname}, " + f"using interpreted module: {e}", file=sys.stderr) + + def _spec(self, fullname, source, is_package, compile_attempted): + if fullname == "RNS.vendor" or fullname.startswith("RNS.vendor."): + rel = fullname.split(".") + locations = [os.path.join(self.source_root, *rel)] if is_package else None + return spec_from_file_location( + fullname, source, + loader=SourceFileLoader(fullname, source), + submodule_search_locations=locations, + ) + + if compile_attempted: + try: + target = self._ensure_compiled(fullname, source, is_package) + self._unmark_buildable(fullname) + if is_package: + locations = [os.path.dirname(target)] + else: + locations = None + return spec_from_file_location( + fullname, target, + loader=ExtensionFileLoader(fullname, target), + submodule_search_locations=locations, + ) + except Exception as e: + self._mark_unbuildable(fullname, source) + print(f"CRNS: could not compile {fullname}, " + f"using interpreted module: {e}", file=sys.stderr) + + rel = fullname.split(".") + locations = [os.path.join(self.source_root, *rel)] if is_package else None + return spec_from_file_location( + fullname, source, + loader=SourceFileLoader(fullname, source), + submodule_search_locations=locations, + ) + + def find_spec(self, fullname, path, target=None): + if not isinstance(fullname, str): + return None + if not (fullname == "RNS" or fullname.startswith("RNS.")): + return None + + source, is_package = self._source(fullname) + if source is None: + return None + + return self._spec(fullname, source, is_package, + compile_attempted=not self._is_unbuildable(fullname, source)) + + _RNSFinder = _RNSImportFinder() + sys.meta_path.insert(0, _RNSFinder) def notice_job(): global noticed @@ -22,5 +242,6 @@ def notice_job(): compiling = True threading.Thread(target=notice_job, daemon=True).start() +_RNSFinder.precompile() import RNS; compiling = False -if noticed: print("Done."); sys.stdout.flush() \ No newline at end of file +if noticed: print("Done."); sys.stdout.flush() diff --git a/RNS/Cryptography/__init__.py b/RNS/Cryptography/__init__.py index 925221de..0ccc23d7 100644 --- a/RNS/Cryptography/__init__.py +++ b/RNS/Cryptography/__init__.py @@ -52,5 +52,7 @@ elif cp.PROVIDER == cp.PROVIDER_PYCA: py_modules = glob.glob(os.path.dirname(__file__)+"/*.py") pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc") -modules = py_modules+pyc_modules -__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))])) \ No newline at end of file +so_modules = glob.glob(os.path.dirname(__file__)+"/*.so") +pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd") +modules = py_modules+pyc_modules+so_modules+pyd_modules +__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"])) \ No newline at end of file diff --git a/RNS/Interfaces/Android/__init__.py b/RNS/Interfaces/Android/__init__.py index a9a74ff3..454b31ff 100644 --- a/RNS/Interfaces/Android/__init__.py +++ b/RNS/Interfaces/Android/__init__.py @@ -33,5 +33,7 @@ import glob py_modules = glob.glob(os.path.dirname(__file__)+"/*.py") pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc") -modules = py_modules+pyc_modules -__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))])) \ No newline at end of file +so_modules = glob.glob(os.path.dirname(__file__)+"/*.so") +pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd") +modules = py_modules+pyc_modules+so_modules+pyd_modules +__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"])) \ No newline at end of file diff --git a/RNS/Interfaces/__init__.py b/RNS/Interfaces/__init__.py index 9a11d8fd..56a541bf 100755 --- a/RNS/Interfaces/__init__.py +++ b/RNS/Interfaces/__init__.py @@ -36,5 +36,7 @@ import RNS.Interfaces.util.netinfo as netinfo py_modules = glob.glob(os.path.dirname(__file__)+"/*.py") pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc") -modules = py_modules+pyc_modules -__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))])) \ No newline at end of file +so_modules = glob.glob(os.path.dirname(__file__)+"/*.so") +pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd") +modules = py_modules+pyc_modules+so_modules+pyd_modules +__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"])) \ No newline at end of file diff --git a/RNS/Interfaces/util/__init__.py b/RNS/Interfaces/util/__init__.py index 72302cb5..e90e61ad 100644 --- a/RNS/Interfaces/util/__init__.py +++ b/RNS/Interfaces/util/__init__.py @@ -3,5 +3,7 @@ import glob py_modules = glob.glob(os.path.dirname(__file__)+"/*.py") pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc") -modules = py_modules+pyc_modules -__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))])) +so_modules = glob.glob(os.path.dirname(__file__)+"/*.so") +pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd") +modules = py_modules+pyc_modules+so_modules+pyd_modules +__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"])) diff --git a/RNS/Utilities/__init__.py b/RNS/Utilities/__init__.py index a9a74ff3..454b31ff 100644 --- a/RNS/Utilities/__init__.py +++ b/RNS/Utilities/__init__.py @@ -33,5 +33,7 @@ import glob py_modules = glob.glob(os.path.dirname(__file__)+"/*.py") pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc") -modules = py_modules+pyc_modules -__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))])) \ No newline at end of file +so_modules = glob.glob(os.path.dirname(__file__)+"/*.so") +pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd") +modules = py_modules+pyc_modules+so_modules+pyd_modules +__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"])) \ No newline at end of file diff --git a/RNS/Utilities/rngit/__init__.py b/RNS/Utilities/rngit/__init__.py index 56fbb793..2a04167a 100644 --- a/RNS/Utilities/rngit/__init__.py +++ b/RNS/Utilities/rngit/__init__.py @@ -35,5 +35,7 @@ import glob py_modules = glob.glob(os.path.dirname(__file__)+"/*.py") pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc") -modules = py_modules+pyc_modules -__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))])) \ No newline at end of file +so_modules = glob.glob(os.path.dirname(__file__)+"/*.so") +pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd") +modules = py_modules+pyc_modules+so_modules+pyd_modules +__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"])) \ No newline at end of file diff --git a/RNS/Utilities/rnsh/__init__.py b/RNS/Utilities/rnsh/__init__.py index fbf23102..a528c782 100644 --- a/RNS/Utilities/rnsh/__init__.py +++ b/RNS/Utilities/rnsh/__init__.py @@ -41,7 +41,9 @@ module_dir = os.path.dirname(module_abs_filename) py_modules = glob.glob(os.path.dirname(__file__)+"/*.py") pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc") -modules = py_modules+pyc_modules -__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))])) +so_modules = glob.glob(os.path.dirname(__file__)+"/*.so") +pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd") +modules = py_modules+pyc_modules+so_modules+pyd_modules +__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"])) def _get_version(): return __version__ diff --git a/RNS/__init__.py b/RNS/__init__.py index 1c63b3b8..487c0bd4 100755 --- a/RNS/__init__.py +++ b/RNS/__init__.py @@ -44,8 +44,10 @@ from functools import partial, wraps py_modules = glob.glob(os.path.dirname(__file__)+"/*.py") pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc") -modules = py_modules+pyc_modules -__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))])) +so_modules = glob.glob(os.path.dirname(__file__)+"/*.so") +pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd") +modules = py_modules+pyc_modules+so_modules+pyd_modules +__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"])) import importlib.util if importlib.util.find_spec("cython"): import cython; compiled = cython.compiled diff --git a/RNS/vendor/__init__.py b/RNS/vendor/__init__.py index ccfb449d..7ec88c11 100755 --- a/RNS/vendor/__init__.py +++ b/RNS/vendor/__init__.py @@ -3,5 +3,7 @@ import glob py_modules = glob.glob(os.path.dirname(__file__)+"/*.py") pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc") -modules = py_modules+pyc_modules -__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))])) \ No newline at end of file +so_modules = glob.glob(os.path.dirname(__file__)+"/*.so") +pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd") +modules = py_modules+pyc_modules+so_modules+pyd_modules +__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"])) \ No newline at end of file