Compare commits

..

No commits in common. "master" and "1.2.8" have entirely different histories.

224 changed files with 8420 additions and 32950 deletions

1
.gitignore vendored
View file

@ -14,4 +14,3 @@ tests/rnsconfig/logfile*
*.data *.data
*.result *.result
.buildinfo.bak .buildinfo.bak
docs/Reticulum Manual.*

1386
AGENTS.md

File diff suppressed because it is too large Load diff

View file

@ -1,278 +1,12 @@
compiling = False compiling = False
noticed = False noticed = False
notice_delay = 0.3 notice_delay = 0.3
import os
import sys
import time import time
import sys
import threading import threading
from importlib.util import find_spec from importlib.util import find_spec
if find_spec("pyximport") and find_spec("cython"): if find_spec("pyximport") and find_spec("cython"):
import json import pyximport; pyxloader = pyximport.install(pyimport=True, language_level=3)[1]
import shutil
import subprocess
from importlib.abc import MetaPathFinder
from importlib.machinery import ExtensionFileLoader, SourceFileLoader
from importlib.machinery import EXTENSION_SUFFIXES
from importlib.util import spec_from_file_location
from distutils.extension import Extension
from pyximport import pyxbuild
COMPILE_DIRECTIVES = { "annotation_typing": False }
_WORKER_SCRIPT = r"""
import sys, os, json
from distutils.extension import Extension
from pyximport import pyxbuild
def main():
tasks_path = sys.argv[1]
with open(tasks_path, "r") as fh: tasks = json.load(fh)
results = []
for task in tasks:
try:
os.makedirs(task["scratch"], exist_ok=True)
extension = Extension(name=task["basename"], sources=[task["source"]])
extension.cython_directives = task["directives"]
so_path = pyxbuild.pyx_to_dll(task["source"], ext=extension, build_in_temp=True, pyxbuild_dir=task["scratch"])
os.makedirs(os.path.dirname(task["target"]), exist_ok=True)
os.replace(so_path, task["target"])
results.append({"fullname": task["fullname"], "ok": True, "error": None})
except Exception as e:
results.append({"fullname": task["fullname"], "ok": False,
"error": "{}: {}".format(type(e).__name__, e)})
with open(tasks_path + ".results", "w") as fh: json.dump(results, fh)
if __name__ == "__main__": main()
"""
def _compile_jobs():
jobs = os.environ.get("CRNS_JOBS")
if jobs: return max(1, int(jobs))
return max(1, min(os.cpu_count() or 1, 16))
class _ImportFinder(MetaPathFinder):
def __init__(self):
self.build_dir = os.path.join(os.path.expanduser("~"), ".pyxbld")
self.mirror_root = os.path.join(self.build_dir, "rnslib")
self.source_root = self._find_source_root()
self.abi_suffix = EXTENSION_SUFFIXES[0]
self.unbuildable = {}
self.unbuildable_path = os.path.join(self.build_dir, "unbuildable.json")
self._verify_mirror_directives()
try:
with open(self.unbuildable_path, "r") as fh: self.unbuildable = json.load(fh)
except Exception: self.unbuildable = {}
def _verify_mirror_directives(self):
# If the compiler directives have changed since the mirror was
# built, the entire mirror is invalidated and rebuilt.
stamp_path = os.path.join(self.build_dir, "rnslib.stamp")
directives = json.dumps(COMPILE_DIRECTIVES, sort_keys=True)
stamp = directives + "\n" + self.abi_suffix
previous = None
try:
with open(stamp_path, "r") as fh: previous = fh.read()
except Exception: pass
if previous != stamp:
shutil.rmtree(self.mirror_root, ignore_errors=True)
os.makedirs(self.mirror_root, exist_ok=True)
try:
os.makedirs(self.build_dir, exist_ok=True)
with open(stamp_path, "w") as fh: fh.write(stamp)
except Exception: pass
def _save_unbuildable(self):
try:
os.makedirs(os.path.dirname(self.unbuildable_path), exist_ok=True)
with open(self.unbuildable_path, "w") as fh: json.dump(self.unbuildable, fh, indent=1)
except Exception: pass
def _mark_unbuildable(self, fullname, source):
try:
self.unbuildable[fullname] = int(os.path.getmtime(source))
self._save_unbuildable()
except Exception: pass
def _unmark_buildable(self, fullname):
if fullname in self.unbuildable:
del self.unbuildable[fullname]
self._save_unbuildable()
def _is_unbuildable(self, fullname, source):
if fullname not in self.unbuildable: return False
try:
if int(os.path.getmtime(source)) == self.unbuildable[fullname]: return True
except Exception: pass
self.unbuildable.pop(fullname, None)
return False
def _find_source_root(self):
here = os.path.dirname(os.path.abspath(__file__))
candidate = os.path.abspath(os.path.join(here, "..", "RNS", "__init__.py"))
if os.path.exists(candidate): return os.path.dirname(os.path.dirname(candidate))
for entry in [os.getcwd()] + list(sys.path):
if not entry: continue
candidate = os.path.join(entry, "RNS", "__init__.py")
if os.path.exists(candidate): return os.path.dirname(candidate)
raise RuntimeError("CRNS could not locate the RNS source tree")
def _all_source_modules(self):
rns_root = os.path.join(self.source_root, "RNS")
for root, dirs, files in os.walk(rns_root):
dirs[:] = [d for d in dirs if d not in ("__pycache__", "i2plib")]
rel = os.path.relpath(root, self.source_root).replace(os.sep, ".")
for fn in sorted(files):
if not fn.endswith(".py"): continue
if fn == "__init__.py": fullname = rel
else: fullname = rel + "." + fn[:-3]
yield fullname
def _mirror_target(self, fullname, is_package):
rel = fullname.split(".")
basename = rel[-1]
if is_package: return os.path.join(self.mirror_root, *rel, "__init__" + self.abi_suffix)
return os.path.join(self.mirror_root, *rel[:-1], basename + self.abi_suffix)
def _source(self, fullname):
rel = fullname.split(".")
package_source = os.path.join(self.source_root, *rel, "__init__.py")
if os.path.exists(package_source): return package_source, True
module_source = os.path.join(self.source_root, *rel) + ".py"
if os.path.exists(module_source): return module_source, False
return None, None
def _compile(self, fullname, source):
basename = fullname.rsplit(".", 1)[-1]
extension = Extension(name=basename, sources=[source])
extension.cython_directives = COMPILE_DIRECTIVES
so_path = pyxbuild.pyx_to_dll(source, ext=extension, build_in_temp=True, pyxbuild_dir=self.build_dir)
abi_suffix = os.path.basename(so_path)[len(basename):]
return so_path, abi_suffix
def _is_current(self, fullname, source, is_package):
target = self._mirror_target(fullname, is_package)
if not os.path.exists(target): return False
try: return os.path.getmtime(target) >= os.path.getmtime(source)
except Exception: return False
def _ensure_compiled(self, fullname, source, is_package):
target = self._mirror_target(fullname, is_package)
if os.path.exists(target) and os.path.getmtime(target) >= os.path.getmtime(source): return target
so_path, abi_suffix = self._compile(fullname, source)
os.makedirs(os.path.dirname(target), exist_ok=True)
os.replace(so_path, target)
return target
def precompile(self):
if not os.path.isdir(self.mirror_root):
os.makedirs(self.mirror_root, exist_ok=True)
pending = []
for fullname in self._all_source_modules():
source, is_package = self._source(fullname)
if source is None:
continue
if self._is_current(fullname, source, is_package):
self._unmark_buildable(fullname)
continue
if self._is_unbuildable(fullname, source):
continue
pending.append((fullname, source, is_package))
if not pending: return
jobs = min(_compile_jobs(), len(pending))
if jobs > 1:
# Round-robin assignment spreads the large modules evenly
# across workers.
chunks = [[] for _ in range(jobs)]
for i, item in enumerate(pending): chunks[i % jobs].append(item)
workers_dir = os.path.join(self.build_dir, "workers")
os.makedirs(workers_dir, exist_ok=True)
procs = []
for w in range(jobs):
tasks = []
for fullname, source, is_package in chunks[w]:
tasks.append({ "fullname": fullname,
"basename": fullname.rsplit(".", 1)[-1],
"source": source,
"target": self._mirror_target(fullname, is_package),
"scratch": os.path.join(workers_dir, "w{}".format(w)),
"directives": COMPILE_DIRECTIVES })
task_path = os.path.join(workers_dir, "tasks{}.json".format(w))
with open(task_path, "w") as fh: json.dump(tasks, fh)
procs.append((w, task_path, subprocess.Popen([sys.executable, "-c", _WORKER_SCRIPT, task_path],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)))
for w, task_path, proc in procs:
stdout, stderr = proc.communicate()
if proc.returncode != 0:
for fullname, _, _ in chunks[w]:
source, _ = self._source(fullname)
self._mark_unbuildable(fullname, source)
print(f"CRNS: compile worker failed for {fullname}, using interpreted module", file=sys.stderr)
if stderr:
tail = stderr.decode(errors="replace").strip().splitlines()[-5:]
print("CRNS: worker output:\n" + "\n".join(tail), file=sys.stderr)
continue
results = {}
try:
with open(task_path + ".results", "r") as fh:
for entry in json.load(fh): results[entry["fullname"]] = entry
except Exception: results = {}
for fullname, _, _ in chunks[w]:
entry = results.get(fullname)
if entry is not None and entry.get("ok"): self._unmark_buildable(fullname)
else:
error = "" if entry is None else entry.get("error", "")
source, _ = self._source(fullname)
self._mark_unbuildable(fullname, source)
print(f"Could not compile {fullname}, using interpreted module: {error}", file=sys.stderr)
else:
for fullname, source, is_package in pending:
try:
self._ensure_compiled(fullname, source, is_package)
self._unmark_buildable(fullname)
except Exception as e:
self._mark_unbuildable(fullname, source)
print(f"Could not compile {fullname}, using interpreted module: {e}", file=sys.stderr)
def _spec(self, fullname, source, is_package, compile_attempted):
if fullname == "RNS.vendor.i2plib" or fullname.startswith("RNS.vendor.i2plib."):
rel = fullname.split(".")
locations = [os.path.join(self.source_root, *rel)] if is_package else None
return spec_from_file_location(fullname, source, loader=SourceFileLoader(fullname, source), submodule_search_locations=locations)
if compile_attempted:
try:
target = self._ensure_compiled(fullname, source, is_package)
self._unmark_buildable(fullname)
if is_package: locations = [os.path.dirname(target)]
else: locations = None
return spec_from_file_location(fullname, target, loader=ExtensionFileLoader(fullname, target),
submodule_search_locations=locations)
except Exception as e:
self._mark_unbuildable(fullname, source)
print(f"Could not compile {fullname}, using interpreted module: {e}", file=sys.stderr)
rel = fullname.split(".")
locations = [os.path.join(self.source_root, *rel)] if is_package else None
return spec_from_file_location(fullname, source,loader=SourceFileLoader(fullname, source),submodule_search_locations=locations)
def find_spec(self, fullname, path, target=None):
if not isinstance(fullname, str): return None
if not (fullname == "RNS" or fullname.startswith("RNS.")): return None
source, is_package = self._source(fullname)
if source is None: return None
return self._spec(fullname, source, is_package, compile_attempted=not self._is_unbuildable(fullname, source))
_importfinder = _ImportFinder()
sys.meta_path.insert(0, _importfinder)
def notice_job(): def notice_job():
global noticed global noticed
@ -280,13 +14,13 @@ def notice_job():
while compiling: while compiling:
if time.time() > started+notice_delay and compiling: if time.time() > started+notice_delay and compiling:
noticed = True noticed = True
print("Compiling RNS object code...") print("Compiling RNS object code... ", end="")
sys.stdout.flush() sys.stdout.flush()
break break
time.sleep(0.1) time.sleep(0.1)
compiling = True compiling = True
threading.Thread(target=notice_job, daemon=True).start() threading.Thread(target=notice_job, daemon=True).start()
_importfinder.precompile()
import RNS; compiling = False import RNS; compiling = False
if noticed: print("Compilation done"); sys.stdout.flush() if noticed: print("Done."); sys.stdout.flush()

View file

@ -1,484 +1,3 @@
### RNS 1.6.1
This release brings yet another round of major efficiency, performance and filtering improvements to RNS, and continues the transition towards full network stamp functionality.
All network stamp types will now be generated and emitted by default. Expect some CPU load once your applications start up for the first time on `1.6.1`, as destination stamps are generated.
**Changes**
- Enabled network stamp generation by default
- Added ability to filter destination types by hash
- Added hop limiter options both globally and per-interface
- Added super fast announce ingress pre-filtering
- Added filtered traffic stats to `rnstatus`
- Added link traffic stats to `rnstatus`
- Added display of remote transport IDs per client interface in `rnstatus`.
- Ability to inspect filtered traffic, announces, et cetera in `rnstatus` per interface
- Ability to inspect tunnel properties, stamp costs, remote implementation info and more in `rnstatus`
- Added ability to set `client_limit` on `BackboneInterface`, defaults to 500
- Added ability to configure the maximum amount of stamp workers and worker niceness
- Added ability to place a `blocked_ips` file in your config directory to permanently block IPs
- Added ability to directly blackhole remote transport nodes
- Added ability to configure stamp values and costs for `rngit`
- Added asynchronous, interleaved held announce queue cleaning across all interfaces based on potential announce usability
- Added bidirectional tunnel synthesis on non shared-medium interfaces, with stamp cost and version signalling
- Added announce rate table persistence between instance restarts
- Added default caps for configured auto-connect interface counts
- Added path state info to API path table getters
- Significantly optimized and improved inbound announce filtering and deduplication
- Significantly optimized and reworked the filtering subsystem for scalability across hundreds of interfaces
- Improved `rnpath` display and filtering options
- Improved traffic counting
- Improved path responsiveness tracking
- Improved protocol violation tracking and logging
- Improved path pool and tunnel table culling
- Tuned announce validation caches and other parameters
- Fixed a regression introduced in `1.6.0`, that could cause path table culling to temporarily remove an instance's own local destination paths
- Fixed dataplane egress control state not resetting on `BackboneInterface` client connection recovery
- Fixed a potential logging deadlock
- Fixed a potential exit handler deadlock if a signal arrives mid RPC call
- Fixed missing wanted destination type for interface discovery when local announce filtering was enabled
- Fixed split-instance initial known-destinations load not being available on Android
- Added mitigation for a CPython process fork deadlock dug in stdin handling
**Verified Retrieval**
You can retrieve and verify this release over Reticulum using the built-in `rngit release` utility. To retrieve only the installation `.whl` package, and the release manifest for future updates, you can use:
```sh
rngit release rns://7649a50d84610232d1416b41d2896aff/reticulum/reticulum fetch "latest:rns-*.whl" --signer bc7291552be7a58f361522990465165c
```
To download all artifacts, including the documentation and source archive, you can use the following command:
```sh
rngit release rns://7649a50d84610232d1416b41d2896aff/reticulum/reticulum fetch latest:all --signer bc7291552be7a58f361522990465165c
```
**Release Signatures**
Release artifacts include a signed `rsm` release manifest and `rsg` signature files that can be validated against the RNS release signing identity `<bc7291552be7a58f361522990465165c>` using `rngit` or `rnid`. To perform an offline verification of all release artifacts using a manifest:
```sh
rngit release rns_*.rsm verify --signer bc7291552be7a58f361522990465165c
```
To verify release artifacts using individual `rsg` files, while also verifying the manifest itself, download the `rsm` and `rsg` signatures, make sure they are in the same folder as the release artifact, and run `rnid` signature verification with the release identity as the required signer:
```sh
rnid -i bc7291552be7a58f361522990465165c -V rns_*.rsm *.rsg
```
The `rnid` utility will then verify the signatures, and display whether they are valid. If the signature cannot be verified, the release has been tampered with and should be discarded.
### RNS 1.6.0
This major release of Reticulum brings significant improvements to resource consumption, performance, stability and resilience, especially for constrained devices and nodes.
**Memory & Performance Improvements**
A new, SQLite based storage backend has been implemented, and is now the default. All legacy file-based storage will be auto-migrated to the database-backed store on the first start. If you are using an esoteric system that does not support the `sqlite3` module, Reticulum will automatically fall back to using the file-based storage system.
The memory management system has been completely reworked, and significantly improved. For long-running transport nodes, this cuts memory consumption by approximately 50%, and for non-transport instances, standalone applications and appliance devices by up to 80%.
**Path Table Sharding**
This release implements the initial transport awareness and mechanisms for emergent path table sharding. Transport instances will no longer attempt to hold full inter-network path tables in memory, but keep only paths relevant to the nodes they are transporting traffic for. Unused paths are now kept for much shorter periods of time, and used as a rolling pool that potential new traffic corridors can form into. The system is entirely passive, and adds no inter-node coordination or additional protocol mechanics. The current release brings the foundation, and the upcoming 1.6.x releases will tune, improve it and add the remaining pieces of the functionality.
**Network Stamps**
The stamp system, first introduced in LXMF, where it was tested and refined over the last two years, has now been brought into the core of Reticulum to help protect against spam and network abuse. All the relevant core primtives now support stamp generation and verification:
- Identities can be created with an embedded stamp value. This adds no additional bytes or identity format changes. Generation and verification is completely transparent, and carried within the Reticulum Identity keyset.
- Destinations can now be stamped, and will embed stamps and required link request cost in their announces. The change is backwards-compatible, so even older versions of Reticulum carry stamped announces transparently. Destination stamps are one-time generations, stored and persisted on success, so these have relatively high default target values. Expect some CPU usage when an unstamped destination has its stamp generated for the first time on 1.6.0+.
- Path requests can now be stamped, and are also backwards-compatible. A stamped path request will be carried transparently by non stamp-aware transport nodes.
- Link requests are stamped according to the announced link request cost of the destination. As with all the other primitives, these are also completely transparent and backwards-compatible. The stamp is carried embedded in the ephemeral link request keyset.
The stamp functionality and requirements will be deployed in a phased transition, so both network operators and application developers have time to update their systems, and any potential bugs and issues can be worked out before the full deployment:
- Version `1.6.0` implements all the core functionality and primitives, and starts excercising the verification mechanics. By default, no destination and path request stamps are generated yet in this release, and while stamps are checked and verified, they are not *enforced*. However, both stamp emission and enforcement can be enabled in the configuration for testing or early opt-in. Link request stamps *will* be generated, if the destination has announced a cost.
- Version `1.6.1` will enable stamp emission for all network primitives by default. Old, unstamped destinations will have their stamps generated automatically on their first announce, and will start announcing link request costs as configured. Enforcement in this phase will still be disabled by default, but can be enabled by configuration.
- Version `1.6.2` will complete the transition to network stamps. All functionality will now be enabled by default, including enforcement. Destination announces, path requests and link requests that do not meet stamp requirements will be dropped. Any old transport nodes that have not updated to `1.6.x` will not be able to communicate with stamp-enforcing nodes, but will still *carry traffic for* stamp-capable systems.
The `1.6.0` and `1.6.1` releases will be an important time for observing the on-network behaviour, functionality and stability of the system, and to make any potential adjustments or fix problems before the final transition to `1.6.2`. If you're operating a transport node, please help out by reporting any problems or issues you discover.
**For application developers**, there is no immediate refactoring or changes required application-side. Simply updating RNS to `1.6.0` is sufficient to transparently enable all the new functionality and improvements. It is very much recommended, however, to familiarize yourself with the new API methods for specifying link request costs and destination values, as these are configurable on a per-destination basis, and might warrant specific configuration for your intended use-case.
**Changes**
- Added `Identity` stamps
- Added destination and announce stamps
- Added path request stamps
- Added link request stamps
- Added link request cost signalling to announces
- Added fractional stamp value support
- Added path usage tracking
- Added automatic path table culling based on traffic and usage
- Added path retainment prioritization for slow interfaces
- Added unused path pool monitoring and handling
- Added managed storage backend using SQLite
- Added fallback to legacy file-based storage if SQLite is unavailable
- Added managed storage for known destinations and ratchets
- Added managed storage for cached announces
- Added memory-density optimized announce rate table implementation
- Added memory-density optimized packet hashlist implementation
- Added heap-tracking memory management
- Added more comprehensive metrics to status output structure
- Added improved filtering backend capabilities
- Added new `[filter]` configuration section
- Added `hop_limiter` configuration option
- Added allow and ignore lists for destination types
- Added ability for applications to receive required announces only
- Added cost-aware interface discovery and auto-connects
- Added cost and value display to `rnstatus` and `rnpath` utilities
- Added `app_flag` signalling to resource advertisements
- Added link MTU aware resource advertisements
- Significantly improved inbound announce handling performance
- Significantly improved stamper performance on Android
- Significantly improved announce handler callback performance
- Optimized and improved stamper implementation
- Optimized announce rate table handling
- Optimized packet filter hashlist handling
- Improved early filtering and malformed packet detection
- Improved path request protocol violation handling
- Fixed `RNodeInterface` phy stats regression on Android
- Fixed missing interface parameter inheritances
- Fixed `rngit` default config path typo for `~/.config/rngit` case
- Fixed invalid RPC error handling in some cases
- Fixed various minor bugs and inconsistencies
- Updated documentation
### 2026-10-05: RNS 1.5.7
This maintenance release fixes a bug in the dataplane egress control handling.
**Changes**
- Fixed egress control TX stall state deadlock on BackboneInterface initiator instances
### 2026-10-02: RNS 1.5.6
This maintenance release improves interface discovery handling.
**Changes**
- Fixed ephemeral transport identity being possible when discoverable interfaces are present on non-transport instances
- Ensure only transport-enabled interfaces are autoconnected at discovery time
### 2026-09-29: RNS 1.5.5
This release adds live interface attach, detach and reload, as well as significant improvements to interface discovery. Auto-connecting interfaces is now also supported on Windows and macOS.
It also brings improvements to `rngit` and fixes discovered `I2PInterface` configuration snippets, along with a number of other bugs.
**Changes**
- Added ability to `--attach`, `--detach` and `--reload` interfaces to the `rnstatus` utility
- Added interface discovery auto-connect support on Windows and macOS
- Added ability to download micron-converted versions of markdown files to the `rngit` page node
- Added workdoc counts to filter scope links to the `rngit` page node
- Added version info to `rnstatus` discovered interfaces output
- Added `--show-stale` and `--show-unknown` options to `rnstatus`
- Added sequential naming of auto-connected interfaces on announced name collisions
- Added auto-connect implementation and version criteria filtering
- Added configuration option for allowing auto-connects to discovered interfaces with missing version info
- Improved sanitization of nonsensical interface discovery IFAC announce data
- Fixed `I2PInterface` discovery config snippets not including `.b32.i2p`
- Fixed timeout and response rejects not being processed for single-packet requests, by **JRG**
- Fixed missing configurable property propagations on spawned interfaces, by **JRG**
- Fixed ingress control producer comprehension at mid-watermark using incorrect variable name for interface type check, by **JRG**
- Fixed instances of `announce_cap` property ensurance logic not storing announce cap value as a fraction, by **JRG**
- Fixed a potential race condition in `LocalInterface` initialization.
- Updated documentation and manual
### 2026-09-11: RNS 1.5.4
This release improves RNode BLE connectivity reliability on desktop operating systems.
**Changes**
- Fixed RNode BLE device address acquisition on windows, by **Nickie Deuxyeux**
- Fixed RNode BLE re-connection deadlock on desktop
- Improved RNode BLE reconnect reliability
### 2026-09-10: RNS 1.5.3
This maintenance release adds functionality to, and fixes bugs in `rngit`, and includes a few other optimizations and improvements.
**Changes**
- Added media request handler to `rngit`, for serving images to `nomadnet` clients
- Added media conversion handler to `rngit`, auto-converting images to webp
- Added importable helper method for using the `rngit` media converter in other programs
- Added a `no_ident` template to the `rngit` page node
- Added immediate activation of changed permissions with `rngit perms`
- Added optimized HDLC framer
- Fixed `rngit` repo admins not being able activate/complete workdocs from other users, by **Bergie**
- Fixed a typo, I'm positively certain this is the last one
### 2026-08-28: RNS 1.5.2
This maintenance release fixes a regression in resource transfers that was introduced in 1.5.1, an `I2PInterface` bug, and sets dataplane control parameters to default values that are actually sensible. If you installed `1.5.1`, it's update time again, baby.
**Changes**
- Tuned dataplane control parameters
- Added example for blocking unidentified page node peers to `rngit` default config
- Fixed regression in resource transfers causing some `rngit` file downloads to fail
- Fixed keepalive frames being passed to transport core on `I2PInterface`
### 2026-08-28: RNS 1.5.1
This release focuses on dataplane control, memory bounding and transport throughput, with adaptive ingress and egress control for interfaces, a new (and very efficient) zero-copy coalescing transmit buffer, an optimized HDLC deframer, and a range of other improvements to the inbound packet processing paths. It also introduces support for live profiling, and various new diagnostics output in `rnstatus`, along with a number of bug fixes and general improvements.
**Changes**
- Added adaptive dataplane egress control
- Added adaptive dataplane ingress control
- Added zero-copy coalescing transmit buffers to interfaces using the `BackboneInterface` backend
- Added early protocol violation checks for invalid frames
- Added transport implementation name and version to discovery information requirements
- Added full live profiling to the built-in `Profiler` primitive, by **K8**
- Added the `@RNS.Profiler.profile` decorator, by **K8**
- Added support for indefinitely running, reentrant profilers with bounded capture, by **K8**
- Added live profiling results output to `rnstatus`, by **K8**
- Added PPS statistics to `rnstatus`
- Added interface MTU display to `rnstatus`
- Added interface TX drop statistics to `rnstatus`
- Added interface TX buffer size output to `rnstatus`
- Added throughput benchmarker to the test suite
- Added support for natively compiled module builds, and the ability to load compiled modules when available
- Added module compilation status reporting from in-wheel build information
- Added parity tests for HDLC, IFAC and HKDF against the legacy implementations
- Added shared medium hints to interfaces
- Improved memory and CPU consumption under high traffic loads
- Improved traffic class handling
- Optimized HKDF to ~5.7x performance, if this worries you, **good**; then see the full parity and standard vector test suite and read the code
- Optimized inbound and outbound IFAC handling to ~90x performance on large frames
- Optimized inbound packet processing by reducing lock acquisitions and path table contention
- Optimized lookups for pending and active links using hash maps for lookup operations
- Optimized announce validation by caching signature validation results, significantly lowering announce storm CPU load
- Optimized the HDLC deframer
- Reduced redundant packet hashing in inbound processing
- Tuned default inbound queue lengths and announce queuing
- Tuned automatic interface MTU configuration
- Fixed `Resource` transfers failing when initialized from stream-based data sources that fell outside `MAX_EFFICIENT_SIZE`
- Fixed an invalid prefix stripping bug in the `rngit` page server (`.something` files not viewable)
- Fixed RSSI/SNR reporting regression
- Fixed keepalive handling on non-epoll backends
- Fixed a bug in `rngit` page navigation content initialization order
- Fixed the `rngit` page node not being able to serve in-tree downloads for large files
- Fixed `rnstatus` blocked IP listing including IPs that were actually not yet blocked
- Fixed `rnstatus` traffic totals counting the local shared instance inter-app transit in totals
- Fixed various minor bugs in `rnsh`, `rnir`, identity handling
- Fixed a latent bug in the AES module, where an exception would not resolve it's exception description correctly, and instead raise another exception.
- Removed dead Python 2 code from `umsgpack`
### 2026-08-22: RNS 1.5.0
This release significantly improves the core Transport handling of RNS, with a priority-based ingress queue backend, substantially improved efficiency of both data and management traffic handling, more efficient ingress/egress limiting, and a long list of other improvements and bugfixes.
**Changes**
- Added ability to include operator LXMF address in interface discovery information
- Added prioritized inbound traffic processing to the transport core
- Added configurable inbound queue lengths for data, announce, path request and ingress limited traffic
- Added early filtering for inbound traffic and improved filtering performance
- Added per-interface protocol violation tracking
- Added in-flight path requests tracking
- Added request and response batching for in-flight path requests
- Added ability to signal blackholed status in announce validation return to the API
- Added full link MDU utilization for `Channel` and `Buffer`
- Added inbound queue pressure and queue drop statistics to `rnstatus`
- Added detailed announce and path request traffic flow statistics per interface to `rnstatus`
- Added total announce and PR count/frequency stats per interface to `rnstatus`
- Added data flow speed and composition stats to `rnstatus`
- Added protocol violation stats to `rnstatus`
- Added active link statistics to `rnstatus`
- Added blocked IP listings to `rnstatus`
- Added medium bitrate based timeout calculation helpers and RPC functions, by **Zenith**
- Added extra timeout for discovery path requests when slow interfaces are online, by **Zenith**
- Added adaptive timeouts to `rncp`, `rnpath`, `rnprobe` and `rnx`, by **Zenith**
- Added adaptive timeout calculation to `rngit`, by **Zenith**
- Improved overall Transport inbound processing speed
- Improved path request handling significantly
- Improved path request ingress limiting and accounting
- Improved egress limiting responsiveness under high incoming path request load
- Improved transport background job processing
- Improved early rejection of packets with excessive hop counts
- Fixed `BackboneInterface` EPOLL receive starvation and ingress control timestamp handling, by **JRG**
- Fixed potential transport deadlock on receipts lock when callbacks sends packets, by **JRG**
- Fixed transport state handling edge cases in path request, announce queue and pending link processing, by **JRG**
- Fixed link watchdog not resetting on receive exceptions, by **JRG**
- Fixed `Resource` cancellation on multi-segment resources, by **JRG**
- Fixed `Resource` transfer part index alignment and rebinding, by **JRG**
- Fixed stale BLE device reference in `RNodeInterface`, by **JRG**
- Fixed ratchet cleaning retained preservation, by **JRG**
- Fixed invalid `rnstatus` statistics handling, by **JRG**
- Fixed various bugs in packet, link and interface handling, by **JRG**
- Fixed per-interface burst count inconsistencies in `rnstatus` output
- Fixed `rngit` file resource operations failing on Windows
- Fixed `rnodeconf` config summary incorrectly displaying WiFi mode
- Fixed speedtest example aborting transfers on stale link status
- Updated documentation, including manual sections on queue tuning and interface discovery options
### 2026-07-26: RNS 1.4.2
This maintenance release fixes bugs in blackholed identity filtering and recursive path request handling for RNode interfaces.
**Changes**
- Fixed a bug where recursive path requests would cause a division by zero error on uninitialized RNode interfaces.
- Fixed slow blackholed filtering for discovered interfaces on Android in some cases.
### 2026-07-24: RNS 1.4.1
This release significantly improves path convergence and stability with the new path re-balancing functionality. It also introduces interface gravity configuration, new API functionality and fixes a range of bugs and inefficiencies.
**Changes**
- Added dynamic path re-balancing
- Added `set_max_request_size` to `Destination` API
- Added `max_response_size` to request API
- Added interface gravity handling
- Added `autoconnect_interface_mode` option
- Added `autoconnect_announces_to_internal` option
- Added `autoconnect_interface_gravity` option
- Added `default_gravity` option
- Added `announces_to_internal` interface option
- Added `gravity` interface option
- Added gravity display and sorting option to `rnstatus`
- Added boundary -> boundary and boundary -> gateway path requests
- Fixed `I2PInterface` tasks getting garbage-collected prematurely, thanks to **welo**
- Fixed various minor bugs in `I2PInterface`
- Fixed ingress control burst active flag deadlocking until new announces arrived under burst timing patterns
- Fixed various memory inefficiencies
- Fixed loglevel `LOG_EXTREME` not being usable in some cases
- Fixed historical interface discoveries not being cleaned according to blackholed identities
### 2026-07-20: RNS 1.4.0
This release focuses on performance, stability and reliability improvements, particularly with resource-constrained systems in mind. It also fixes several bugs, reduces logging noise, and makes information about blocked `BackboneInterface` clients available in `ifstats`.
**Changes**
- Optimized transport data persistence to avoid CPU spikes on low-powered systems
- Optimized interface hashes calculation and access to reduce overall processing load
- Implemented known destinations cleaning as background priority to avoid lock contention and reduce load on CPU-constrained systems
- Deprecated known destination on-disk recombination on background data persist to alleviate lock contention on CPU-constrained systems
- Implemented caching of invalid discovery stamps to reduce CPU load on low-powered systems
- Implemented valid discovery announce caching and sequential validation lock to reduce processing load on CPU-constrained systems
- Fixed link stale teardown erroneously occurring due to missing keepalive from initiator side when destination continously sends packets but initiator stays silent
- Fixed race condition in link watchdog timing
- Fixed missing None-check in `BackboneInterface` fast-flap detection
- Fixed invalid exception log handler in `BackboneInterface`
- Increased default discovery stamp value to 16
- Added blocked IPs list to ifstats
- Improved backbone interface logging
- Reduced overall logging noise
### 2026-07-19: RNS 1.3.9
**Critical Security Update**: This release fixes a severe security flaw in `rnsh`. Due to the nature of the issue, I will not disclose any further details for the time being. Once operators have had time to update, I will provide a full report for transparency. If you use `rnsh`, update **right now**.
**Important**: The new version of `rnsh` changes default identity file locations, and these will now be sourced from `~/.rnsh/identity` (initiator) and `~/.rnsh/identity.default` (listener). Make sure you copy your old files to this directory, or specify a custom identity path using the command line arguments. The `--config` argument has also been renamed to `--rnsconfig`, and the `--config` argument will now specify the `rnsh` configuration directory instead of the RNS configuration directory, bringing the behavior into alignment with other RNS utilities.
Additionally, this release includes automated blocking of fast-flapping clients on `BackboneInterface` listeners (see the Interfaces chapter of the manual for details), and a number of improvements to resource handling. The logging system has also been improved, and path and destination information moved to a new `LOG_PATHING` loglevel, to decrease log noise.
**Changes**
- Fixed a critical security issue in `rnsh`
- Added automated blocking of fast-flapping clients to `BackboneInterface`
- Added new `LOG_PATHING` loglevel, improved logging
- Added ability to make internal-mode interfaces discoverable
- Added ability to get discoverable interface location from external script
- Added `RESOURCE_RCL` signal on resource receiver cancel
- Improved resource handling and reliability
- Updated `rnsh` config args to work similarly to other RNS utilities
### 2026-07-10: RNS 1.3.8
This release fixes various inconsistencies in link and hop-count related APIs.
**Changes**
- Fixed inconsistent link traffic stats calculation
- Fixed link hop-count metric only being available on initiator side
- Fixed potential hop-count serialization error on transport
- Updated `WeaveInterface` to support latest Weave firmware
### 2026-07-03: RNS 1.3.7
This maintenance release improves announces propagation logic, and adds additional options for configuring announce propagation and interface behavior in transport mode.
**Changes**
- Added `internal` interface mode
- Added `recursive_prs` interface option
- Added `announces_from_internal` interface option
- Added new options to the manual
- Improved and cleaned up announce propagation logic
### 2026-07-03: RNS 1.3.6
This release contained a bug in the local instance transport handling, and was superseded by version `1.3.7`.
### 2026-06-01: RNS 1.3.5
This maintenance release contains an important fix for `AutoInterface` reliability when roaming between different physical networks.
**Changes**
- Fixed UDP listener replacement deadlocking inbound AutoInterface traffic when fast-roaming between physical interfaces or WiFi APs
- Fixed some paths never resolving when using other interfaces at the same time as a deadlocked AutoInterface
### 2026-05-29: RNS 1.3.4
This release fixes a regression that could cause sub-optimal path selection under conditions where the same announce was received within a very short timespan on different interfaces, as well as a few other bugs and inefficiencies.
**Changes**
- Fixed regression in inbound announce de-duplication
- Fixed missing check for shared instance type configuration conflict
- Fixed superfluous path state configuration on new announce from new destinations
- Improved cleanup of stale known destinations
- Improved shared instance RPC handling
### 2026-05-28: RNS 1.3.3
This release fixes a regression in persistence of known destination on Windows.
**Changes**
- Fixed regression in known destinations persist on Windows
### 2026-05-28: RNS 1.3.2
This release adds commit signing and validation support to the `rngit` system, as well as improvements to the blackhole functionality.
**Changes**
- Extended blackhole functionality to immediately terminate links from blackholed identities
- Added commit signing and validation to `rngit`
- Added commit hash inclusion in generated release manifest to `rngit`
- Added local `verify` operation shorthand to `rngit release`
- Added option to configure blackhole update interval
- Added configuration option to log without timestamps
### 2026-05-22: RNS 1.3.1
This maintenance release fixes a single bug.
**Changes**
- Fixed regression in request response transfer size accumulator
### 2026-05-21: RNS 1.3.0
This maintenance release fixes a number of bugs.
**Changes**
- Added ability to use wildcards and pattern matches in `rngit` artifact fetch targets
- Fixed channel outlet sequence holes and ghost envelopes on dying outlets by **neutral**
- Fixed known destination iteration races by **neutral**
- Fixed timeout deadlock in `rnsh` by **neutral**
- Fixed commit message rendering in `rngit`
- Fixed various minor bugs and output inconsistencies in `rngit`
- Adjusted timeouts for remote operations in `rngit`
- Updated documentation
### 2026-05-19: RNS 1.2.9
This release completes the operational functionality of the `rngit` system, which now has full release creation, fetch and verified update support using the `rngit release` command. Additionally, two chapters have been added to the manual should cover all the things that `rngit` is currently capable of.
**Changes**
- Added full `rngit` documentation to the manual
- Added offline `.rsm` release manifest verification
- Added the ability to fetch release updates directly from `.rsm` manifests
- Added canonical `.rsm` release structure validator to `rnid` for import
- Added `.rsm` manifest saving when using `rngit release fetch`
- Added remote `HEAD` tracking for forks and mirros to `rngit`
- Improved known destinations persist reliability
- Improved page node ref link handling in `rngit`
- Improved logging in various locations
### 2026-05-18: RNS 1.2.8 ### 2026-05-18: RNS 1.2.8
This release improves the `rngit` system with signed release manifest generation and automatic artifact signing. It also includes several additions to `rnid` and various minor fixes and improvements to the `rngit` system. This release improves the `rngit` system with signed release manifest generation and automatic artifact signing. It also includes several additions to `rnid` and various minor fixes and improvements to the `rngit` system.
@ -499,6 +18,15 @@ This release improves the `rngit` system with signed release manifest generation
- Fixed various minor bugs and inconsistencies in `rngit` - Fixed various minor bugs and inconsistencies in `rngit`
- Dropped `note` metadata field requirement from `rsg` structure - Dropped `note` metadata field requirement from `rsg` structure
**Release Signatures**
Release artifacts include a signed `rsm` release manifest and `rsg` signature files that can be validated against the RNS release signing identity `<bc7291552be7a58f361522990465165c>` using `rnid`. To verify files, download the `rsm` and `rsg` signatures, make sure they are in the same folder as the release artifact, and run `rnid` signature verification with the release identity as the required signer:
```sh
rnid -i bc7291552be7a58f361522990465165c -V manifest.rsm *.rsg
```
The `rnid` utility will then verify the signatures, and display whether they are valid. If the signature cannot be verified, the release has been tampered with and should be discarded.
### 2026-05-17: RNS 1.2.7 ### 2026-05-17: RNS 1.2.7
This release significantly improves the `rngit` system with fork, mirroring and empty repository creation functionality, a new work document proposals feature, improvements to the transport core reliability and efficiency and various other tweaks and improvements. This release significantly improves the `rngit` system with fork, mirroring and empty repository creation functionality, a new work document proposals feature, improvements to the transport core reliability and efficiency and various other tweaks and improvements.
@ -516,6 +44,15 @@ This release significantly improves the `rngit` system with fork, mirroring and
- Improved transfer completed feedback in `rncp`, thanks to **neutral** - Improved transfer completed feedback in `rncp`, thanks to **neutral**
- Improved interface transport insertion and removal - Improved interface transport insertion and removal
**Release Signatures**
Release artifacts include `rsg` signature files that can be validated against the RNS release signing identity `<bc7291552be7a58f361522990465165c>` using `rnid`. To verify files, download the `rsg` signatures, make sure they are in the same folder as the release artifact, and run `rnid` signature verification with the release identity as the required signer:
```sh
rnid -i bc7291552be7a58f361522990465165c -V rns*.whl
```
The `rnid` utility will then verify the signatures, and display whether it is valid. If the signature cannot be verified, the file has been tampered with and should be thrown very far away in a jiffy.
### 2026-05-14: RNS 1.2.6 ### 2026-05-14: RNS 1.2.6
This release adds further improvements to the `rnid` and `rngit` utilities, and includes several bugfixes and other improvements. This release adds further improvements to the `rnid` and `rngit` utilities, and includes several bugfixes and other improvements.
@ -537,6 +74,15 @@ This release adds further improvements to the `rnid` and `rngit` utilities, and
- Fixed potential race condition in interface discovery - Fixed potential race condition in interface discovery
- Fixed `rngit` remote helper hanging on startup if no client config had been created previously, and RNS loglevel was configured at debug or higher - Fixed `rngit` remote helper hanging on startup if no client config had been created previously, and RNS loglevel was configured at debug or higher
**Release Signatures**
Release artifacts include `rsg` signature files that can be validated against the RNS release signing identity `<bc7291552be7a58f361522990465165c>` using `rnid`. To verify files, download the `rsg` signatures, make sure they are in the same folder as the release artifact, and run `rnid` signature verification with the release identity as the required signer:
```sh
rnid -i bc7291552be7a58f361522990465165c -V rns*.whl
```
The `rnid` utility will then verify the signatures, and display whether it is valid. If the signature cannot be verified, the file has been tampered with and should be thrown very far away in a jiffy.
### 2026-05-09: RNS 1.2.5 ### 2026-05-09: RNS 1.2.5
This release brings substantial improvements to path request handling, and should significantly reduce overall network and local transport node processing loads. Path requests are now automatically ingress and egress limited per interface and sub-interface. Although the defaults are effective and sane, and should work right out of the box bring an end to practically all the PR and announce spam going on lately, the backend is fully configurable for both defaults and per interface, if you want to fiddle with the settings. This release brings substantial improvements to path request handling, and should significantly reduce overall network and local transport node processing loads. Path requests are now automatically ingress and egress limited per interface and sub-interface. Although the defaults are effective and sane, and should work right out of the box bring an end to practically all the PR and announce spam going on lately, the backend is fully configurable for both defaults and per interface, if you want to fiddle with the settings.
@ -565,6 +111,15 @@ For all node ops out there, I'd recomment updating to this at some sort of semi-
- Ensured canonical validation functions in `rngit` - Ensured canonical validation functions in `rngit`
- Lots of other small fixes and stability improvements to `rngit` - Lots of other small fixes and stability improvements to `rngit`
**Release Signatures**
Release artifacts include `rsg` signature files that can be validated against the RNS release signing identity `<bc7291552be7a58f361522990465165c>` using `rnid`. To verify files, download the `rsg` signatures, make sure they are in the same folder as the release artifact, and run `rnid` signature verification with the release identity as the required signer:
```sh
rnid -i bc7291552be7a58f361522990465165c -V rns-1.2.5-py3-none-any.whl
```
The `rnid` utility will then verify the signatures, and display whether it is valid. If the signature cannot be verified, the file has been tampered with and should be thrown very far away in a jiffy.
### 2026-05-07: RNS 1.2.4 ### 2026-05-07: RNS 1.2.4
This release brings a complete rewrite and update to the `rnid` utility, which is now a lot more useful, and better at finding and saving identities. It also includes a bunch of other improvements, such as expanded `rngit` functionality, better transport performance and a few bugfixes. Enjoy! This release brings a complete rewrite and update to the `rnid` utility, which is now a lot more useful, and better at finding and saving identities. It also includes a bunch of other improvements, such as expanded `rngit` functionality, better transport performance and a few bugfixes. Enjoy!

View file

@ -2,66 +2,57 @@
Welcome, and thank you for your interest in contributing to Reticulum! Welcome, and thank you for your interest in contributing to Reticulum!
Before interacting with this community, read these short and simple guidelines. Apart from writing code, there are many ways in which you can contribute. Before interacting with this community, read these short and simple guidelines.
If you didn't shit your pants after reading this document, you may well find that we are all really quite cuddly and nice to be around. Mostly.
## Expected Conduct ## Expected Conduct
First and foremost, there is one simple requirement for taking part in the development of this technology: While we primarily interact virtually, your actions matter and have real consequences. Therefore: **Act like a responsible, civilized person** - especially in the face of disputes and heated disagreements. Speak your mind here; discussions are welcome. Just do so in the spirit of being face-to-face with everyone else. Thank you. First and foremost, there is one simple requirement for taking part in this community: While we primarily interact virtually, your actions matter and have real consequences. Therefore: **Act like a responsible, civilized person** - especially in the face of disputes and heated disagreements. Speak your mind here; discussions are welcome. Just do so in the spirit of being face-to-face with everyone else. Thank you.
In order to keep the discourse navigable and useful, the following types of posts and inquiries will be deleted and ignored: In order to keep the discussion forums and issue trackers navigable and useful, the following types of posts will be deleted without notice:
- Spam. - Spam.
- Questions that have already been adequately answered elsewhere. Utilize search functions. - Questions that have already been adequately answered elsewhere. Use the search function.
- Low-effort posts or comments that contain no actual information or useful substance. [This is not a tea-house](./This%20Is%20Not%20a%20Teahouse.md). - Low-effort posts or comments that contain no actual information or useful content. This is not a tea-house.
- Post or comments solely containing personal opinions or beliefs without adding anything to the discussion. Facebook and X exists. - Post or comments solely containing personal opinions or beliefs without adding anything to the discussion. Facebook and X exists.
- Content that simply waste the developer's / maintainer's time with completely obvious "ideas", "insights" or "recommendations". Yes, we have *at least* 8 neurons ourselves. - Content that simply waste the developer's / maintainer's time with completely obvious "ideas", "insights" or "recommendations". Yes, we have *at least* 8 neurons ourselves.
- Posts that fail to understand that developing a highly complex software project with a very small amount of resources and people takes time. Imagining perfection on our behalf is useless. - Posts that fail to understand that developing a highly complex software project with a very small amount of resources and people takes time. Imagining perfection on our behalf is useless.
If you're new to the community and start out your engagement with any of the above transgressions, you will simply be banned permanently without notice or explanation, and any further communication from you will be ignored. If you're new to the community and start out your engagement with any of the above transgressions, you will simply be banned without notice or explanation, and your post will be deleted.
Except for the case of real, well-grounded, clearly documented, human-written security advisories, technical investigations or bug reports, if your inquiry or communication is not attached to a Reticulum Identity, expect it to be ignored.
If you find this "harsh", "unfair" or "unwelcoming", go somewhere else. This is not social club, but a work environment for the people contributing to the project. If you find this "harsh", "unfair" or "unwelcoming", go somewhere else. This is not social club, but a work environment for the people contributing to the project.
## Reporting Bugs ## Asking Questions
If you have found a bug or issue in this project, please report it to the maintainer over LXMF, or if strictly not possible, over email. Detailed and serious bug reports can also be posted in the various on-network forums and RRC channels that the developers frequent. Due to the realities described in [This Is Not a Teahouse](./This%20Is%20Not%20a%20Teahouse.md), there is no way to report issues or request features over the internet. If you want to ask a question, **do not open an issue**. The issue tracker is used by people *working on Reticulum* to track bugs, issues and improvements. Instead, ask away on the [discussions](https://github.com/markqvist/Reticulum/discussions).
Do not post feature requests or general ideas on the issue tracker, or in direct messages to the primary developers. You are much more likely to get a response and start a constructive discussion by posting your ideas in the public channels created for these purposes.
## Reporting Issues
If you have found a bug or issue in this project, please report it using the [issue tracker](https://github.com/markqvist/Reticulum/issues). Be sure to include details on how to reproduce the bug.
Anything submitted to the issue tracker that does not follow these guidelines will be closed and removed without comments or explanation.
## Writing Code ## Writing Code
Patches and code contributions are *generally* only welcomed from developers already known to the community, with a proven track-record of competence, skill, careful and considerate design and responsible conduct. If we do not know you yet, but you have created something that you believe lives up to such standards, we are eager to hear from you. If you are interested in contributing code, fixing open issues or adding features, please coordinate the effort with the maintainer or one of the main developers **before** submitting a pull request. Before deciding to contribute, it is also a good idea to ensure your efforts are in alignment with the [Roadmap](./Roadmap.md) and current development focus.
Current development priorities and work documents are not publicly available on the internet, but shared and coordinated directly between the developers over Reticulum itself, following the principles outlined in [Distributed Development](https://reticulum.network/manual/distributed.html). Pull requests have a high chance of being accepted if they are:
## Commit Signing - In alignment with the [Roadmap](./Roadmap.md) or solve an open issue or feature request
- Sufficiently tested to work with all API functions, and pass the standard test suite
- Functionally and conceptually complete and well-designed
- Not simply formatting or code style changes
- Well-documented
All commits in Reticulum and related projects **must** be signed with a Reticulum identity. If you want your author credentials to be included directly in the commit log, ensure that you have [configured commit signing](https://reticulum.network/manual/git.html#commit-signing). Bare patches submitted manually or fetched remote commits without signatures are still very welcome, but we will review, squash merge and sign with the key of the maintainer that approved the changes. Attribution will be provided in the commit message instead (unless you wish to remain anonymous). Even new ideas and proposals that have not been approved by a maintainer, or fall outside the established roadmap, are *occasionally* accepted - if they possess the remaining of the above qualities. If not, they will be closed and removed without comments or explanation.
## Machine Assistance & LLMs ## Generative AI Policy
For a highly complex, critical piece of cryptographically based networking infrastructure, we should not need to state the following, but alas, experience has proven otherwise. Therefore: Contributions written using large language models (LLMs) or other generative 'AI' programs are prohibited. LLMs produce errors so frequently and in a way that is so unlike human error that such issues are incredibly time-consuming to spot and fix. This is not a worthwhile tradeoff for Reticulum.
**Including material generated using LLMs or other "AI" programs in Reticulum is, almost without exception, prohibited.** This applies to all Reticulum-related projects and documentation, as well as all submitted issues and discussion in official channels, except in cases where language translation and/or speech recogntion technologies are required for communication.
Before even *thinking* about using a generative software system to "create" something for Reticulum, read [Brandolini's Reference](https://reticulum.network/manual/brandolinis.html), in full. If you do not have the capacity for that, you have failed the first requirement for contributing: Basic literacy.
*If* you are not already a known contributor with a proven record of being able to responsibly handle these tools, a **full** audit trail of all ML-utilizing workflows, including prompts, LLM reasoning traces, tool calls and outputs (that is, the *entire*, unaltered session log) will need to be provided with any proposed contribution. Without that, we will not even *consider* looking at it.
This applies to all Reticulum-related projects and documentation, as well as all submitted issues, reports and discussion, **except** of course in cases where language translation, speech recogntion, or other assistive technologies are required.
In **all** circumstances, clear disclosure is **mandatory**. Any piece of material, file, code snippet, method, function or similar that was generated with machine assistance must be prepended with a disclosure header or comment that states the framework, exact model used and reviewer, such as:
```python
# Generated with lc/deepseek_v4_flash_q8_k_xl
# Reviewed and corrected by Mark Qvist
```
Additionally, any explanatory comments or doc-strings generated by software must be manually re-written in clear, concise language by the reviewer stated in the disclosure header.
## Contributor License Agreement ## Contributor License Agreement
By contributing code to this project, you agree that copyright for the code is transferred to the Reticulum maintainers and that the code is irrevocably placed under the [Reticulum License](./LICENSE). By contributing code to this project, you agree that copyright for the code is transferred to the Reticulum maintainers and that the code is irrevocably placed under the [Reticulum License](./LICENSE).
For a more in-depth elaboration on the current CLA position, see [this post](https://rns.recipes/forum/general/brandolinis-reference-or-alternatively-agentsmd?page=1#post-3002).

View file

@ -42,9 +42,6 @@ def server(configpath):
"linkexample" "linkexample"
) )
# We can optionally configure a link request stamp cost
server_destination.set_link_cost(14.2)
# We configure a function that will get called every time # We configure a function that will get called every time
# a new client creates a link to this destination. # a new client creates a link to this destination.
server_destination.set_link_established_callback(client_connected) server_destination.set_link_established_callback(client_connected)

View file

@ -92,7 +92,6 @@ def resource_concluded(resource):
if resource.status == RNS.Resource.COMPLETE: if resource.status == RNS.Resource.COMPLETE:
RNS.log(f"Resource {resource} received") RNS.log(f"Resource {resource} received")
RNS.log(f"Metadata: {resource.metadata}") RNS.log(f"Metadata: {resource.metadata}")
RNS.log(f"Application flag is {'set' if resource.app_flag else 'not set'}")
RNS.log(f"Data length: {os.stat(resource.data.name).st_size}") RNS.log(f"Data length: {os.stat(resource.data.name).st_size}")
RNS.log(f"Data can be read directly from: {resource.data}") RNS.log(f"Data can be read directly from: {resource.data}")
RNS.log(f"Data can be moved or copied from: {resource.data.name}") RNS.log(f"Data can be moved or copied from: {resource.data.name}")
@ -193,22 +192,16 @@ def client_loop():
RNS.log(f"First 32 bytes of data: {RNS.hexrep(data[:32])}") RNS.log(f"First 32 bytes of data: {RNS.hexrep(data[:32])}")
# Generate some metadata # Generate some metadata
metadata = {"text": random_text_generator(), metadata = {"text": random_text_generator(), "numbers": [1,2,3,4], "blob": os.urandom(16)}
"numbers": [1,2,3,4], "blob": os.urandom(16)}
# An optional application-specific flag can be set
flag_set = True
# Send the resource # Send the resource
resource = RNS.Resource(data, server_link, metadata=metadata, app_flag=flag_set, resource = RNS.Resource(data, server_link, metadata=metadata, callback=resource_concluded_sending, auto_compress=False)
callback=resource_concluded_sending, auto_compress=False)
# Alternatively, you can stream data # Alternatively, you can stream data
# directly from an open file descriptor # directly from an open file descriptor
# with open("/path/to/file", "rb") as data_file: # with open("/path/to/file", "rb") as data_file:
# resource = RNS.Resource(data_file, server_link, metadata=metadata, # resource = RNS.Resource(data_file, server_link, metadata=metadata, callback=resource_concluded_sending, auto_compress=False)
# callback=resource_concluded_sending, auto_compress=False)
except Exception as e: except Exception as e:
RNS.log("Error while sending resource over the link: "+str(e)) RNS.log("Error while sending resource over the link: "+str(e))

View file

@ -239,7 +239,7 @@ def link_established(link):
RNS.log("Link established with server, sending...") RNS.log("Link established with server, sending...")
rd = os.urandom(link.mdu) rd = os.urandom(link.mdu)
started = time.time() started = time.time()
while link.status != RNS.Link.CLOSED and data_sent < data_cap*1.25: while link.status == RNS.Link.ACTIVE and data_sent < data_cap*1.25:
RNS.Packet(server_link, rd, create_receipt=False).send() RNS.Packet(server_link, rd, create_receipt=False).send()
data_sent += len(rd) data_sent += len(rd)

View file

@ -21,19 +21,6 @@ clean:
@-rm -rf ./tests/__pycache__ @-rm -rf ./tests/__pycache__
@-rm -rf ./tests/rnsconfig/storage @-rm -rf ./tests/rnsconfig/storage
@-rm -rf ./*.egg-info @-rm -rf ./*.egg-info
@-rm -rf ./RNS/*.c
@-rm -rf ./RNS/Cryptography/*.c
@-rm -rf ./RNS/Cryptography/aes/*.c
@-rm -rf ./RNS/Cryptography/pure25519/*.c
@-rm -rf ./RNS/Interfaces/*.c
@-rm -rf ./RNS/Interfaces/util/*.c
@-rm -rf ./RNS/Interfaces/Android/*.c
@-rm -rf ./RNS/Utilities/*.c
@-rm -rf ./RNS/Utilities/rngit/*.c
@-rm -rf ./RNS/Utilities/rnsh/*.c
@-rm -rf ./RNS/vendor/*.c
@-rm -rf ./RNS/vendor/i2plib/*.c
@-rm -rf
@make -C docs clean @make -C docs clean
@echo Done @echo Done
@ -63,46 +50,28 @@ build_wheel:
build_pure_wheel: build_pure_wheel:
python3 setup.py bdist_wheel --pure python3 setup.py bdist_wheel --pure
build_native:
python3 setup.py bdist_wheel --native
documentation: documentation:
make -C docs html markdown make -C docs html markdown
manual: manual:
make -C docs latexpdf epub make -C docs latexpdf epub
distcollect:
mv docs/Reticulum\ Manual.* dist
build_spkg: remove_symlinks build_sdist create_symlinks build_spkg: remove_symlinks build_sdist create_symlinks
release: test remove_symlinks build_sdist build_wheel build_pure_wheel documentation manual distcollect create_symlinks release: test remove_symlinks build_sdist build_wheel build_pure_wheel documentation manual create_symlinks
debug: remove_symlinks build_wheel build_pure_wheel create_symlinks debug: remove_symlinks build_wheel build_pure_wheel create_symlinks
native: remove_symlinks build_native create_symlinks upload: upload-rns upload-rnspure
local: release sign upload-rns:
sign:
rngit release rns://7649a50d84610232d1416b41d2896aff/reticulum/reticulum create $$(python setup.py --getversion):dist --name rns --local
upload:
@echo Ready to publish release over Reticulum
@read VOID
rngit release rns://7649a50d84610232d1416b41d2896aff/reticulum/reticulum create $$(python setup.py --getversion):dist --name rns
upload-pip: upload-rns-pip upload-rnspure-pip
upload-rns-pip:
@echo Ready to publish rns release, hit enter to continue @echo Ready to publish rns release, hit enter to continue
@read VOID @read VOID
@echo Uploading to PyPi... @echo Uploading to PyPi...
twine upload dist/rns-*.whl dist/rns-*.tar.gz twine upload dist/rns-*.whl dist/rns-*.tar.gz
@echo Release published @echo Release published
upload-rnspure-pip: upload-rnspure:
@echo Ready to publish rnspure release, hit enter to continue @echo Ready to publish rnspure release, hit enter to continue
@read VOID @read VOID
@echo Uploading to PyPi... @echo Uploading to PyPi...

274
README.md
View file

@ -1,37 +1,45 @@
Reticulum Network Stack <img align="right" src="https://static.pepy.tech/personalized-badge/rns?period=month&units=international_system&left_color=grey&right_color=blue&left_text=Installs/month" style="padding-left:10px"/><a href="https://github.com/markqvist/Reticulum/actions/workflows/build.yml"><img align="right" src="https://github.com/markqvist/Reticulum/actions/workflows/build.yml/badge.svg"/></a> Reticulum Network Stack <img align="right" src="https://static.pepy.tech/personalized-badge/rns?period=month&units=international_system&left_color=grey&right_color=blue&left_text=Installs/month" style="padding-left:10px"/><a href="https://github.com/markqvist/Reticulum/actions/workflows/build.yml"><img align="right" src="https://github.com/markqvist/Reticulum/actions/workflows/build.yml/badge.svg"/></a>
========== ==========
> [!WARNING]
> A significant number of hastily launched, incorrect, dubious, directly harmful and, in most cases, entirely LLM-generated fakes of Reticulum implementations, "documentation", "guides" and user-facing applications are currently being marketed under polished, but ultimately misleading or outright deceptive claims.
>
> For more detailed information, see the relevant sections of the [Brandolini's Reference](https://reticulum.network/manual/brandolinis.html) chapter of the manual. Known good community implementations are listed in the [Community Implementations](#community-implementations) section of this readme. For software and projects recognized by the Reticulum community, see the [Programs Using Reticulum](https://reticulum.network/manual/software.html) chapter of the manual.
>
> Machine-generated content derived from the RNS source code or documentation (such as "implementations", "guides" for sale, et cetera), are in several, well-documented cases violating the open source licenses that granted rights of copying and distribution.
>
> Any claimed assertion of copyright or grant of license (Apache, GPL, et cetera) that such projects are making are [void grants](https://reticulum.network/manual/brandolinis.html#void-grants-legal-foundations-of-machine-generated-code). Both the violators themselves, and all downstream projects using the infringing derivatives, are **directly** and personally liable for the legal consequences.
>
> Do **not** use these "projects", they are a hazard to the entire ecosystem we have been carefully building over the last ten years.
<p align="center"><img width="200" src="https://raw.githubusercontent.com/markqvist/Reticulum/master/docs/source/graphics/rns_logo_512.png"></p> <p align="center"><img width="200" src="https://raw.githubusercontent.com/markqvist/Reticulum/master/docs/source/graphics/rns_logo_512.png"></p>
*This repository is [a public mirror](./MIRROR.md). All development is happening elsewhere.* *This repository is [a public mirror](./MIRROR.md). All development is happening elsewhere.*
To understand the foundational philosophy and goals of this system, read the [Zen of Reticulum](Zen%20of%20Reticulum.md). To understand the foundational philosophy and goals of this system, read the [Zen of Reticulum](Zen%20of%20Reticulum.md).
Reticulum is the cryptography-based networking stack for building local and wide-area networks with readily available hardware. It can operate even with very high latency and extremely low bandwidth. Reticulum allows you to build wide-area networks with off-the-shelf tools, and offers end-to-end encryption and connectivity, initiator anonymity, autoconfiguring cryptographically backed multi-hop transport, efficient addressing, unforgeable delivery acknowledgements and more. Reticulum is the cryptography-based networking stack for building local and wide-area
networks with readily available hardware. It can operate even with very high latency
and extremely low bandwidth. Reticulum allows you to build wide-area networks
with off-the-shelf tools, and offers end-to-end encryption and connectivity,
initiator anonymity, autoconfiguring cryptographically backed multi-hop
transport, efficient addressing, unforgeable delivery acknowledgements and
more.
The vision of Reticulum is to allow anyone to be their own network operator, and to make it cheap and easy to cover vast areas with a myriad of independent, inter-connectable and autonomous networks. Reticulum **is not** *one* network. It is **a tool** for building *thousands of networks*. Networks without kill-switches, surveillance, censorship and control. Networks that can freely interoperate, associate and disassociate with each other, and require no central oversight. Networks for human beings. *Networks for the people*. The vision of Reticulum is to allow anyone to be their own network operator,
and to make it cheap and easy to cover vast areas with a myriad of independent,
inter-connectable and autonomous networks. Reticulum **is not** *one* network.
It is **a tool** for building *thousands of networks*. Networks without
kill-switches, surveillance, censorship and control. Networks that can freely
interoperate, associate and disassociate with each other, and require no
central oversight. Networks for human beings. *Networks for the people*.
Reticulum is a complete networking stack, and does not rely on IP or higher layers, but it is possible to use IP as the underlying carrier for Reticulum. It is therefore trivial to tunnel Reticulum over the Internet or private IP networks. Reticulum is a complete networking stack, and does not rely on IP or higher
layers, but it is possible to use IP as the underlying carrier for Reticulum.
It is therefore trivial to tunnel Reticulum over the Internet or private IP
networks.
Having no dependencies on traditional networking stacks frees up overhead that has been used to implement a networking stack built directly on cryptographic principles, allowing resilience and stable functionality, even in open and trustless networks. Having no dependencies on traditional networking stacks frees up overhead that
has been used to implement a networking stack built directly on cryptographic
principles, allowing resilience and stable functionality, even in open and
trustless networks.
No kernel modules or drivers are required. Reticulum runs completely in userland, and can run on practically any system that runs Python 3. No kernel modules or drivers are required. Reticulum runs completely in
userland, and can run on practically any system that runs Python 3.
## Read The Manual ## Read The Manual
The full documentation for Reticulum is available at [markqvist.github.io/Reticulum/manual/](https://markqvist.github.io/Reticulum/manual/). The full documentation for Reticulum is available at [markqvist.github.io/Reticulum/manual/](https://markqvist.github.io/Reticulum/manual/).
You can also download the [Reticulum manual as a PDF](https://github.com/markqvist/Reticulum/releases/latest/download/Reticulum.Manual.pdf) or [as an e-book in EPUB format](https://github.com/markqvist/Reticulum/releases/latest/download/Reticulum.Manual.epub). You can also download the [Reticulum manual as a PDF](https://github.com/markqvist/Reticulum/raw/master/docs/Reticulum%20Manual.pdf) or [as an e-book in EPUB format](https://github.com/markqvist/Reticulum/raw/master/docs/Reticulum%20Manual.epub).
For more info, see [reticulum.network](https://reticulum.network/) and [the FAQ section of the wiki](https://github.com/markqvist/Reticulum/wiki/Frequently-Asked-Questions). For more info, see [reticulum.network](https://reticulum.network/) and [the FAQ section of the wiki](https://github.com/markqvist/Reticulum/wiki/Frequently-Asked-Questions).
@ -70,60 +78,25 @@ For more info, see [reticulum.network](https://reticulum.network/) and [the FAQ
- Low cost of keeping links open at only 0.44 bits per second - Low cost of keeping links open at only 0.44 bits per second
- Reliable sequential delivery with Channel and Buffer mechanisms - Reliable sequential delivery with Channel and Buffer mechanisms
## Community Resources
The references listed here are not run, managed or moderated by the Reticulum project or its maintainer. They are provided by dedicated, independent volunteers in the community, for the benefit of anyone interested in learning more about Reticulum.
- [Reticulum Community Wiki](https://reticulum.miraheze.org/wiki)
- [RNS Recipes](https://rns.recipes/)
- [RNS Moscow](https://rns.moscow/)
- [RMAP.world](https://rmap.world/)
If you find anything that is no longer accurate, incorrect or simply not quite up to date, do not demand or complain towards the maintainers of these resources. Instead, take the task upon yourself to help out and improve things by working on, and offering the contributions you would like to see, like the creators of the above did themselves.
## Reference Implementation ## Reference Implementation
The Python code in this repository is the Reference Implementation of Reticulum. The Reticulum Protocol is defined entirely and authoritatively by this reference implementation, and its associated manual. It is maintained by Mark Qvist, identified by the Reticulum Identity `<bc7291552be7a58f361522990465165c>`. The Python code in this repository is the Reference Implementation of Reticulum.
The Reticulum Protocol is defined entirely and authoritatively by this reference
implementation, and its associated manual. It is maintained by Mark Qvist,
identified by the Reticulum Identity `<bc7291552be7a58f361522990465165c>`.
No second-order protocol specification derivatives currently do, or will ever exist. You can't get a better specification than one that is executable, functional, testable, deployed and operating 24/7 across the planet. Compatibility with the Reticulum Protocol is defined as having full interoperability,
and sufficient functional parity with this reference implementation. Any specific protocol
There is not, and never will be a separate, derived "formal spec", "RFC" or anything similar for Reticulum: implementation that achieves this is Reticulum. Any that does not is not Reticulum.
**The reference implementation *is* the final, authoritative specification.**
Compatibility with the Reticulum Protocol is defined as having full interoperability, and sufficient functional parity with this reference implementation. Any specific protocol implementation that achieves this is Reticulum. Any that does not is not Reticulum.
The reference implementation is licensed under the Reticulum License. The reference implementation is licensed under the Reticulum License.
The Reticulum Protocol was dedicated to the Public Domain in 2016. The Reticulum Protocol was dedicated to the Public Domain in 2016.
## Community Implementations
The implementations listed here have demonstrated wire-compatibility, sufficient feature parity, sensible security and development practices, long-term commitment and care for implementation quality. Additionally, they have a track record of at least 18 months of active development.
Most importantly, their developers and maintainers are recognized by the community to actually know what they are doing, and as having a commitment to improving the overall Reticulum ecosystem.
- [Chad Atterman's](https://github.com/attermann) [microReticulum](https://github.com/attermann/microReticulum/) is a C++ implementation targeting 32-bit micro-controllers.
- [Quad4's](https://quad4.io/) [Reticulum-Go](https://reticulum-go.quad4.io/) is a Go implementation targeting both desktop, mobile and embedded systems.
*If you are the maintainer of an implementation not listed above, and believe that it deserves inclusion by following the same standards of rigor and commitment, your proposal will be welcomed.*
---
**Warning!** A significant number of hastily launched, incorrect, dubious, directly harmful and, in most cases, entirely LLM-generated fakes of Reticulum implementations (and user facing applications) are currently being marketed and pushed under polished, but ultimately misleading or outright deceptive claims. Most of these are either quick attention or money grabs, completely unreviewed, incompatible, without maintenance or human oversight, actively trying to fragment the community, or a combination thereof.
---
For more extensive information on this topic, read the [Brandolini's Reference](https://reticulum.network/manual/brandolinis.html) chapter of the manual (for the legal foundations, see [Void Grants](https://reticulum.network/manual/brandolinis.html#void-grants-legal-foundations-of-machine-generated-code) specifically). The chapter is very long, very detailed, and is the polite, measured analysis, intended as an evolving, sectioned reference on these issues.
If you would rather just have the short, unfiltered and quite angry version, which consists of an enacted response to one particular manifestation of the phenomenon as a whole, used as an example, you can read [Yes, I am Angry](docs/history/2026_09_18_Yes_I_am_Angry.md). It gets the same point across, in a very different language, if you can handle that sort of communication.
The core Reticulum maintainers and developers **strongly advise against using any such "implementation"**. The [deceitful and self-serving behavior](https://reticulum.network/manual/brandolinis.html#a-persistent-but-fragile-triad) of a small number of either malintented, attention-seeking or incompetent individuals has caused significant [damage and waste of time and resources](https://reticulum.network/manual/brandolinis.html#intent-consequence) for both developers, node operators, users and the community and Reticulum project at large.
We kindly, but urgently ask all users to excercise caution, common sense and healthy skepticism when [evaluating](https://reticulum.network/manual/brandolinis.html#evaluating-a-reticulum-implementation) the potential use of any implementation not listed above; and to accept that **choosing to do so regardless translates the responsibility for potential damage from an irresponsible or malicious individual onto yourself**.
## Examples of Reticulum Applications ## Examples of Reticulum Applications
If you want to quickly get an idea of what Reticulum can do, take a look at the [Programs Using Reticulum](https://reticulum.network/manual/software.html) section of the manual, or the following resources: If you want to quickly get an idea of what Reticulum can do, take a look at the
[Programs Using Reticulum](https://reticulum.network/manual/software.html)
section of the manual, or the following resources:
- [LXMF](https://github.com/markqvist/lxmf) is a distributed, delay and disruption tolerant message transfer protocol built on Reticulum - [LXMF](https://github.com/markqvist/lxmf) is a distributed, delay and disruption tolerant message transfer protocol built on Reticulum
- The [LXST](https://github.com/markqvist/lxst) protocol and framework provides real-time audio and signals transport over Reticulum. It includes primitives and utilities for building voice-based applications and hardware devices, such as the `rnphone` program, that can be used to build hardware telephones. - The [LXST](https://github.com/markqvist/lxst) protocol and framework provides real-time audio and signals transport over Reticulum. It includes primitives and utilities for building voice-based applications and hardware devices, such as the `rnphone` program, that can be used to build hardware telephones.
@ -131,27 +104,50 @@ If you want to quickly get an idea of what Reticulum can do, take a look at the
- The Android, Linux, macOS and Windows app [Sideband](https://github.com/markqvist/Sideband) has a graphical interface and many advanced features, such as file transfers, image and voice messages, real-time voice calls, a distributed telemetry system, mapping capabilities and full plugin extensibility. - The Android, Linux, macOS and Windows app [Sideband](https://github.com/markqvist/Sideband) has a graphical interface and many advanced features, such as file transfers, image and voice messages, real-time voice calls, a distributed telemetry system, mapping capabilities and full plugin extensibility.
- [MeshChatX](https://git.quad4.io/RNS-Things/MeshChatX) is a full-featured LXMF client with many built-in tools and functionalities, that also supports image and voice messages, file transfers and voice calls. It also includes a built-in page browser for browsing Nomad Network nodes. - [MeshChatX](https://git.quad4.io/RNS-Things/MeshChatX) is a full-featured LXMF client with many built-in tools and functionalities, that also supports image and voice messages, file transfers and voice calls. It also includes a built-in page browser for browsing Nomad Network nodes.
- You can use the included [rnsh](https://reticulum.network/manual/using.html#the-rnsh-utility) program to establish remote shell sessions over Reticulum. - You can use the included [rnsh](https://reticulum.network/manual/using.html#the-rnsh-utility) program to establish remote shell sessions over Reticulum.
- The included [rngit](https://reticulum.network/manual/git.html) system provides a full solution for managing, hosting, collaborating on and interacting with Git repositories over Reticulum.
## Where can Reticulum be used? ## Where can Reticulum be used?
Over practically any medium that can support at least a half-duplex channel with greater throughput than 5 bits per second, and an MTU of 500 bytes. Data radios, modems, LoRa radios, serial lines, AX.25 TNCs, amateur radio digital modes, WiFi and Ethernet devices, free-space optical links, and similar systems are all examples of the types of physical devices Reticulum can use. Over practically any medium that can support at least a half-duplex channel
with greater throughput than 5 bits per second, and an MTU of 500 bytes. Data radios,
modems, LoRa radios, serial lines, AX.25 TNCs, amateur radio digital modes,
WiFi and Ethernet devices, free-space optical links, and similar systems are
all examples of the types of physical devices Reticulum can use.
An open-source LoRa-based interface called [RNode](https://markqvist.github.io/Reticulum/manual/hardware.html#rnode) has been designed specifically for use with Reticulum. It is possible to build yourself, or it can be purchased as a complete transceiver that just needs a USB, WiFi or Bluetooth connection to the host. An open-source LoRa-based interface called
[RNode](https://markqvist.github.io/Reticulum/manual/hardware.html#rnode) has
been designed specifically for use with Reticulum. It is possible to build
yourself, or it can be purchased as a complete transceiver that just needs a
USB connection to the host.
Reticulum can also be encapsulated over existing IP networks, so there's nothing stopping you from using it over wired Ethernet, your local WiFi network or the Internet, where it'll work just as well. In fact, one of the strengths of Reticulum is how easily it allows you to connect different mediums into a self-configuring, resilient and encrypted mesh, using any available mixture of available infrastructure. Reticulum can also be encapsulated over existing IP networks, so there's
nothing stopping you from using it over wired Ethernet, your local WiFi network
or the Internet, where it'll work just as well. In fact, one of the strengths
of Reticulum is how easily it allows you to connect different mediums into a
self-configuring, resilient and encrypted mesh, using any available mixture of
available infrastructure.
As an example, it's possible to set up a Raspberry Pi connected to both a LoRa radio, a packet radio TNC and a WiFi network. Once the interfaces are configured, Reticulum will take care of the rest, and any device on the WiFi network can communicate with nodes on the LoRa and packet radio sides of the network, and vice versa. As an example, it's possible to set up a Raspberry Pi connected to both a LoRa
radio, a packet radio TNC and a WiFi network. Once the interfaces are
configured, Reticulum will take care of the rest, and any device on the WiFi
network can communicate with nodes on the LoRa and packet radio sides of the
network, and vice versa.
## How do I get started? ## How do I get started?
The best way to get started with the Reticulum Network Stack depends on what you want to do. For full details and examples, have a look at the [Getting Started Fast](https://markqvist.github.io/Reticulum/manual/gettingstartedfast.html) section of the [Reticulum Manual](https://markqvist.github.io/Reticulum/manual/). The best way to get started with the Reticulum Network Stack depends on what
you want to do. For full details and examples, have a look at the
[Getting Started Fast](https://markqvist.github.io/Reticulum/manual/gettingstartedfast.html)
section of the [Reticulum Manual](https://markqvist.github.io/Reticulum/manual/).
To simply install Reticulum and related utilities on your system, the easiest way is via `pip`. You can then start any program that uses Reticulum, or start Reticulum as a system service with [the rnsd utility](https://markqvist.github.io/Reticulum/manual/using.html#the-rnsd-utility). To simply install Reticulum and related utilities on your system, the easiest way is via `pip`.
You can then start any program that uses Reticulum, or start Reticulum as a system service with
[the rnsd utility](https://markqvist.github.io/Reticulum/manual/using.html#the-rnsd-utility).
```bash ```bash
pip install rns pip install rns
``` ```
If you are using an operating system that blocks normal user package installation via `pip`, you can return `pip` to normal behaviour by editing the `~/.config/pip/pip.conf` file, and adding the following directive in the `[global]` section: If you are using an operating system that blocks normal user package installation via `pip`,
you can return `pip` to normal behaviour by editing the `~/.config/pip/pip.conf` file,
and adding the following directive in the `[global]` section:
```text ```text
[global] [global]
@ -164,14 +160,23 @@ Alternatively, you can use the `pipx` tool to install Reticulum in an isolated e
pipx install rns pipx install rns
``` ```
When first started, Reticulum will create a default configuration file, providing basic connectivity to other Reticulum peers that might be locally reachable. The default config file contains a few examples, and references for creating a more complex configuration. When first started, Reticulum will create a default configuration file,
providing basic connectivity to other Reticulum peers that might be locally
reachable. The default config file contains a few examples, and references for
creating a more complex configuration.
If you have an old version of `pip` on your system, you may need to upgrade it first with `pip install pip --upgrade`. If you no not already have `pip` installed, you can install it using the package manager of your system with `sudo apt install python3-pip` or similar. If you have an old version of `pip` on your system, you may need to upgrade it first with `pip install pip --upgrade`. If you no not already have `pip` installed, you can install it using the package manager of your system with `sudo apt install python3-pip` or similar.
For more detailed examples on how to expand communication over many mediums such as packet radio or LoRa, serial ports, or over fast IP links and the Internet using the UDP and TCP interfaces, take a look at the [Supported Interfaces](https://markqvist.github.io/Reticulum/manual/interfaces.html) section of the [Reticulum Manual](https://markqvist.github.io/Reticulum/manual/). For more detailed examples on how to expand communication over many mediums such
as packet radio or LoRa, serial ports, or over fast IP links and the Internet using
the UDP and TCP interfaces, take a look at the [Supported Interfaces](https://markqvist.github.io/Reticulum/manual/interfaces.html)
section of the [Reticulum Manual](https://markqvist.github.io/Reticulum/manual/).
## Included Utilities ## Included Utilities
Reticulum includes a range of useful utilities for managing your networks, viewing status and information, and other tasks. You can read more about these programs in the [Included Utility Programs](https://markqvist.github.io/Reticulum/manual/using.html#included-utility-programs) section of the [Reticulum Manual](https://markqvist.github.io/Reticulum/manual/). Reticulum includes a range of useful utilities for managing your networks,
viewing status and information, and other tasks. You can read more about these
programs in the [Included Utility Programs](https://markqvist.github.io/Reticulum/manual/using.html#included-utility-programs)
section of the [Reticulum Manual](https://markqvist.github.io/Reticulum/manual/).
- The system daemon `rnsd` for running Reticulum as an always-available service - The system daemon `rnsd` for running Reticulum as an always-available service
- An interface status utility called `rnstatus`, that displays information about interfaces - An interface status utility called `rnstatus`, that displays information about interfaces
@ -184,13 +189,19 @@ Reticulum includes a range of useful utilities for managing your networks, viewi
- The `rngit` program provides a full multi-repository Git node for serving repositories over Reticulum - The `rngit` program provides a full multi-repository Git node for serving repositories over Reticulum
- The included `git-remote-rns` helper allows you to interact with Git repositories over Reticulum - The included `git-remote-rns` helper allows you to interact with Git repositories over Reticulum
All tools, including `rnx` and `rncp`, work reliably and well even over very low-bandwidth links like LoRa or Packet Radio. For full-featured remote shells over Reticulum, also have a look at the [rnsh](https://github.com/acehoss/rnsh) program. All tools, including `rnx` and `rncp`, work reliably and well even over very
low-bandwidth links like LoRa or Packet Radio. For full-featured remote shells
over Reticulum, also have a look at the [rnsh](https://github.com/acehoss/rnsh)
program.
## Supported interface types and devices ## Supported interface types and devices
Reticulum implements a range of generalised interface types that covers most of the communications hardware that Reticulum can run over. If your hardware is not supported, it's [simple to implement a custom interface module](https://markqvist.github.io/Reticulum/manual/interfaces.html#custom-interfaces). Reticulum implements a range of generalised interface types that covers most of
the communications hardware that Reticulum can run over. If your hardware is
not supported, it's [simple to implement a custom interface module](https://markqvist.github.io/Reticulum/manual/interfaces.html#custom-interfaces).
Contributions of custom interfaces are gratefully accepted, provided they are generally useful and well-tested in real-world usage. Pull requests for custom interfaces are gratefully accepted, provided they are
generally useful and well-tested in real-world usage.
Currently, the following built-in interfaces are supported: Currently, the following built-in interfaces are supported:
@ -205,42 +216,62 @@ Currently, the following built-in interfaces are supported:
- Custom hardware via stdio or pipes - Custom hardware via stdio or pipes
## Performance ## Performance
*All performance numbers here assume `rnsd` running in fully interpreted Python mode, limited to a single CPU core.* Reticulum targets a *very* wide usable performance envelope, but prioritises
functionality and performance on low-bandwidth mediums. The goal is to
provide a dynamic performance envelope from 250 bits per second, to 1 gigabit
per second on normal hardware.
Reticulum targets a *very* wide usable performance envelope, but prioritises functionality and performance on low-bandwidth mediums. The goal is to provide a dynamic performance envelope from 100 bits per second, to 10 gigabits per second on normal hardware. Currently, the usable performance envelope is approximately 150 bits per second
to 500 megabits per second, with physical mediums faster than that not being
Currently, the on-network, usable performance envelope is approximately 150 bits per second to 1 gigabit per second on modest hardware. Real-world performance will vary widely with the speed of your system, availability of cryptographic hardware acceleration, and other factors. saturated. Performance beyond the current level is intended for future
upgrades, but not highly prioritised at this point in time.
Raw routing throughput for a transport node hosted on a Raspberry Pi 5 is approximately 5 gigabits per second. Systems with modern crypto hardware acceleration can achieve transport throughput of 15 gigabits per second and higher.
While the reference implementation is perfectly capable of reaching 100+ Gbps, by utilizing multiple CPU cores and vectorized transport, performance beyond the current level is not higly prioritised, since RNS is already able to saturate available physical medium bandwidth in almost all real-world situations.
The primary objectives for RNS are *reliability*, *correctness*, *security*, *proper traffic management* and *actually practical performance*, not hollow on-paper "benchmarks" with little to no bearing on real-world usability.
All alternative-implementation maintainers are *highly* encouraged to adopt a similar approach, and especially to focus on getting their implementations working properly and reliably before publishing empirically dubious or factually wrong performance claims.
## Current Status ## Current Status
All core protocol features are implemented and functioning, but additions will probably occur as real-world use is explored and understood. The API and wire-format can be considered stable. All core protocol features are implemented and functioning, but additions will
probably occur as real-world use is explored and understood. The API and wire-format
can be considered stable.
## Dependencies ## Dependencies
The installation of the default `rns` package requires only two external dependencies, listed below. Almost all systems and distributions have readily available packages for these dependencies, and when the `rns` package is installed with `pip`, they will be downloaded and installed as well. The installation of the default `rns` package requires only two external dependencies, listed
below. Almost all systems and distributions have readily available packages for
these dependencies, and when the `rns` package is installed with `pip`, they
will be downloaded and installed as well.
- [PyCA/cryptography](https://github.com/pyca/cryptography) - [PyCA/cryptography](https://github.com/pyca/cryptography)
- [pyserial](https://github.com/pyserial/pyserial) - [pyserial](https://github.com/pyserial/pyserial)
On more unusual systems, and in some rare cases, it might not be possible to install or even compile one or more of the above modules. In such situations, you can use the `rnspure` package instead, which require no external dependencies for installation. Please note that the contents of the `rns` and `rnspure` packages are *identical*. The only difference is that the `rnspure` package lists no dependencies required for installation. On more unusual systems, and in some rare cases, it might not be possible to
install or even compile one or more of the above modules. In such situations,
you can use the `rnspure` package instead, which require no external
dependencies for installation. Please note that the contents of the `rns` and
`rnspure` packages are *identical*. The only difference is that the `rnspure`
package lists no dependencies required for installation.
No matter how Reticulum is installed and started, it will load external dependencies only if they are *needed* and *available*. If for example you want to use Reticulum on a system that cannot support [pyserial](https://github.com/pyserial/pyserial), it is perfectly possible to do so using the `rnspure` package, but Reticulum will not be able to use serial-based interfaces. All other available modules will still be loaded when needed. No matter how Reticulum is installed and started, it will load external
dependencies only if they are *needed* and *available*. If for example you want
to use Reticulum on a system that cannot support
[pyserial](https://github.com/pyserial/pyserial), it is perfectly possible to
do so using the `rnspure` package, but Reticulum will not be able to use
serial-based interfaces. All other available modules will still be loaded when
needed.
**Please Note!** If you use the `rnspure` package to run Reticulum on systems that do not support [PyCA/cryptography](https://github.com/pyca/cryptography), it is important that you read and understand the [Cryptographic Primitives](#cryptographic-primitives) section of this document. **Please Note!** If you use the `rnspure` package to run Reticulum on systems
that do not support [PyCA/cryptography](https://github.com/pyca/cryptography),
it is important that you read and understand the [Cryptographic
Primitives](#cryptographic-primitives) section of this document.
## Bootstrapping Connectivity ## Bootstrapping Connectivity
Reticulum is not a service you subscribe to, nor is it a single global network you "join". Reticulum provides functionality for discovering available public interfaces over the network itself, and the broader community has provided various directories of publicly available entrypoints to bootstrap connectivity. Reticulum is not a service you subscribe to, nor is it a single global network you "join".
Reticulum provides functionality for discovering available public interfaces
over the network itself, and the broader community has provided various directories
of publicly available entrypoints to bootstrap connectivity.
To learn how to establish initial connectivity over Reticulum, read the [Bootstrapping Connectivity](https://reticulum.network/manual/gettingstartedfast.html#bootstrapping-connectivity) section of the manual. To learn how to establish initial connectivity over Reticulum, read the [Bootstrapping Connectivity](https://reticulum.network/manual/gettingstartedfast.html#bootstrapping-connectivity) section of the manual.
If you already have a general idea of how this works, you can use community-run sites such as [directory.rns.recipes](https://directory.rns.recipes/) and [rmap.world](https://rmap.world) to find interface definitions for initial connectivity to the global distributed Reticulum backbone. If you already have a general idea of how this works, you can use community-run
sites such as [directory.rns.recipes](https://directory.rns.recipes/) and [rmap.world](https://rmap.world)
to find interface definitions for initial connectivity to the global distributed Reticulum backbone.
## Public Testnet ## Public Testnet
***Important!** Historically, a developer-targeted testnet was made available by the Reticulum project itself. As the amount of global Reticulum nodes and entrypoints have grown to a substantial quantity, this public testnet, including the Amsterdam Testnet entrypoint, has now been decommissioned. If your still have instances that relied on this entrypoint for connectivity, transition to using the distributed backbone instead. Reticulum now includes a full on-network interface discovery and connectivity bootstrapping system. Read the [Bootstrapping Connectivity](https://reticulum.network/manual/gettingstartedfast.html#bootstrapping-connectivity) section of the manual for pointers.* ***Important!** Historically, a developer-targeted testnet was made available by the Reticulum project itself. As the amount of global Reticulum nodes and entrypoints have grown to a substantial quantity, this public testnet, including the Amsterdam Testnet entrypoint, has now been decommissioned. If your still have instances that relied on this entrypoint for connectivity, transition to using the distributed backbone instead. Reticulum now includes a full on-network interface discovery and connectivity bootstrapping system. Read the [Bootstrapping Connectivity](https://reticulum.network/manual/gettingstartedfast.html#bootstrapping-connectivity) section of the manual for pointers.*
@ -265,11 +296,17 @@ For this to be possible, I need your help. Please support the continued developm
- Ko-Fi: https://ko-fi.com/markqvist - Ko-Fi: https://ko-fi.com/markqvist
## Cryptographic Primitives ## Cryptographic Primitives
Reticulum uses a simple suite of efficient, strong and well-tested cryptographic primitives, with widely available implementations that can be used both on general-purpose CPUs and on microcontrollers. Reticulum uses a simple suite of efficient, strong and well-tested cryptographic
primitives, with widely available implementations that can be used both on
general-purpose CPUs and on microcontrollers.
One of the primary considerations for choosing this particular set of primitives is that they can be implemented *safely* with relatively few pitfalls, on practically all current computing platforms. One of the primary considerations for choosing this particular set of primitives is
that they can be implemented *safely* with relatively few pitfalls, on practically
all current computing platforms.
The primitives listed here **are authoritative**. Anything claiming to be Reticulum, but not using these exact primitives **is not** Reticulum, and possibly an intentionally compromised or weakened clone. The utilised primitives are: The primitives listed here **are authoritative**. Anything claiming to be Reticulum,
but not using these exact primitives **is not** Reticulum, and possibly an
intentionally compromised or weakened clone. The utilised primitives are:
- Reticulum Identity Keys are 512-bit Curve25519 keysets - Reticulum Identity Keys are 512-bit Curve25519 keysets
- A 256-bit Ed25519 key for signatures - A 256-bit Ed25519 key for signatures
@ -284,7 +321,13 @@ The primitives listed here **are authoritative**. Anything claiming to be Reticu
- SHA-256 - SHA-256
- SHA-512 - SHA-512
In the default installation configuration, the `X25519`, `Ed25519`, and `AES-256-CBC` primitives are provided by [OpenSSL](https://www.openssl.org/) (via the [PyCA/cryptography](https://github.com/pyca/cryptography) package). The hashing functions `SHA-256` and `SHA-512` are provided by the standard Python [hashlib](https://docs.python.org/3/library/hashlib.html). The `HKDF`, `HMAC`, `Token` primitives, and the `PKCS7` padding function are always provided by the following internal implementations: In the default installation configuration, the `X25519`, `Ed25519`,
and `AES-256-CBC` primitives are provided by [OpenSSL](https://www.openssl.org/)
(via the [PyCA/cryptography](https://github.com/pyca/cryptography) package).
The hashing functions `SHA-256` and `SHA-512` are provided by the standard
Python [hashlib](https://docs.python.org/3/library/hashlib.html). The `HKDF`,
`HMAC`, `Token` primitives, and the `PKCS7` padding function are always
provided by the following internal implementations:
- [HKDF.py](RNS/Cryptography/HKDF.py) - [HKDF.py](RNS/Cryptography/HKDF.py)
- [HMAC.py](RNS/Cryptography/HMAC.py) - [HMAC.py](RNS/Cryptography/HMAC.py)
@ -292,18 +335,39 @@ In the default installation configuration, the `X25519`, `Ed25519`, and `AES-256
- [PKCS7.py](RNS/Cryptography/PKCS7.py) - [PKCS7.py](RNS/Cryptography/PKCS7.py)
Reticulum also includes a complete implementation of all necessary primitives in pure Python. If OpenSSL and PyCA are not available on the system when Reticulum is started, Reticulum will instead use the internal pure-python primitives. A trivial consequence of this is performance, with the OpenSSL backend being *much* faster. The most important consequence however, is the potential loss of security by using primitives that has not seen the same amount of scrutiny, testing and review as those from OpenSSL. Reticulum also includes a complete implementation of all necessary primitives
in pure Python. If OpenSSL and PyCA are not available on the system when
Reticulum is started, Reticulum will instead use the internal pure-python
primitives. A trivial consequence of this is performance, with the OpenSSL
backend being *much* faster. The most important consequence however, is the
potential loss of security by using primitives that has not seen the same
amount of scrutiny, testing and review as those from OpenSSL.
Please note that by default, installing Reticulum will **require** OpenSSL and PyCA to also be automatically installed if not already available. It is only possible to use the pure-python primitives if this requirement is specifically overridden by the user, for example by installing the `rnspure` package instead of the normal `rns` package, or by running directly from local source-code. Please note that by default, installing Reticulum will **require** OpenSSL and
PyCA to also be automatically installed if not already available. It is only
possible to use the pure-python primitives if this requirement is specifically
overridden by the user, for example by installing the `rnspure` package instead
of the normal `rns` package, or by running directly from local source-code.
If you want to use the internal pure-python primitives, it is **highly advisable** that you have a good understanding of the risks that this pose, and make an informed decision on whether those risks are acceptable to you. If you want to use the internal pure-python primitives, it is **highly
advisable** that you have a good understanding of the risks that this pose, and
make an informed decision on whether those risks are acceptable to you.
Having been in development for ten, and wider deployment for around five years, Reticulum is relatively young software, and should be considered as such. While it has been carefully built with cryptography best-practices very foremost in mind, there could very well still be privacy or security breaking bugs. Reticulum is relatively young software, and should be considered as such. While
it has been built with cryptography best-practices very foremost in mind, it
_has not_ been externally security audited, and there could very well be
privacy or security breaking bugs. If you want to help out, or help sponsor an
audit, please do get in touch.
## Acknowledgements & Credits ## Acknowledgements & Credits
Reticulum can only exist because of the mountain of Open Source work it was built on top of, the contributions of everyone involved, and everyone that has supported the project through the years. To everyone who has helped, thank you so much. Reticulum can only exist because of the mountain of Open Source work it was
built on top of, the contributions of everyone involved, and everyone that has
supported the project through the years. To everyone who has helped, thank you
so much.
A number of other modules and projects are either part of, or used by Reticulum. Sincere thanks to the authors and contributors of the following projects: A number of other modules and projects are either part of, or used by
Reticulum. Sincere thanks to the authors and contributors of the following
projects:
- [PyCA/cryptography](https://github.com/pyca/cryptography), *BSD License* - [PyCA/cryptography](https://github.com/pyca/cryptography), *BSD License*
- [Pure-25519](https://github.com/warner/python-pure25519) by [Brian Warner](https://github.com/warner), *MIT License* - [Pure-25519](https://github.com/warner/python-pure25519) by [Brian Warner](https://github.com/warner), *MIT License*

View file

@ -4,8 +4,6 @@ To understand the foundational philosophy and goals of this system, read the `_`
Reticulum is the cryptography-based networking stack for building local and wide-area networks with readily available hardware. It can operate even with very high latency and extremely low bandwidth. Reticulum allows you to build wide-area networks with off-the-shelf tools, and offers end-to-end encryption and connectivity, initiator anonymity, autoconfiguring cryptographically backed multi-hop transport, efficient addressing, unforgeable delivery acknowledgements and more. Reticulum is the cryptography-based networking stack for building local and wide-area networks with readily available hardware. It can operate even with very high latency and extremely low bandwidth. Reticulum allows you to build wide-area networks with off-the-shelf tools, and offers end-to-end encryption and connectivity, initiator anonymity, autoconfiguring cryptographically backed multi-hop transport, efficient addressing, unforgeable delivery acknowledgements and more.
`(RNS Logo`w=18`a=c`:/media/reticulum/reticulum/HEAD/docs/source/graphics/rns_logo_256.webp)
The vision of Reticulum is to allow anyone to be their own network operator, and to make it cheap and easy to cover vast areas with a myriad of independent, inter-connectable and autonomous networks. Reticulum `!is not`! `*one`* network. It is `!a tool`! for building `*thousands of networks`*. Networks without kill-switches, surveillance, censorship and control. Networks that can freely interoperate, associate and disassociate with each other, and require no central oversight. Networks for human beings. `*Networks for the people`*. The vision of Reticulum is to allow anyone to be their own network operator, and to make it cheap and easy to cover vast areas with a myriad of independent, inter-connectable and autonomous networks. Reticulum `!is not`! `*one`* network. It is `!a tool`! for building `*thousands of networks`*. Networks without kill-switches, surveillance, censorship and control. Networks that can freely interoperate, associate and disassociate with each other, and require no central oversight. Networks for human beings. `*Networks for the people`*.
Reticulum is a complete networking stack, and does not rely on IP or higher layers, but it is possible to use IP as the underlying carrier for Reticulum. It is therefore trivial to tunnel Reticulum over the Internet or private IP networks. Reticulum is a complete networking stack, and does not rely on IP or higher layers, but it is possible to use IP as the underlying carrier for Reticulum. It is therefore trivial to tunnel Reticulum over the Internet or private IP networks.
@ -18,7 +16,7 @@ No kernel modules or drivers are required. Reticulum runs completely in userland
The full documentation for Reticulum is available on `_`!`[this node`:/page/blob.mu`g=reticulum|r=reticulum|ref=HEAD|path=docs/markdown/index.md]`!`_. The full documentation for Reticulum is available on `_`!`[this node`:/page/blob.mu`g=reticulum|r=reticulum|ref=HEAD|path=docs/markdown/index.md]`!`_.
You can also download the `_`!`[Reticulum manual as a PDF`:/file/artifact`g=reticulum|r=reticulum|t=latest|a=Reticulum+Manual.pdf]`!`_ or `_`!`[as an e-book in EPUB format`:/file/artifact`g=reticulum|r=reticulum|t=latest|a=Reticulum+Manual.epub]`!`_. You can also download the `_`!`[Reticulum manual as a PDF`:/file/download`g=reticulum|r=reticulum|ref=HEAD|path=docs%2FReticulum+Manual.pdf]`!`_ or `_`!`[as an e-book in EPUB format`:/file/download`g=reticulum|r=reticulum|ref=HEAD|path=docs%2FReticulum+Manual.pdf]`!`_.
>> Notable Features >> Notable Features
@ -60,12 +58,6 @@ You can also download the `_`!`[Reticulum manual as a PDF`:/file/artifact`g=reti
The Python code in this repository is the Reference Implementation of Reticulum. The Reticulum Protocol is defined entirely and authoritatively by this reference implementation, and its associated manual. It is maintained by Mark Qvist, identified by the Reticulum Identity `B333<bc7291552be7a58f361522990465165c>`b. The Python code in this repository is the Reference Implementation of Reticulum. The Reticulum Protocol is defined entirely and authoritatively by this reference implementation, and its associated manual. It is maintained by Mark Qvist, identified by the Reticulum Identity `B333<bc7291552be7a58f361522990465165c>`b.
No second-order protocol specification derivatives currently do, or will ever exist. You can't get a better specification than one that is executable, functional, testable, deployed and operating 24/7 across the planet.
There is not, and never will be a separate, derived "formal spec", "RFC" or anything similar for Reticulum:
`!The reference implementation `*is`* the final, authoritative specification.`!
Compatibility with the Reticulum Protocol is defined as having full interoperability, and sufficient functional parity with this reference implementation. Any specific protocol implementation that achieves this is Reticulum. Any that does not is not Reticulum. Compatibility with the Reticulum Protocol is defined as having full interoperability, and sufficient functional parity with this reference implementation. Any specific protocol implementation that achieves this is Reticulum. Any that does not is not Reticulum.
The reference implementation is licensed under the Reticulum License. The reference implementation is licensed under the Reticulum License.
@ -172,19 +164,9 @@ Currently, the following built-in interfaces are supported:
>> Performance >> Performance
`*All performance numbers here assume `B333rnsd`b running in fully interpreted Python mode, limited to a single CPU core.`* Reticulum targets a `*very`* wide usable performance envelope, but prioritises functionality and performance on low-bandwidth mediums. The goal is to provide a dynamic performance envelope from 250 bits per second, to 1 gigabit per second on normal hardware.
Reticulum targets a `*very`* wide usable performance envelope, but prioritises functionality and performance on low-bandwidth mediums. The goal is to provide a dynamic performance envelope from 100 bits per second, to 10 gigabits per second on normal hardware. Currently, the usable performance envelope is approximately 150 bits per second to 500 megabits per second, with physical mediums faster than that not being saturated. Performance beyond the current level is intended for future upgrades, but not highly prioritised at this point in time.
Currently, the on-network, usable performance envelope is approximately 150 bits per second to 1 gigabit per second on modest hardware. Real-world performance will vary widely with the speed of your system, availability of cryptographic hardware acceleration, and other factors.
Raw routing throughput for a transport node hosted on a Raspberry Pi 5 is approximately 4 gigabits per second. Systems with modern crypto hardware acceleration can achieve transport throughput of 15 gigabits per second and higher.
While the reference implementation is perfectly capable of reaching 100+ Gbps, by utilizing multiple CPU cores and vectorized transport, performance beyond the current level is not higly prioritised, since RNS is already able to saturate available physical medium bandwidth in almost all real-world situations.
The primary objectives for RNS are `*reliability`*, `*correctness`*, `*security`*, `*proper traffic management`* and `*actually practical performance`*, not hollow on-paper "benchmarks" with little to no bearing on real-world usability.
All alternative-implementation maintainers are `*highly`* encouraged to adopt a similar approach, and especially to focus on getting their implementations working properly before publishing empirically dubious or factually wrong performance claims.
>> Current Status >> Current Status
@ -268,7 +250,7 @@ Please note that by default, installing Reticulum will `!require`! OpenSSL and P
If you want to use the internal pure-python primitives, it is `!highly advisable`! that you have a good understanding of the risks that this pose, and make an informed decision on whether those risks are acceptable to you. If you want to use the internal pure-python primitives, it is `!highly advisable`! that you have a good understanding of the risks that this pose, and make an informed decision on whether those risks are acceptable to you.
Having been in development for ten, and wider deployment for around five years, Reticulum is relatively young software, and should be considered as such. While it has been carefully built with cryptography best-practices very foremost in mind, there could very well still be privacy or security breaking bugs. Reticulum is relatively young software, and should be considered as such. While it has been built with cryptography best-practices very foremost in mind, it _has not_ been externally security audited, and there could very well be privacy or security breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch.
>> Acknowledgements & Credits >> Acknowledgements & Credits

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -53,8 +53,7 @@ class StreamDataMessage(MessageBase):
The stream id is limited to 2 bytes - 2 bit The stream id is limited to 2 bytes - 2 bit
""" """
HEADER_LEN = 2 # 2 for stream data message header OVERHEAD = 2 + 6 # 2 for stream data message header, 6 for channel envelope
OVERHEAD = HEADER_LEN + 6 # 6 for channel envelope
MAX_DATA_LEN = RNS.Link.MDU - OVERHEAD MAX_DATA_LEN = RNS.Link.MDU - OVERHEAD
""" """
When the Buffer package is imported, this value is When the Buffer package is imported, this value is
@ -227,7 +226,7 @@ class RawChannelWriter(RawIOBase, AbstractContextManager):
self._stream_id = stream_id self._stream_id = stream_id
self._channel = channel self._channel = channel
self._eof = False self._eof = False
self._mdu = channel.mdu - StreamDataMessage.HEADER_LEN self._mdu = channel.mdu - StreamDataMessage.OVERHEAD
def write(self, __b: bytes) -> int | None: def write(self, __b: bytes) -> int | None:
try: try:
@ -243,7 +242,7 @@ class RawChannelWriter(RawIOBase, AbstractContextManager):
chunk_segment_length = int(chunk_len/comp_try) chunk_segment_length = int(chunk_len/comp_try)
compressed_chunk = bz2.compress(__b[:chunk_segment_length]) compressed_chunk = bz2.compress(__b[:chunk_segment_length])
compressed_length = len(compressed_chunk) compressed_length = len(compressed_chunk)
if compressed_length < self._mdu and compressed_length < chunk_segment_length: if compressed_length < StreamDataMessage.MAX_DATA_LEN and compressed_length < chunk_segment_length:
comp_success = True comp_success = True
break break
else: else:
@ -253,7 +252,7 @@ class RawChannelWriter(RawIOBase, AbstractContextManager):
chunk = compressed_chunk chunk = compressed_chunk
processed_length = chunk_segment_length processed_length = chunk_segment_length
else: else:
chunk = bytes(__b[:self._mdu]) chunk = bytes(__b[:StreamDataMessage.MAX_DATA_LEN])
processed_length = len(chunk) processed_length = len(chunk)
message = StreamDataMessage(self._stream_id, chunk, self._eof, comp_success) message = StreamDataMessage(self._stream_id, chunk, self._eof, comp_success)

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -45,6 +45,63 @@ TPacket = TypeVar("TPacket")
class SystemMessageTypes(enum.IntEnum): class SystemMessageTypes(enum.IntEnum):
SMT_STREAM_DATA = 0xff00 SMT_STREAM_DATA = 0xff00
class ChannelOutletBase(ABC, Generic[TPacket]):
"""
An abstract transport layer interface used by Channel.
DEPRECATED: This was created for testing; eventually
Channel will use Link or a LinkBase interface
directly.
"""
@abstractmethod
def send(self, raw: bytes) -> TPacket:
raise NotImplemented()
@abstractmethod
def resend(self, packet: TPacket) -> TPacket:
raise NotImplemented()
@property
@abstractmethod
def mdu(self):
raise NotImplemented()
@property
@abstractmethod
def rtt(self):
raise NotImplemented()
@property
@abstractmethod
def is_usable(self):
raise NotImplemented()
@abstractmethod
def get_packet_state(self, packet: TPacket) -> MessageState:
raise NotImplemented()
@abstractmethod
def timed_out(self):
raise NotImplemented()
@abstractmethod
def __str__(self):
raise NotImplemented()
@abstractmethod
def set_packet_timeout_callback(self, packet: TPacket, callback: Callable[[TPacket], None] | None,
timeout: float | None = None):
raise NotImplemented()
@abstractmethod
def set_packet_delivered_callback(self, packet: TPacket, callback: Callable[[TPacket], None] | None):
raise NotImplemented()
@abstractmethod
def get_packet_id(self, packet: TPacket) -> any:
raise NotImplemented()
class CEType(enum.IntEnum): class CEType(enum.IntEnum):
""" """
ChannelException type codes ChannelException type codes
@ -56,6 +113,7 @@ class CEType(enum.IntEnum):
ME_ALREADY_SENT = 4 ME_ALREADY_SENT = 4
ME_TOO_BIG = 5 ME_TOO_BIG = 5
class ChannelException(Exception): class ChannelException(Exception):
""" """
An exception thrown by Channel, with a type code. An exception thrown by Channel, with a type code.
@ -64,6 +122,7 @@ class ChannelException(Exception):
super().__init__(args) super().__init__(args)
self.type = ce_type self.type = ce_type
class MessageState(enum.IntEnum): class MessageState(enum.IntEnum):
""" """
Set of possible states for a Message Set of possible states for a Message
@ -73,6 +132,7 @@ class MessageState(enum.IntEnum):
MSGSTATE_DELIVERED = 2 MSGSTATE_DELIVERED = 2
MSGSTATE_FAILED = 3 MSGSTATE_FAILED = 3
class MessageBase(abc.ABC): class MessageBase(abc.ABC):
""" """
Base type for any messages sent or received on a Channel. Base type for any messages sent or received on a Channel.
@ -110,6 +170,7 @@ class MessageBase(abc.ABC):
MessageCallbackType = NewType("MessageCallbackType", Callable[[MessageBase], bool]) MessageCallbackType = NewType("MessageCallbackType", Callable[[MessageBase], bool])
class Envelope: class Envelope:
""" """
Internal wrapper used to transport messages over a channel and Internal wrapper used to transport messages over a channel and
@ -119,7 +180,8 @@ class Envelope:
msgtype, self.sequence, length = struct.unpack(">HHH", self.raw[:6]) msgtype, self.sequence, length = struct.unpack(">HHH", self.raw[:6])
raw = self.raw[6:] raw = self.raw[6:]
ctor = message_factories.get(msgtype, None) ctor = message_factories.get(msgtype, None)
if ctor is None: raise ChannelException(CEType.ME_NOT_REGISTERED, f"Unable to find constructor for Channel MSGTYPE {hex(msgtype)}") if ctor is None:
raise ChannelException(CEType.ME_NOT_REGISTERED, f"Unable to find constructor for Channel MSGTYPE {hex(msgtype)}")
message = ctor() message = ctor()
message.unpack(raw) message.unpack(raw)
self.unpacked = True self.unpacked = True
@ -128,13 +190,14 @@ class Envelope:
return message return message
def pack(self) -> bytes: def pack(self) -> bytes:
if self.message.__class__.MSGTYPE is None: raise ChannelException(CEType.ME_NO_MSG_TYPE, f"{self.message.__class__} lacks MSGTYPE") if self.message.__class__.MSGTYPE is None:
raise ChannelException(CEType.ME_NO_MSG_TYPE, f"{self.message.__class__} lacks MSGTYPE")
data = self.message.pack() data = self.message.pack()
self.raw = struct.pack(">HHH", self.message.MSGTYPE, self.sequence, len(data)) + data self.raw = struct.pack(">HHH", self.message.MSGTYPE, self.sequence, len(data)) + data
self.packed = True self.packed = True
return self.raw return self.raw
def __init__(self, outlet: LinkChannelOutlet, message: MessageBase = None, raw: bytes = None, sequence: int = None): def __init__(self, outlet: ChannelOutletBase, message: MessageBase = None, raw: bytes = None, sequence: int = None):
self.ts = time.time() self.ts = time.time()
self.id = id(self) self.id = id(self)
self.message = message self.message = message
@ -215,14 +278,13 @@ class Channel(contextlib.AbstractContextManager):
SEQ_MAX = 0xFFFF SEQ_MAX = 0xFFFF
SEQ_MODULUS = SEQ_MAX+1 SEQ_MODULUS = SEQ_MAX+1
def __init__(self, outlet: LinkChannelOutlet): def __init__(self, outlet: ChannelOutletBase):
""" """
@param outlet: @param outlet:
""" """
self._outlet = outlet self._outlet = outlet
self._lock = threading.RLock() self._lock = threading.RLock()
self._send_lock = threading.Lock()
self._tx_ring: collections.deque[Envelope] = collections.deque() self._tx_ring: collections.deque[Envelope] = collections.deque()
self._rx_ring: collections.deque[Envelope] = collections.deque() self._rx_ring: collections.deque[Envelope] = collections.deque()
self._message_callbacks: [MessageCallbackType] = [] self._message_callbacks: [MessageCallbackType] = []
@ -244,9 +306,11 @@ class Channel(contextlib.AbstractContextManager):
self.window_min = Channel.WINDOW_MIN self.window_min = Channel.WINDOW_MIN
self.window_flexibility = Channel.WINDOW_FLEXIBILITY self.window_flexibility = Channel.WINDOW_FLEXIBILITY
def __enter__(self) -> Channel: return self def __enter__(self) -> Channel:
return self
def __exit__(self, __exc_type: Type[BaseException] | None, __exc_value: BaseException | None, __traceback: TracebackType | None) -> bool | None: def __exit__(self, __exc_type: Type[BaseException] | None, __exc_value: BaseException | None,
__traceback: TracebackType | None) -> bool | None:
self._shutdown() self._shutdown()
return False return False
@ -262,13 +326,20 @@ class Channel(contextlib.AbstractContextManager):
def _register_message_type(self, message_class: Type[MessageBase], *, is_system_type: bool = False): def _register_message_type(self, message_class: Type[MessageBase], *, is_system_type: bool = False):
with self._lock: with self._lock:
if not issubclass(message_class, MessageBase): raise ChannelException(CEType.ME_INVALID_MSG_TYPE, f"{message_class} is not a subclass of {MessageBase}.") if not issubclass(message_class, MessageBase):
if message_class.MSGTYPE is None: raise ChannelException(CEType.ME_INVALID_MSG_TYPE, f"{message_class} has invalid MSGTYPE class attribute.") raise ChannelException(CEType.ME_INVALID_MSG_TYPE,
if message_class.MSGTYPE >= 0xf000 and not is_system_type: raise ChannelException(CEType.ME_INVALID_MSG_TYPE, f"{message_class} has system-reserved message type.") f"{message_class} is not a subclass of {MessageBase}.")
if message_class.MSGTYPE is None:
try: message_class() raise ChannelException(CEType.ME_INVALID_MSG_TYPE,
f"{message_class} has invalid MSGTYPE class attribute.")
if message_class.MSGTYPE >= 0xf000 and not is_system_type:
raise ChannelException(CEType.ME_INVALID_MSG_TYPE,
f"{message_class} has system-reserved message type.")
try:
message_class()
except Exception as ex: except Exception as ex:
raise ChannelException(CEType.ME_INVALID_MSG_TYPE, f"{message_class} raised an exception when constructed with no arguments: {ex}") raise ChannelException(CEType.ME_INVALID_MSG_TYPE,
f"{message_class} raised an exception when constructed with no arguments: {ex}")
self._message_factories[message_class.MSGTYPE] = message_class self._message_factories[message_class.MSGTYPE] = message_class
@ -311,11 +382,6 @@ class Channel(contextlib.AbstractContextManager):
if envelope.packet is not None: if envelope.packet is not None:
self._outlet.set_packet_timeout_callback(envelope.packet, None) self._outlet.set_packet_timeout_callback(envelope.packet, None)
self._outlet.set_packet_delivered_callback(envelope.packet, None) self._outlet.set_packet_delivered_callback(envelope.packet, None)
envelope.tracked = False
for envelope in self._rx_ring:
envelope.tracked = False
self._tx_ring.clear() self._tx_ring.clear()
self._rx_ring.clear() self._rx_ring.clear()
@ -324,12 +390,14 @@ class Channel(contextlib.AbstractContextManager):
i = 0 i = 0
for existing in ring: for existing in ring:
if envelope.sequence == existing.sequence: if envelope.sequence == existing.sequence:
RNS.log(f"Envelope: Emplacement of duplicate envelope with sequence "+str(envelope.sequence), RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None RNS.log(f"Envelope: Emplacement of duplicate envelope with sequence "+str(envelope.sequence), RNS.LOG_EXTREME)
return False return False
if envelope.sequence < existing.sequence and not (self._next_rx_sequence - envelope.sequence) > (Channel.SEQ_MAX//2): if envelope.sequence < existing.sequence and not (self._next_rx_sequence - envelope.sequence) > (Channel.SEQ_MAX//2):
ring.insert(i, envelope) ring.insert(i, envelope)
envelope.tracked = True envelope.tracked = True
return True return True
@ -345,8 +413,10 @@ class Channel(contextlib.AbstractContextManager):
for cb in cbs: for cb in cbs:
try: try:
if cb(message): return if cb(message):
except Exception as e: RNS.log(f"Channel {str(self)} experienced an error while running a message callback. The contained exception was: {e}", RNS.LOG_ERROR) return
except Exception as e:
RNS.log("Channel "+str(self)+" experienced an error while running a message callback. The contained exception was: "+str(e), RNS.LOG_ERROR)
def _receive(self, raw: bytes): def _receive(self, raw: bytes):
try: try:
@ -358,20 +428,16 @@ class Channel(contextlib.AbstractContextManager):
window_overflow = (self._next_rx_sequence+Channel.WINDOW_MAX) % Channel.SEQ_MODULUS window_overflow = (self._next_rx_sequence+Channel.WINDOW_MAX) % Channel.SEQ_MODULUS
if window_overflow < self._next_rx_sequence: if window_overflow < self._next_rx_sequence:
if envelope.sequence > window_overflow: if envelope.sequence > window_overflow:
RNS.log(f"Invalid packet sequence ({envelope.sequence}) received on channel {self}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log("Invalid packet sequence ("+str(envelope.sequence)+") received on channel "+str(self), RNS.LOG_EXTREME)
return return
else: else:
RNS.log(f"Invalid packet sequence ({envelope.sequence}) received on channel {self}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log("Invalid packet sequence ("+str(envelope.sequence)+") received on channel "+str(self), RNS.LOG_EXTREME)
return
elif envelope.sequence > self._next_rx_sequence + self.WINDOW_MAX:
RNS.log(f"Invalid packet sequence ({envelope.sequence}) received on channel {self}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return return
is_new = self._emplace_envelope(envelope, self._rx_ring) is_new = self._emplace_envelope(envelope, self._rx_ring)
if not is_new: if not is_new:
RNS.log("Duplicate message received on channel "+str(self), RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None RNS.log("Duplicate message received on channel "+str(self), RNS.LOG_EXTREME)
return return
else: else:
with self._lock: with self._lock:
@ -387,8 +453,10 @@ class Channel(contextlib.AbstractContextManager):
self._next_rx_sequence = (self._next_rx_sequence + 1) % Channel.SEQ_MODULUS self._next_rx_sequence = (self._next_rx_sequence + 1) % Channel.SEQ_MODULUS
for e in contigous: for e in contigous:
if not e.unpacked: m = e.unpack(self._message_factories) if not e.unpacked:
else: m = e.message m = e.unpack(self._message_factories)
else:
m = e.message
self._rx_ring.remove(e) self._rx_ring.remove(e)
self._run_callbacks(m) self._run_callbacks(m)
@ -402,7 +470,9 @@ class Channel(contextlib.AbstractContextManager):
:return: True if ready :return: True if ready
""" """
if not self._outlet.is_usable: return False if not self._outlet.is_usable:
return False
with self._lock: with self._lock:
outstanding = 0 outstanding = 0
for envelope in self._tx_ring: for envelope in self._tx_ring:
@ -410,38 +480,57 @@ class Channel(contextlib.AbstractContextManager):
if not envelope.packet or not self._outlet.get_packet_state(envelope.packet) == MessageState.MSGSTATE_DELIVERED: if not envelope.packet or not self._outlet.get_packet_state(envelope.packet) == MessageState.MSGSTATE_DELIVERED:
outstanding += 1 outstanding += 1
if outstanding >= self.window: return False if outstanding >= self.window:
return False
return True return True
def _packet_tx_op(self, packet: TPacket, op: Callable[[TPacket], bool]): def _packet_tx_op(self, packet: TPacket, op: Callable[[TPacket], bool]):
target_id = self._outlet.get_packet_id(packet)
with self._lock: with self._lock:
envelope = next(filter(lambda e: e.packet is not None and self._outlet.get_packet_id(e.packet) == target_id, self._tx_ring), None) envelope = next(filter(lambda e: self._outlet.get_packet_id(e.packet) == self._outlet.get_packet_id(packet),
self._tx_ring), None)
if envelope and op(envelope): if envelope and op(envelope):
envelope.tracked = False envelope.tracked = False
if envelope in self._tx_ring: if envelope in self._tx_ring:
self._tx_ring.remove(envelope) self._tx_ring.remove(envelope)
if self.window < self.window_max: self.window += 1 if self.window < self.window_max:
self.window += 1
# TODO: Remove at some point
# RNS.log("Increased "+str(self)+" window to "+str(self.window), RNS.LOG_DEBUG)
if self._outlet.rtt != 0: if self._outlet.rtt != 0:
if self._outlet.rtt > Channel.RTT_FAST: if self._outlet.rtt > Channel.RTT_FAST:
self.fast_rate_rounds = 0 self.fast_rate_rounds = 0
if self._outlet.rtt > Channel.RTT_MEDIUM: self.medium_rate_rounds = 0
if self._outlet.rtt > Channel.RTT_MEDIUM:
self.medium_rate_rounds = 0
else: else:
self.medium_rate_rounds += 1 self.medium_rate_rounds += 1
if self.window_max < Channel.WINDOW_MAX_MEDIUM and self.medium_rate_rounds == Channel.FAST_RATE_THRESHOLD: if self.window_max < Channel.WINDOW_MAX_MEDIUM and self.medium_rate_rounds == Channel.FAST_RATE_THRESHOLD:
self.window_max = Channel.WINDOW_MAX_MEDIUM self.window_max = Channel.WINDOW_MAX_MEDIUM
self.window_min = Channel.WINDOW_MIN_LIMIT_MEDIUM self.window_min = Channel.WINDOW_MIN_LIMIT_MEDIUM
# TODO: Remove at some point
# RNS.log("Increased "+str(self)+" max window to "+str(self.window_max), RNS.LOG_DEBUG)
# RNS.log("Increased "+str(self)+" min window to "+str(self.window_min), RNS.LOG_DEBUG)
else: else:
self.fast_rate_rounds += 1 self.fast_rate_rounds += 1
if self.window_max < Channel.WINDOW_MAX_FAST and self.fast_rate_rounds == Channel.FAST_RATE_THRESHOLD: if self.window_max < Channel.WINDOW_MAX_FAST and self.fast_rate_rounds == Channel.FAST_RATE_THRESHOLD:
self.window_max = Channel.WINDOW_MAX_FAST self.window_max = Channel.WINDOW_MAX_FAST
self.window_min = Channel.WINDOW_MIN_LIMIT_FAST self.window_min = Channel.WINDOW_MIN_LIMIT_FAST
# TODO: Remove at some point
# RNS.log("Increased "+str(self)+" max window to "+str(self.window_max), RNS.LOG_DEBUG)
# RNS.log("Increased "+str(self)+" min window to "+str(self.window_min), RNS.LOG_DEBUG)
else: RNS.log("Envelope not found in TX ring for "+str(self), RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
if not envelope: RNS.log("Spurious message received on "+str(self), RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None else:
RNS.log("Envelope not found in TX ring for "+str(self), RNS.LOG_EXTREME)
if not envelope:
RNS.log("Spurious message received on "+str(self), RNS.LOG_EXTREME)
def _packet_delivered(self, packet: TPacket): def _packet_delivered(self, packet: TPacket):
self._packet_tx_op(packet, lambda env: True) self._packet_tx_op(packet, lambda env: True)
@ -450,47 +539,44 @@ class Channel(contextlib.AbstractContextManager):
for envelope in self._tx_ring: for envelope in self._tx_ring:
updated_timeout = self._get_packet_timeout_time(envelope.tries) updated_timeout = self._get_packet_timeout_time(envelope.tries)
if envelope.packet and hasattr(envelope.packet, "receipt") and envelope.packet.receipt and envelope.packet.receipt.timeout: if envelope.packet and hasattr(envelope.packet, "receipt") and envelope.packet.receipt and envelope.packet.receipt.timeout:
if updated_timeout > envelope.packet.receipt.timeout: envelope.packet.receipt.set_timeout(updated_timeout) if updated_timeout > envelope.packet.receipt.timeout:
envelope.packet.receipt.set_timeout(updated_timeout)
def _get_packet_timeout_time(self, tries: int) -> float: def _get_packet_timeout_time(self, tries: int) -> float:
to = pow(1.5, tries - 1) * max(self._outlet.rtt*2.5, 0.025) * (len(self._tx_ring)+1.5) to = pow(1.5, tries - 1) * max(self._outlet.rtt*2.5, 0.025) * (len(self._tx_ring)+1.5)
return to return to
def _packet_timeout(self, packet: TPacket): def _packet_timeout(self, packet: TPacket):
if self._outlet.get_packet_state(packet) == MessageState.MSGSTATE_DELIVERED: return def retry_envelope(envelope: Envelope) -> bool:
if envelope.tries >= self._max_tries:
RNS.log("Retry count exceeded on "+str(self)+", tearing down Link.", RNS.LOG_ERROR)
self._shutdown() # start on separate thread?
self._outlet.timed_out()
return True
target_id = self._outlet.get_packet_id(packet)
envelope_to_resend: Envelope | None = None
should_teardown = False
with self._lock:
envelope = next(filter(lambda e: e.packet is not None and self._outlet.get_packet_id(e.packet) == target_id, self._tx_ring), None)
if envelope is None: return
if envelope.tries >= self._max_tries: should_teardown = True
else:
envelope.tries += 1 envelope.tries += 1
envelope_to_resend = envelope self._outlet.resend(envelope.packet)
self._outlet.set_packet_delivered_callback(envelope.packet, self._packet_delivered)
self._outlet.set_packet_timeout_callback(envelope.packet, self._packet_timeout, self._get_packet_timeout_time(envelope.tries))
self._update_packet_timeouts()
if self.window > self.window_min: if self.window > self.window_min:
self.window -= 1 self.window -= 1
# TODO: Remove at some point
# RNS.log("Decreased "+str(self)+" window to "+str(self.window), RNS.LOG_DEBUG)
if self.window_max > (self.window_min+self.window_flexibility): if self.window_max > (self.window_min+self.window_flexibility):
self.window_max -= 1 self.window_max -= 1
# TODO: Remove at some point
# RNS.log("Decreased "+str(self)+" max window to "+str(self.window_max), RNS.LOG_DEBUG)
if should_teardown: # TODO: Remove at some point
RNS.log("Retry count exceeded on "+str(self)+", tearing down Link.", RNS.LOG_ERROR) # RNS.log("Decreased "+str(self)+" window to "+str(self.window), RNS.LOG_EXTREME)
self._shutdown()
self._outlet.timed_out()
return
if envelope_to_resend is not None: return False
self._outlet.resend(envelope_to_resend.packet)
with self._lock:
self._outlet.set_packet_delivered_callback(envelope_to_resend.packet, self._packet_delivered)
self._outlet.set_packet_timeout_callback(envelope_to_resend.packet, self._packet_timeout, self._get_packet_timeout_time(envelope_to_resend.tries))
self._update_packet_timeouts()
already_delivered = (self._outlet.get_packet_state(envelope_to_resend.packet) == MessageState.MSGSTATE_DELIVERED)
if already_delivered: self._packet_delivered(envelope_to_resend.packet) if self._outlet.get_packet_state(packet) != MessageState.MSGSTATE_DELIVERED:
self._packet_tx_op(packet, retry_envelope)
def send(self, message: MessageBase) -> Envelope: def send(self, message: MessageBase) -> Envelope:
""" """
@ -499,33 +585,27 @@ class Channel(contextlib.AbstractContextManager):
:param message: an instance of a ``MessageBase`` subclass :param message: an instance of a ``MessageBase`` subclass
""" """
with self._send_lock: envelope: Envelope | None = None
with self._lock: with self._lock:
if not self.is_ready_to_send(): raise ChannelException(CEType.ME_LINK_NOT_READY, f"Link is not ready") if not self.is_ready_to_send():
raise ChannelException(CEType.ME_LINK_NOT_READY, f"Link is not ready")
envelope = Envelope(self._outlet, message=message, sequence=self._next_sequence)
self._next_sequence = (self._next_sequence + 1) % Channel.SEQ_MODULUS
self._emplace_envelope(envelope, self._tx_ring)
if envelope is None:
raise BlockingIOError()
reserved_sequence = self._next_sequence
envelope = Envelope(self._outlet, message=message, sequence=reserved_sequence)
envelope.pack() envelope.pack()
if len(envelope.raw) > self._outlet.mdu: if len(envelope.raw) > self._outlet.mdu:
raise ChannelException(CEType.ME_TOO_BIG, f"Packed message too big for packet: {len(envelope.raw)} > {self._outlet.mdu}") raise ChannelException(CEType.ME_TOO_BIG, f"Packed message too big for packet: {len(envelope.raw)} > {self._outlet.mdu}")
self._next_sequence = (reserved_sequence + 1) % Channel.SEQ_MODULUS
envelope.packet = self._outlet.send(envelope.raw) envelope.packet = self._outlet.send(envelope.raw)
if envelope.packet is None or getattr(envelope.packet, "raw", None) is None or (hasattr(envelope.packet, "receipt") and envelope.packet.receipt is None):
with self._lock: self._next_sequence = reserved_sequence
raise ChannelException(CEType.ME_LINK_NOT_READY, "Outlet did not transmit packet")
with self._lock:
self._emplace_envelope(envelope, self._tx_ring)
envelope.tries += 1 envelope.tries += 1
self._outlet.set_packet_delivered_callback(envelope.packet, self._packet_delivered) self._outlet.set_packet_delivered_callback(envelope.packet, self._packet_delivered)
self._outlet.set_packet_timeout_callback(envelope.packet, self._packet_timeout, self._get_packet_timeout_time(envelope.tries)) self._outlet.set_packet_timeout_callback(envelope.packet, self._packet_timeout, self._get_packet_timeout_time(envelope.tries))
self._update_packet_timeouts() self._update_packet_timeouts()
already_delivered = (self._outlet.get_packet_state(envelope.packet) == MessageState.MSGSTATE_DELIVERED)
# Prevent _tx_ring envelope leak
if already_delivered: self._packet_delivered(envelope.packet)
return envelope return envelope
@ -540,13 +620,14 @@ class Channel(contextlib.AbstractContextManager):
:return: number of bytes available :return: number of bytes available
""" """
mdu = self._outlet.mdu - 6 # sizeof(msgtype) + sizeof(length) + sizeof(sequence) mdu = self._outlet.mdu - 6 # sizeof(msgtype) + sizeof(length) + sizeof(sequence)
if mdu > 0xFFFF: mdu = 0xFFFF if mdu > 0xFFFF:
mdu = 0xFFFF
return mdu return mdu
class LinkChannelOutlet(ABC, Generic[TPacket]): class LinkChannelOutlet(ChannelOutletBase):
""" """
A channel outlet implementation for RNS.Link. An implementation of ChannelOutletBase for RNS.Link.
Allows Channel to send packets over an RNS Link with Allows Channel to send packets over an RNS Link with
Packets. Packets.
@ -557,51 +638,68 @@ class LinkChannelOutlet(ABC, Generic[TPacket]):
def send(self, raw: bytes) -> RNS.Packet: def send(self, raw: bytes) -> RNS.Packet:
packet = RNS.Packet(self.link, raw, context=RNS.Packet.CHANNEL) packet = RNS.Packet(self.link, raw, context=RNS.Packet.CHANNEL)
if self.link.status == RNS.Link.ACTIVE: packet.send() if self.link.status == RNS.Link.ACTIVE:
packet.send()
return packet return packet
def resend(self, packet: RNS.Packet) -> RNS.Packet: def resend(self, packet: RNS.Packet) -> RNS.Packet:
receipt = packet.resend() receipt = packet.resend()
if not receipt: RNS.log("Failed to resend packet", RNS.LOG_ERROR) if not receipt:
RNS.log("Failed to resend packet", RNS.LOG_ERROR)
return packet return packet
@property @property
def mdu(self): return self.link.mdu def mdu(self):
return self.link.mdu
@property @property
def rtt(self): return self.link.rtt def rtt(self):
return self.link.rtt
# TODO: Initial implementation had issues
# proxying this to link status. Should
# return False when link is not ACTIVE.
# Investigate what happened here.
@property @property
def is_usable(self): return True def is_usable(self):
return True # had issues looking at Link.status
def get_packet_state(self, packet: TPacket) -> MessageState: def get_packet_state(self, packet: TPacket) -> MessageState:
if packet.receipt == None: if packet.receipt == None:
return MessageState.MSGSTATE_FAILED return MessageState.MSGSTATE_FAILED
status = packet.receipt.get_status() status = packet.receipt.get_status()
if status == RNS.PacketReceipt.SENT: return MessageState.MSGSTATE_SENT if status == RNS.PacketReceipt.SENT:
if status == RNS.PacketReceipt.DELIVERED: return MessageState.MSGSTATE_DELIVERED return MessageState.MSGSTATE_SENT
if status == RNS.PacketReceipt.FAILED: return MessageState.MSGSTATE_FAILED if status == RNS.PacketReceipt.DELIVERED:
else: raise Exception(f"Unexpected receipt state: {status}") return MessageState.MSGSTATE_DELIVERED
if status == RNS.PacketReceipt.FAILED:
return MessageState.MSGSTATE_FAILED
else:
raise Exception(f"Unexpected receipt state: {status}")
def set_packet_timeout_callback(self, packet: RNS.Packet, callback: Callable[[RNS.Packet], None] | None, timeout: float | None = None): def timed_out(self):
if timeout and packet.receipt: packet.receipt.set_timeout(timeout) self.link.teardown()
def inner(receipt: RNS.PacketReceipt): callback(packet)
if packet and packet.receipt: packet.receipt.set_timeout_callback(inner if callback else None) def __str__(self):
return f"{self.__class__.__name__}({self.link})"
def set_packet_timeout_callback(self, packet: RNS.Packet, callback: Callable[[RNS.Packet], None] | None,
timeout: float | None = None):
if timeout and packet.receipt:
packet.receipt.set_timeout(timeout)
def inner(receipt: RNS.PacketReceipt):
callback(packet)
if packet and packet.receipt:
packet.receipt.set_timeout_callback(inner if callback else None)
def set_packet_delivered_callback(self, packet: RNS.Packet, callback: Callable[[RNS.Packet], None] | None): def set_packet_delivered_callback(self, packet: RNS.Packet, callback: Callable[[RNS.Packet], None] | None):
def inner(receipt: RNS.PacketReceipt): callback(packet) def inner(receipt: RNS.PacketReceipt):
if packet and packet.receipt: packet.receipt.set_delivery_callback(inner if callback else None) callback(packet)
if packet and packet.receipt:
packet.receipt.set_delivery_callback(inner if callback else None)
def get_packet_id(self, packet: RNS.Packet) -> any: def get_packet_id(self, packet: RNS.Packet) -> any:
if packet and getattr(packet, "raw", None) is not None and hasattr(packet, "get_hash") and callable(packet.get_hash): if packet and hasattr(packet, "get_hash") and callable(packet.get_hash):
return packet.get_hash() return packet.get_hash()
else: return None else:
return None
def timed_out(self): self.link.teardown()
def __str__(self): return f"{self.__class__.__name__}({self.link})"

View file

@ -43,7 +43,7 @@ elif cp.PROVIDER == cp.PROVIDER_PYCA:
class AES_128_CBC: class AES_128_CBC:
@staticmethod @staticmethod
def encrypt(plaintext, key, iv): def encrypt(plaintext, key, iv):
if len(key) != 16: raise ValueError(f"Invalid key length {len(key)*8} for AES_128_CBC") if len(key) != 16: raise ValueError(f"Invalid key length {len(key)*8} for {self}")
if cp.PROVIDER == cp.PROVIDER_INTERNAL: if cp.PROVIDER == cp.PROVIDER_INTERNAL:
cipher = AES128(key) cipher = AES128(key)
return cipher.encrypt(plaintext, iv) return cipher.encrypt(plaintext, iv)
@ -60,7 +60,7 @@ class AES_128_CBC:
@staticmethod @staticmethod
def decrypt(ciphertext, key, iv): def decrypt(ciphertext, key, iv):
if len(key) != 16: raise ValueError(f"Invalid key length {len(key)*8} for AES_128_CBC") if len(key) != 16: raise ValueError(f"Invalid key length {len(key)*8} for {self}")
if cp.PROVIDER == cp.PROVIDER_INTERNAL: if cp.PROVIDER == cp.PROVIDER_INTERNAL:
cipher = AES128(key) cipher = AES128(key)
return cipher.decrypt(ciphertext, iv) return cipher.decrypt(ciphertext, iv)
@ -78,7 +78,7 @@ class AES_128_CBC:
class AES_256_CBC: class AES_256_CBC:
@staticmethod @staticmethod
def encrypt(plaintext, key, iv): def encrypt(plaintext, key, iv):
if len(key) != 32: raise ValueError(f"Invalid key length {len(key)*8} for AES_256_CBC") if len(key) != 32: raise ValueError(f"Invalid key length {len(key)*8} for {self}")
if cp.PROVIDER == cp.PROVIDER_INTERNAL: if cp.PROVIDER == cp.PROVIDER_INTERNAL:
cipher = AES256(key) cipher = AES256(key)
return cipher.encrypt_cbc(plaintext, iv) return cipher.encrypt_cbc(plaintext, iv)
@ -95,7 +95,7 @@ class AES_256_CBC:
@staticmethod @staticmethod
def decrypt(ciphertext, key, iv): def decrypt(ciphertext, key, iv):
if len(key) != 32: raise ValueError(f"Invalid key length {len(key)*8} for AES_256_CBC") if len(key) != 32: raise ValueError(f"Invalid key length {len(key)*8} for {self}")
if cp.PROVIDER == cp.PROVIDER_INTERNAL: if cp.PROVIDER == cp.PROVIDER_INTERNAL:
cipher = AES256(key) cipher = AES256(key)
return cipher.decrypt_cbc(ciphertext, iv) return cipher.decrypt_cbc(ciphertext, iv)

View file

@ -32,59 +32,8 @@ import hashlib
from math import ceil from math import ceil
from RNS.Cryptography import HMAC from RNS.Cryptography import HMAC
# Precomputed byte-permutation tables used to key the inner
# and outer hashing states of the HMAC construction
trans_5C = bytes((x ^ 0x5C) for x in range(256))
trans_36 = bytes((x ^ 0x36) for x in range(256))
# Precomputed HKDF counter bytes
_counters = [bytes([(i + 1) % 0x100]) for i in range(0x100)]
def hkdf(length=None, derive_from=None, salt=None, context=None): def hkdf(length=None, derive_from=None, salt=None, context=None):
hash_len = 32 hash_len = 32
hash_block_size = 64
if length == None or length < 1: raise ValueError("Invalid output key length")
if derive_from == None or derive_from == "": raise ValueError("Cannot derive key from empty input material")
if salt == None or len(salt) == 0: salt = bytes([0] * hash_len)
if context == None: context = b""
pseudorandom_key = HMAC.new(salt, derive_from).digest()
# The pseudorandom key is always a 32-byte SHA-256 digest, so the
# HMAC key schedule below never needs the long-key hashing branch.
key = pseudorandom_key.ljust(hash_block_size, b"\x00")
# Precompute the keyed inner and outer hashing states of the
# HMAC construction once. Each expansion block then only requires
# two cheap hash-object copies instead of constructing a complete
# HMAC object per 32 bytes of output. This is possible since the
# PRK is **always** 32-byte SHA-256 HMAC output.
inner_state = hashlib.sha256(key.translate(trans_36))
outer_state = hashlib.sha256(key.translate(trans_5C))
# Expand
has_context = len(context) > 0
block = b""
derived = bytearray()
for i in range(ceil(length / hash_len)):
inner = inner_state.copy()
inner.update(block)
if has_context: inner.update(context)
inner.update(_counters[i & 0xFF])
outer = outer_state.copy()
outer.update(inner.digest())
block = outer.digest()
derived += block
return bytes(derived[:length])
def hkdf_legacy(length=None, derive_from=None, salt=None, context=None):
hash_len = 32
def hmac_sha256(key, data): def hmac_sha256(key, data):
return HMAC.new(key, data).digest() return HMAC.new(key, data).digest()

View file

@ -181,8 +181,3 @@ def digest(key, msg, digest):
inner.update(msg) inner.update(msg)
outer.update(inner.digest()) outer.update(inner.digest())
return outer.digest() return outer.digest()
def _cd_internal(a, b): return a == b
try: from hmac import compare_digest as _cd
except ImportError: _cd = _cd_internal
compare_digest = _cd

View file

@ -65,6 +65,4 @@ def sha512(data):
def file_sha256(file): def file_sha256(file):
if not hashlib: raise SystemError("The hashlib module is not available on this system") if not hashlib: raise SystemError("The hashlib module is not available on this system")
# TODO: Could implement fallback for old snakes here
if not hasattr(hashlib, "file_digest"): raise SystemError("The file_digest method is not available on this system. This functionality requires Python 3.11 or later.")
else: return hashlib.file_digest(file, "sha256").digest() else: return hashlib.file_digest(file, "sha256").digest()

View file

@ -80,7 +80,7 @@ class Token():
received_hmac = token[-32:] received_hmac = token[-32:]
expected_hmac = HMAC.new(self._signing_key, token[:-32]).digest() expected_hmac = HMAC.new(self._signing_key, token[:-32]).digest()
if HMAC.compare_digest(received_hmac, expected_hmac): return True if received_hmac == expected_hmac: return True
else: return False else: return False

View file

@ -52,7 +52,5 @@ elif cp.PROVIDER == cp.PROVIDER_PYCA:
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py") py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc") pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
so_modules = glob.glob(os.path.dirname(__file__)+"/*.so") modules = py_modules+pyc_modules
pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd") __all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
modules = py_modules+pyc_modules+so_modules+pyd_modules
__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -35,8 +35,6 @@ import threading
import RNS import RNS
from RNS.Cryptography import Token from RNS.Cryptography import Token
from RNS.Stamps import generate_stamp, store_destination_stamp, load_destination_stamp
from RNS.Stamps import cost_to_bytes, quantize_cost
from .vendor import umsgpack as umsgpack from .vendor import umsgpack as umsgpack
class Callbacks: class Callbacks:
@ -148,14 +146,6 @@ class Destination:
return Destination.hash(identity, app_name, *aspects) return Destination.hash(identity, app_name, *aspects)
@staticmethod
def name_hash(full_name):
components = Destination.app_and_aspects_from_name(full_name)
app_name = str(components[0])
aspects = f".{'.'.join(components[1])}" if components[1] else ""
name = f"{app_name}{aspects}".encode("utf-8")
return RNS.Identity.full_hash(name)[:(RNS.Identity.NAME_HASH_LENGTH//8)]
def __init__(self, identity, direction, type, app_name, *aspects): def __init__(self, identity, direction, type, app_name, *aspects):
# Check input values and build name string # Check input values and build name string
if "." in app_name: raise ValueError("Dots can't be used in app names") if "." in app_name: raise ValueError("Dots can't be used in app names")
@ -165,15 +155,9 @@ class Destination:
self.accept_link_requests = True self.accept_link_requests = True
self.callbacks = Callbacks() self.callbacks = Callbacks()
self.request_handlers = {} self.request_handlers = {}
self.max_request_size = None
self.type = type self.type = type
self.direction = direction self.direction = direction
self.proof_strategy = Destination.PROVE_NONE self.proof_strategy = Destination.PROVE_NONE
self.stamp = None
self.stamp_value = None
self.stamp_generation_lock = threading.Lock()
self._target_value = RNS.Transport.DESTINATION_VALUE
self._link_cost = None
self.ratchets = None self.ratchets = None
self.ratchets_path = None self.ratchets_path = None
self.ratchet_interval = Destination.RATCHET_INTERVAL self.ratchet_interval = Destination.RATCHET_INTERVAL
@ -205,13 +189,6 @@ class Destination:
self.name_hash = RNS.Identity.full_hash(self.expand_name(None, app_name, *aspects).encode("utf-8"))[:(RNS.Identity.NAME_HASH_LENGTH//8)] self.name_hash = RNS.Identity.full_hash(self.expand_name(None, app_name, *aspects).encode("utf-8"))[:(RNS.Identity.NAME_HASH_LENGTH//8)]
self.hexhash = self.hash.hex() self.hexhash = self.hash.hex()
# Load existing stamp if available
if self.direction == Destination.IN and self.identity != None and self.identity.prv != None and RNS.Transport.STAMP_DESTINATIONS:
stamp_entry = load_destination_stamp(self.hash, self.stamp_material(), RNS.Transport.DESTINATION_COST)
if stamp_entry != None:
self.stamp, self.stamp_value = stamp_entry
RNS.log(f"Loaded stamp with value {RNS.prettycost(self.stamp_value)} for {self}", RNS.LOG_DEBUG)
self.default_app_data = None self.default_app_data = None
self.callback = None self.callback = None
self.proofcallback = None self.proofcallback = None
@ -228,7 +205,7 @@ class Destination:
def _clean_ratchets(self): def _clean_ratchets(self):
if self.ratchets != None: if self.ratchets != None:
if len (self.ratchets) > self.retained_ratchets: if len (self.ratchets) > self.retained_ratchets:
self.ratchets = self.ratchets[:self.retained_ratchets] self.ratchets = self.ratchets[:Destination.RATCHET_COUNT]
def _persist_ratchets(self): def _persist_ratchets(self):
try: try:
@ -263,26 +240,6 @@ class Destination:
return False return False
def _await_stamp(self):
with self.stamp_generation_lock:
if self.stamp == None or self.stamp_value == None or self.stamp_value < quantize_cost(self.target_value):
stamp_entry = load_destination_stamp(self.hash, self.stamp_material(), self.target_value)
if stamp_entry != None: self.stamp, self.stamp_value = stamp_entry
else:
stamp, stamp_value = generate_stamp(self.stamp_material(), stamp_cost=self.target_value,
expand_rounds=0, size=RNS.Identity.STAMPLENGTH)
if stamp == None:
RNS.log(f"Destination stamp generation for {RNS.prettyhexrep(self.hash)} failed or was aborted", RNS.LOG_DEBUG)
return False
self.stamp = stamp
self.stamp_value = stamp_value
store_destination_stamp(self.hash, stamp, stamp_value)
RNS.log(f"Generated and persisted stamp with value {self.stamp_value} for {self}", RNS.LOG_DEBUG)
if self.stamp == None: return False
else: return True
def announce(self, app_data=None, path_response=False, attached_interface=None, tag=None, send=True): def announce(self, app_data=None, path_response=False, attached_interface=None, tag=None, send=True):
""" """
Creates an announce packet for this destination and broadcasts it on all Creates an announce packet for this destination and broadcasts it on all
@ -291,20 +248,12 @@ class Destination:
:param app_data: *bytes* containing the app_data. :param app_data: *bytes* containing the app_data.
:param path_response: Internal flag used by :ref:`RNS.Transport<api-transport>`. Ignore. :param path_response: Internal flag used by :ref:`RNS.Transport<api-transport>`. Ignore.
""" """
if self.type != Destination.SINGLE: raise TypeError("Only SINGLE destination types can be announced") if self.type != Destination.SINGLE:
if self.direction != Destination.IN: raise TypeError("Only IN destination types can be announced") raise TypeError("Only SINGLE destination types can be announced")
if send and RNS.Transport.STAMP_DESTINATIONS and self.stamp == None: if self.direction != Destination.IN:
# If no stamp is available, queue for background generation raise TypeError("Only IN destination types can be announced")
# and announce emission once ready.
RNS.Transport.enqueue_stamped_announce(self, app_data=app_data, path_response=path_response,
attached_interface=attached_interface, tag=tag)
else:
# Otherwise, dispatch immediately.
return self._announce(app_data=app_data, path_response=path_response,
attached_interface=attached_interface, tag=tag, send=send)
def _announce(self, app_data=None, path_response=False, attached_interface=None, tag=None, send=True):
ratchet = b"" ratchet = b""
now = time.time() now = time.time()
stale_responses = [] stale_responses = []
@ -345,17 +294,6 @@ class Destination:
if isinstance(returned_app_data, bytes): if isinstance(returned_app_data, bytes):
app_data = returned_app_data app_data = returned_app_data
if RNS.Transport.STAMP_DESTINATIONS:
if not self._await_stamp():
RNS.log(f"Could not generate destination stamp for {RNS.prettyhexrep(self.hash)}, announce emission cancelled", RNS.LOG_WARNING)
return None
cost_bytes = cost_to_bytes(self.get_link_cost())
ext_payload = bytes([RNS.Identity.EXTPAYLOADMARKER, self.get_ext_flags()])+self.stamp+bytes([cost_bytes[1], cost_bytes[0]])
if app_data == None: app_data = ext_payload
else: app_data = app_data+ext_payload
signed_data = self.hash+self.identity.get_public_key()+self.name_hash+random_hash+ratchet signed_data = self.hash+self.identity.get_public_key()+self.name_hash+random_hash+ratchet
if app_data != None: signed_data += app_data if app_data != None: signed_data += app_data
@ -378,44 +316,6 @@ class Destination:
if send: announce_packet.send() if send: announce_packet.send()
else: return announce_packet else: return announce_packet
def stamp_material(self):
return self.hash+self.identity.get_public_key()+self.name_hash
def get_ext_flags(self):
return 0x00
@property
def target_value(self):
return min(RNS.Transport.STAMP_COST_CAP, max(float(self._target_value), RNS.Transport.DESTINATION_VALUE))
def set_value(self, value):
"""
Sets the target stamp value for this destination.
:param value: The target value as a *float*.
"""
self._target_value = min(RNS.Transport.STAMP_COST_CAP, max(float(value), RNS.Transport.DESTINATION_VALUE))
def set_link_cost(self, cost):
"""
Sets the required link establishment stamp cost for this destination. Set to 0 to disable.
:param cost: The required cost as a *float*.
"""
self._link_cost = max(0, min(float(cost), RNS.Transport.LINK_COST_CAP))
def get_link_cost(self):
"""
Returns the link request stamp cost to be announced for this
destination, as a float. If no destination-specific cost has
been set, the default link request cost is returned.
:returns: The announced link request cost as a *float*.
"""
cost = self._link_cost
if cost == None: cost = RNS.Transport.LR_COST
return max(0.0, min(float(cost), RNS.Transport.STAMP_COST_CAP))
def accepts_links(self, accepts = None): def accepts_links(self, accepts = None):
""" """
Set or query whether the destination accepts incoming link requests. Set or query whether the destination accepts incoming link requests.
@ -462,20 +362,10 @@ class Destination:
:param proof_strategy: One of ``RNS.Destination.PROVE_NONE``, ``RNS.Destination.PROVE_ALL`` or ``RNS.Destination.PROVE_APP``. If ``RNS.Destination.PROVE_APP`` is set, the `proof_requested_callback` will be called to determine whether a proof should be sent or not. :param proof_strategy: One of ``RNS.Destination.PROVE_NONE``, ``RNS.Destination.PROVE_ALL`` or ``RNS.Destination.PROVE_APP``. If ``RNS.Destination.PROVE_APP`` is set, the `proof_requested_callback` will be called to determine whether a proof should be sent or not.
""" """
if not proof_strategy in Destination.proof_strategies: raise TypeError("Unsupported proof strategy") if not proof_strategy in Destination.proof_strategies:
else: self.proof_strategy = proof_strategy raise TypeError("Unsupported proof strategy")
else:
def set_max_request_size(self, max_request_size): self.proof_strategy = proof_strategy
"""
Sets the maximum accepted request size for registered request handlers.
:param max_request_size: The maximum accepted request size in bytes, as an integer.
:raises: ``TypeError`` or ``ValueError`` if any of the argument is invalid.
"""
try: max_request_size = int(max_request_size)
except: raise TypeError("Invalid maximum request size specified")
if max_request_size < 0: raise ValueError("Maximum request size cannot be negative")
self.max_request_size = max_request_size
def register_request_handler(self, path, response_generator = None, allow = ALLOW_NONE, allowed_list = None, auto_compress = True): def register_request_handler(self, path, response_generator = None, allow = ALLOW_NONE, allowed_list = None, auto_compress = True):
""" """
@ -530,7 +420,8 @@ class Destination:
def incoming_link_request(self, data, packet): def incoming_link_request(self, data, packet):
if self.accept_link_requests: if self.accept_link_requests:
link = RNS.Link.validate_request(self, data, packet) link = RNS.Link.validate_request(self, data, packet)
if link != None: self.links.append(link) if link != None:
self.links.append(link)
def _reload_ratchets(self, ratchets_path): def _reload_ratchets(self, ratchets_path):
if os.path.isfile(ratchets_path): if os.path.isfile(ratchets_path):

View file

@ -1,33 +1,3 @@
# Reticulum License
#
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# - The Software shall not be used in any kind of system which includes amongst
# its functions the ability to purposefully do harm to human beings.
#
# - The Software shall not be used, directly or indirectly, in the creation of
# an artificial intelligence, machine learning or language model training
# dataset, including but not limited to any use that contributes to the
# training or development of such a model or algorithm.
#
# - The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import os import os
import re import re
import RNS import RNS
@ -38,21 +8,9 @@ import ipaddress
import subprocess import subprocess
from threading import Lock from threading import Lock
from .vendor import umsgpack as msgpack from .vendor import umsgpack as msgpack
from collections import deque
# Discoverable interfaces must include a short,
# unique implementation-specific identifier and
# version tag in announced discovery information.
IMPLEMENTATION_NAME = "RNS"
from RNS._version import __version__ as RNS_VERSION
import RNS.Stamps
NAME = 0xFF NAME = 0xFF
TRANSPORT_ID = 0xFE TRANSPORT_ID = 0xFE
TRANSPORT_IMPL = 0xFD
TRANSPORT_VERS = 0xFC
DST_COST = 0xFB
PR_COST = 0xFA
INTERFACE_TYPE = 0x00 INTERFACE_TYPE = 0x00
TRANSPORT = 0x01 TRANSPORT = 0x01
REACHABLE_ON = 0x02 REACHABLE_ON = 0x02
@ -68,23 +26,29 @@ SPREADINGFACTOR = 0x0B
CODINGRATE = 0x0C CODINGRATE = 0x0C
MODULATION = 0x0D MODULATION = 0x0D
CHANNEL = 0x0E CHANNEL = 0x0E
OP_ADDR = 0xF0
APP_NAME = "rnstransport" APP_NAME = "rnstransport"
class InterfaceAnnouncer(): class InterfaceAnnouncer():
JOB_INTERVAL = 60 JOB_INTERVAL = 60
DEFAULT_STAMP_VALUE = 16 DEFAULT_STAMP_VALUE = 14
WORKBLOCK_EXPAND_ROUNDS = 20 WORKBLOCK_EXPAND_ROUNDS = 20
DISCOVERABLE_INTERFACE_TYPES = ["BackboneInterface", "TCPServerInterface", "TCPClientInterface", DISCOVERABLE_INTERFACE_TYPES = ["BackboneInterface", "TCPServerInterface", "TCPClientInterface",
"RNodeInterface", "WeaveInterface", "I2PInterface", "KISSInterface"] "RNodeInterface", "WeaveInterface", "I2PInterface", "KISSInterface"]
def __init__(self, owner): def __init__(self, owner):
import importlib.util
if importlib.util.find_spec('LXMF') != None: from LXMF import LXStamper
else:
RNS.log("Using on-network interface discovery requires the LXMF module to be installed.", RNS.LOG_CRITICAL)
RNS.log("You can install it with the command: pip install lxmf", RNS.LOG_CRITICAL)
RNS.panic()
self.owner = owner self.owner = owner
self.should_run = False self.should_run = False
self.job_interval = self.JOB_INTERVAL self.job_interval = self.JOB_INTERVAL
self.stamper = RNS.Stamps self.stamper = LXStamper
self.stamp_cache = {} self.stamp_cache = {}
if self.owner.has_network_identity(): identity = self.owner.network_identity if self.owner.has_network_identity(): identity = self.owner.network_identity
@ -135,54 +99,20 @@ class InterfaceAnnouncer():
if not interface_type in self.DISCOVERABLE_INTERFACE_TYPES: return None if not interface_type in self.DISCOVERABLE_INTERFACE_TYPES: return None
else: else:
if not RNS.vendor.platformutils.is_windows() and interface.discovery_location:
try:
discovery_location = self.sanitize(interface.discovery_location)
exec_path = os.path.expanduser(discovery_location)
if os.path.isfile(exec_path) and os.access(exec_path, os.X_OK):
RNS.log(f"Evaluating discovery location from executable at {exec_path}", RNS.LOG_DEBUG)
exec_result = subprocess.run([exec_path], stdout=subprocess.PIPE)
exec_stdout = exec_result.stdout.decode("utf-8")
if exec_result.returncode != 0: raise ValueError("Non-zero exit code from subprocess")
discovery_location = self.sanitize(exec_stdout)
location_components = discovery_location.replace(" ", "").split(",")
if len(location_components) != 3: raise ValueError(f"Invalid location component count: {len(location_components)}")
dlat = float(location_components[0])
dlon = float(location_components[1])
dhgt = float(location_components[2])
if dlat < -90 or dlat > 90: raise ValueError(f"Invalid latitude: {dlat}")
if dlon < -180 or dlat > 180: raise ValueError(f"Invalid longitude: {dlon}")
if dhgt < -4000 or dhgt > 1e6: raise ValueError(f"Invalid height: {dhgt}")
interface.discovery_latitude = dlat
interface.discovery_longitude = dlon
interface.discovery_height = dhgt
except Exception as e:
RNS.log(f"Error while getting discovery location from executable at {interface.reachable_on}: {e}", RNS.LOG_ERROR)
RNS.log(f"Aborting discovery announce", RNS.LOG_ERROR)
return None
flags = 0x00 flags = 0x00
info = {INTERFACE_TYPE: interface_type, info = {INTERFACE_TYPE: interface_type,
TRANSPORT: RNS.Reticulum.transport_enabled(), TRANSPORT: RNS.Reticulum.transport_enabled(),
TRANSPORT_ID: RNS.Transport.identity.hash, TRANSPORT_ID: RNS.Transport.identity.hash,
TRANSPORT_IMPL: IMPLEMENTATION_NAME,
TRANSPORT_VERS: RNS_VERSION,
DST_COST: RNS.Stamps.quantize_cost(interface.destination_cost),
PR_COST: RNS.Stamps.quantize_cost(interface.pr_cost),
NAME: self.sanitize(interface.discovery_name), NAME: self.sanitize(interface.discovery_name),
LATITUDE: interface.discovery_latitude, LATITUDE: interface.discovery_latitude,
LONGITUDE: interface.discovery_longitude, LONGITUDE: interface.discovery_longitude,
HEIGHT: interface.discovery_height} HEIGHT: interface.discovery_height}
if interface.discovery_lxmf_address: info[OP_ADDR] = interface.discovery_lxmf_address
if interface_type == "TCPClientInterface" and not interface.kiss_framing: if interface_type == "TCPClientInterface" and not interface.kiss_framing:
RNS.log(f"Invalid interface discovery configuration for {interface}, aborting discovery announce", RNS.LOG_ERROR) RNS.log(f"Invalid interface discovery configuration for {interface}, aborting discovery announce", RNS.LOG_ERROR)
return None return None
if interface_type in ["BackboneInterface", "TCPServerInterface"]: if interface_type in ["BackboneInterface", "TCPServerInterface"]:
if not interface.reachable_on: return None
reachable_on = self.sanitize(interface.reachable_on) reachable_on = self.sanitize(interface.reachable_on)
if not RNS.vendor.platformutils.is_windows(): if not RNS.vendor.platformutils.is_windows():
@ -232,8 +162,8 @@ class InterfaceAnnouncer():
info[MODULATION] = self.sanitize(interface.discovery_modulation) info[MODULATION] = self.sanitize(interface.discovery_modulation)
if interface.discovery_publish_ifac == True: if interface.discovery_publish_ifac == True:
if interface.ifac_netname: info[IFAC_NETNAME] = self.sanitize(interface.ifac_netname) info[IFAC_NETNAME] = self.sanitize(interface.ifac_netname)
if interface.ifac_netkey: info[IFAC_NETKEY] = self.sanitize(interface.ifac_netkey) info[IFAC_NETKEY] = self.sanitize(interface.ifac_netkey)
packed = msgpack.packb(info) packed = msgpack.packb(info)
infohash = RNS.Identity.full_hash(packed) infohash = RNS.Identity.full_hash(packed)
@ -260,14 +190,17 @@ class InterfaceAnnounceHandler:
FLAG_ENCRYPTED = 0b00000010 FLAG_ENCRYPTED = 0b00000010
def __init__(self, required_value=InterfaceAnnouncer.DEFAULT_STAMP_VALUE, callback=None): def __init__(self, required_value=InterfaceAnnouncer.DEFAULT_STAMP_VALUE, callback=None):
import importlib.util
if importlib.util.find_spec('LXMF') != None: from LXMF import LXStamper
else:
RNS.log("Using on-network interface discovery requires the LXMF module to be installed.", RNS.LOG_CRITICAL)
RNS.log("You can install it with the command: pip install lxmf", RNS.LOG_CRITICAL)
RNS.panic()
self.aspect_filter = APP_NAME+".discovery.interface" self.aspect_filter = APP_NAME+".discovery.interface"
self.required_value = required_value self.required_value = required_value
self.callback = callback self.callback = callback
self.stamper = RNS.Stamps self.stamper = LXStamper
self.valid_cache = {}
self.valid_cache_max = 2048
self.invalid_cache = deque(maxlen=2048)
self.validation_lock = threading.Lock()
@staticmethod @staticmethod
def sanitize_name(name): def sanitize_name(name):
@ -290,70 +223,30 @@ class InterfaceAnnounceHandler:
app_data = app_data[1:] app_data = app_data[1:]
signed = flags & self.FLAG_SIGNED signed = flags & self.FLAG_SIGNED
encrypted = flags & self.FLAG_ENCRYPTED encrypted = flags & self.FLAG_ENCRYPTED
fullhash = RNS.Identity.full_hash(app_data)
hops = RNS.Transport.hops_to(destination_hash)
if hops > RNS.Transport.PATHFINDER_M-1:
RNS.log(f"Ignored discovered interface with invalid hop count", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
return
if fullhash in self.invalid_cache:
RNS.log(f"Ignored previously discovered interface with insufficient stamp value", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
return
if fullhash in self.valid_cache:
RNS.log(f"Discovery announce cache hit", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
cache_entry = self.valid_cache[fullhash]
stamp = cache_entry["stamp"]
packed = cache_entry["packed"]
value = cache_entry["value"]
valid = cache_entry["valid"]
else:
if encrypted: if encrypted:
if not RNS.Transport.has_network_identity(): return if not RNS.Transport.has_network_identity(): return
app_data = RNS.Transport.network_identity.decrypt(app_data) app_data = RNS.Transport.network_identity.decrypt(app_data)
if not app_data: return if not app_data: return
if self.validation_lock.locked():
RNS.log(f"Dropping received interface discovery announce, already validating other discovery stamp", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return
with self.validation_lock:
stamp = app_data[-self.stamper.STAMP_SIZE:] stamp = app_data[-self.stamper.STAMP_SIZE:]
packed = app_data[:-self.stamper.STAMP_SIZE] packed = app_data[:-self.stamper.STAMP_SIZE]
infohash = RNS.Identity.full_hash(packed) infohash = RNS.Identity.full_hash(packed)
# Transitory mechanism, check non-expanded workblock validity first.
workblock_ne = self.stamper.stamp_workblock(infohash, expand_rounds=0)
valid_ne = self.stamper.stamp_valid(stamp, self.required_value, workblock_ne)
if valid_ne:
value = self.stamper.stamp_value(workblock_ne, stamp)
valid = valid_ne
else:
workblock = self.stamper.stamp_workblock(infohash, expand_rounds=InterfaceAnnouncer.WORKBLOCK_EXPAND_ROUNDS) workblock = self.stamper.stamp_workblock(infohash, expand_rounds=InterfaceAnnouncer.WORKBLOCK_EXPAND_ROUNDS)
value = self.stamper.stamp_value(workblock, stamp) value = self.stamper.stamp_value(workblock, stamp)
valid = self.stamper.stamp_valid(stamp, self.required_value, workblock) valid = self.stamper.stamp_valid(stamp, self.required_value, workblock)
self.valid_cache[fullhash] = {"valid": valid, "value": value, "packed": packed, "stamp": stamp}
while len(self.valid_cache) > self.valid_cache_max: self.valid_cache.pop(next(iter(self.valid_cache)))
if not valid: if not valid:
RNS.log(f"Ignored discovered interface with insufficient stamp value {RNS.prettycost(value)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log(f"Ignored discovered interface with invalid stamp", RNS.LOG_DEBUG)
self.invalid_cache.append(fullhash)
return return
if value < self.required_value: RNS.log(f"Ignored discovered interface with stamp value {value}", RNS.LOG_DEBUG)
else: else:
info = None info = None
unpacked = msgpack.unpackb(packed) unpacked = msgpack.unpackb(packed)
if INTERFACE_TYPE in unpacked: if INTERFACE_TYPE in unpacked:
interface_type = unpacked[INTERFACE_TYPE] interface_type = unpacked[INTERFACE_TYPE]
name = self.sanitize_name(unpacked[NAME]) name = self.sanitize_name(unpacked[NAME])
impl_name = unpacked[TRANSPORT_IMPL] if TRANSPORT_IMPL in unpacked else None
impl_version = unpacked[TRANSPORT_VERS] if TRANSPORT_VERS in unpacked else None
dst_cost = RNS.Stamps.quantize_cost(unpacked[DST_COST]) if DST_COST in unpacked else None
pr_cost = RNS.Stamps.quantize_cost(unpacked[PR_COST]) if PR_COST in unpacked else None
if type(unpacked[TRANSPORT]) != bool: raise ValueError("Invalid data in transport field of announce") if type(unpacked[TRANSPORT]) != bool: raise ValueError("Invalid data in transport field of announce")
if type(unpacked[LATITUDE]) not in [type(None), float]: raise ValueError("Invalid data in latitude field of announce") if type(unpacked[LATITUDE]) not in [type(None), float]: raise ValueError("Invalid data in latitude field of announce")
@ -368,8 +261,6 @@ class InterfaceAnnounceHandler:
raise ValueError("Invalid data in reachable_on field of announce") raise ValueError("Invalid data in reachable_on field of announce")
info = {"type": interface_type, info = {"type": interface_type,
"impl_name": impl_name,
"version": impl_version,
"transport": unpacked[TRANSPORT], "transport": unpacked[TRANSPORT],
"name": name or f"Discovered {interface_type}", "name": name or f"Discovered {interface_type}",
"received": time.time(), "received": time.time(),
@ -377,18 +268,16 @@ class InterfaceAnnounceHandler:
"value": value, "value": value,
"transport_id": RNS.hexrep(unpacked[TRANSPORT_ID], delimit=False), "transport_id": RNS.hexrep(unpacked[TRANSPORT_ID], delimit=False),
"network_id": RNS.hexrep(announced_identity.hash, delimit=False), "network_id": RNS.hexrep(announced_identity.hash, delimit=False),
"dst_cost": dst_cost, "hops": RNS.Transport.hops_to(destination_hash),
"pr_cost": pr_cost,
"hops": hops,
"latitude": unpacked[LATITUDE], "latitude": unpacked[LATITUDE],
"longitude": unpacked[LONGITUDE], "longitude": unpacked[LONGITUDE],
"height": unpacked[HEIGHT]} "height": unpacked[HEIGHT]}
if IFAC_NETNAME in unpacked and unpacked[IFAC_NETNAME] and type(unpacked[IFAC_NETNAME]) == str: info["ifac_netname"] = str(unpacked[IFAC_NETNAME]) if IFAC_NETNAME in unpacked: info["ifac_netname"] = str(unpacked[IFAC_NETNAME])
if IFAC_NETKEY in unpacked and unpacked[IFAC_NETKEY] and type(unpacked[IFAC_NETKEY]) == str: info["ifac_netkey"] = str(unpacked[IFAC_NETKEY]) if IFAC_NETKEY in unpacked: info["ifac_netkey"] = str(unpacked[IFAC_NETKEY])
if interface_type in ["BackboneInterface", "TCPServerInterface"]: if interface_type in ["BackboneInterface", "TCPServerInterface"]:
backbone_support = not RNS.vendor.platformutils.is_windows() and not RNS.vendor.platformutils.is_darwin() backbone_support = not RNS.vendor.platformutils.is_windows()
info["reachable_on"] = unpacked[REACHABLE_ON] info["reachable_on"] = unpacked[REACHABLE_ON]
info["port"] = unpacked[PORT] info["port"] = unpacked[PORT]
connection_interface = "BackboneInterface" if backbone_support else "TCPClientInterface" connection_interface = "BackboneInterface" if backbone_support else "TCPClientInterface"
@ -414,7 +303,7 @@ class InterfaceAnnounceHandler:
cfg_netname_str = f"\n network_name = {cfg_netname}" if cfg_netname else "" cfg_netname_str = f"\n network_name = {cfg_netname}" if cfg_netname else ""
cfg_netkey_str = f"\n passphrase = {cfg_netkey}" if cfg_netkey else "" cfg_netkey_str = f"\n passphrase = {cfg_netkey}" if cfg_netkey else ""
cfg_identity_str = f"\n transport_identity = {cfg_identity}" cfg_identity_str = f"\n transport_identity = {cfg_identity}"
info["config_entry"] = f"[[{cfg_name}]]\n type = I2PInterface\n enabled = yes\n peers = {cfg_remote}.b32.i2p{cfg_identity_str}{cfg_netname_str}{cfg_netkey_str}" info["config_entry"] = f"[[{cfg_name}]]\n type = I2PInterface\n enabled = yes\n peers = {cfg_remote}{cfg_identity_str}{cfg_netname_str}{cfg_netkey_str}"
if interface_type == "RNodeInterface": if interface_type == "RNodeInterface":
info["frequency"] = unpacked[FREQUENCY] info["frequency"] = unpacked[FREQUENCY]
@ -467,15 +356,10 @@ class InterfaceAnnounceHandler:
discovery_hash_material = info["transport_id"]+info["name"] discovery_hash_material = info["transport_id"]+info["name"]
info["discovery_hash"] = RNS.Identity.full_hash(discovery_hash_material.encode("utf-8")) info["discovery_hash"] = RNS.Identity.full_hash(discovery_hash_material.encode("utf-8"))
if info and OP_ADDR in unpacked:
if type(unpacked[OP_ADDR]) not in [type(None), bytes]: raise ValueError("Invalid data in operator LXMF address field of announce")
if unpacked[OP_ADDR] and len(unpacked[OP_ADDR]) == RNS.Identity.TRUNCATED_HASHLENGTH//8:
info["operator_lxmf_address"] = RNS.hexrep(unpacked[OP_ADDR], delimit=False)
if self.callback and callable(self.callback): self.callback(info) if self.callback and callable(self.callback): self.callback(info)
except Exception as e: except Exception as e:
RNS.log(f"Could not decode discovered interface. The contained exception was: {e}", RNS.LOG_DEBUG) RNS.log(f"An error occurred while trying to decode discovered interface. The contained exception was: {e}", RNS.LOG_DEBUG)
class InterfaceDiscovery(): class InterfaceDiscovery():
THRESHOLD_UNKNOWN = 24*60*60 THRESHOLD_UNKNOWN = 24*60*60
@ -489,17 +373,10 @@ class InterfaceDiscovery():
STATUS_UNKNOWN = 100 STATUS_UNKNOWN = 100
STATUS_AVAILABLE = 1000 STATUS_AVAILABLE = 1000
STATUS_CODE_MAP = {"available": STATUS_AVAILABLE, "unknown": STATUS_UNKNOWN, "stale": STATUS_STALE} STATUS_CODE_MAP = {"available": STATUS_AVAILABLE, "unknown": STATUS_UNKNOWN, "stale": STATUS_STALE}
AUTOCONNECT_TYPES = ["BackboneInterface"] AUTOCONNECT_TYPES = ["BackboneInterface", "TCPServerInterface"]
DISCOVERABLE_TYPES = ["BackboneInterface", "TCPServerInterface", "I2PInterface", "RNodeInterface", "WeaveInterface", "KISSInterface"] DISCOVERABLE_TYPES = ["BackboneInterface", "TCPServerInterface", "I2PInterface", "RNodeInterface", "WeaveInterface", "KISSInterface"]
AUTOCONNECT_IMPLS = ["RNS"]
AUTOCONNECT_MIN_V = "1.5.7"
AC_TRANSPORT_MODE = RNS.Interfaces.Interface.Interface.MODE_GATEWAY
AC_GRAVITY = 0
discovery_lock = Lock() discovery_lock = Lock()
autoconnect_lock = Lock()
def __init__(self, required_value=InterfaceAnnouncer.DEFAULT_STAMP_VALUE, callback=None, discover_interfaces=True): def __init__(self, required_value=InterfaceAnnouncer.DEFAULT_STAMP_VALUE, callback=None, discover_interfaces=True):
if not required_value: required_value = InterfaceAnnouncer.DEFAULT_STAMP_VALUE if not required_value: required_value = InterfaceAnnouncer.DEFAULT_STAMP_VALUE
@ -512,31 +389,20 @@ class InterfaceDiscovery():
self.monitor_interval = self.MONITOR_INTERVAL self.monitor_interval = self.MONITOR_INTERVAL
self.detach_threshold = self.DETACH_THRESHOLD self.detach_threshold = self.DETACH_THRESHOLD
self.initial_autoconnect_ran = False self.initial_autoconnect_ran = False
self.blackholed_updated = 0
self.__blackholed = {}
if not self.rns_instance: raise SystemError("Attempt to start interface discovery listener without an active RNS instance") if not self.rns_instance: raise SystemError("Attempt to start interface discovery listener without an active RNS instance")
self.storagepath = os.path.join(RNS.Reticulum.storagepath, "discovery", "interfaces") self.storagepath = os.path.join(RNS.Reticulum.storagepath, "discovery", "interfaces")
if not os.path.isdir(self.storagepath): os.makedirs(self.storagepath) if not os.path.isdir(self.storagepath): os.makedirs(self.storagepath)
if discover_interfaces: if discover_interfaces:
RNS.Reticulum.get_instance().wants_destination_type("rnstransport.discovery.interface")
self.handler = InterfaceAnnounceHandler(callback=self.interface_discovered, required_value=self.required_value) self.handler = InterfaceAnnounceHandler(callback=self.interface_discovered, required_value=self.required_value)
RNS.Transport.register_announce_handler(self.handler) RNS.Transport.register_announce_handler(self.handler)
threading.Thread(target=self.connect_discovered, daemon=True).start() threading.Thread(target=self.connect_discovered, daemon=True).start()
def __blackholed_identities(self):
st = time.time()
if not self.__blackholed or time.time() > self.blackholed_updated+60:
self.__blackholed = self.rns_instance.get_blackholed_identities()
self.blackholed_updated = time.time()
return self.__blackholed
def list_discovered_interfaces(self, only_available=False, only_transport=False): def list_discovered_interfaces(self, only_available=False, only_transport=False):
now = time.time() now = time.time()
discovered_interfaces = [] discovered_interfaces = []
discovery_sources = RNS.Reticulum.interface_discovery_sources() discovery_sources = RNS.Reticulum.interface_discovery_sources()
blackholed_identities = self.__blackholed_identities()
for filename in os.listdir(self.storagepath): for filename in os.listdir(self.storagepath):
try: try:
with self.discovery_lock: with self.discovery_lock:
@ -547,23 +413,10 @@ class InterfaceDiscovery():
heard_delta = now-info["last_heard"] heard_delta = now-info["last_heard"]
info["name"] = InterfaceAnnounceHandler.sanitize_name(info["name"]) info["name"] = InterfaceAnnounceHandler.sanitize_name(info["name"])
# TODO: Temporary measure to sanitize invalidly persisted "None" values
# for incorrect IFAC parameters published due to unguarded nonsensical
# node-side configuration. Potentially remove once nodes upgrade to RNS
# versions that guard against the configuration mismatch.
try:
if "ifac_netname" in info and info["ifac_netname"] == "None": info.pop("ifac_netname")
if "ifac_netkey" in info and info["ifac_netkey"] == "None": info.pop("ifac_netkey")
except Exception as e: RNS.log(f"Error while sanitizing discovery info: {e}", RNS.LOG_ERROR)
if heard_delta > self.THRESHOLD_REMOVE: should_remove = True if heard_delta > self.THRESHOLD_REMOVE: should_remove = True
elif not "transport_id" in info or not info["transport_id"]: should_remove = True
elif not "network_id" in info or not info["network_id"]: should_remove = True
elif discovery_sources and not "network_id" in info: should_remove = True elif discovery_sources and not "network_id" in info: should_remove = True
elif discovery_sources and not bytes.fromhex(info["network_id"]) in discovery_sources: should_remove = True elif discovery_sources and not bytes.fromhex(info["network_id"]) in discovery_sources: should_remove = True
elif not "type" in info or ("type" in info and not info["type"] in self.DISCOVERABLE_TYPES): should_remove = True elif not "type" in info or ("type" in info and not info["type"] in self.DISCOVERABLE_TYPES): should_remove = True
elif bytes.fromhex(info["network_id"]) in blackholed_identities: should_remove = True
elif bytes.fromhex(info["transport_id"]) in blackholed_identities: should_remove = True
elif "reachable_on" in info: elif "reachable_on" in info:
if not (is_ip_address(info["reachable_on"]) or is_hostname(info["reachable_on"])): should_remove = True if not (is_ip_address(info["reachable_on"]) or is_hostname(info["reachable_on"])): should_remove = True
@ -601,13 +454,11 @@ class InterfaceDiscovery():
interface_type = info["type"] interface_type = info["type"]
discovery_hash = info["discovery_hash"] discovery_hash = info["discovery_hash"]
discovered_type = info["type"] discovered_type = info["type"]
has_impl_info = "impl_name" in info and "version" in info and info["impl_name"] and info["version"]
version_str = f" ({info['impl_name']} {info['version']})" if has_impl_info else " (unknown implementation)"
if not discovered_type in self.DISCOVERABLE_TYPES: return if not discovered_type in self.DISCOVERABLE_TYPES: return
hops = info["hops"]; ms = "" if hops == 1 else "s" hops = info["hops"]; ms = "" if hops == 1 else "s"
filename = RNS.hexrep(discovery_hash, delimit=False) filename = RNS.hexrep(discovery_hash, delimit=False)
filepath = os.path.join(self.storagepath, filename) filepath = os.path.join(self.storagepath, filename)
RNS.log(f"Discovered {interface_type}{version_str} {hops} hop{ms} away with stamp value {RNS.prettycost(value)}: {name}", RNS.LOG_DEBUG) RNS.log(f"Discovered {interface_type} {hops} hop{ms} away with stamp value {value}: {name}", RNS.LOG_DEBUG)
with self.discovery_lock: with self.discovery_lock:
if not os.path.isfile(filepath): if not os.path.isfile(filepath):
try: try:
@ -675,11 +526,6 @@ class InterfaceDiscovery():
autoconnected_interfaces = self.autoconnect_count() autoconnected_interfaces = self.autoconnect_count()
for interface in self.monitored_interfaces: for interface in self.monitored_interfaces:
try: try:
if not interface in RNS.Transport.interfaces:
RNS.log(f"A monitored auto-connected interface was manually detached, removing from monitoring", RNS.LOG_DEBUG)
detached_interfaces.append(interface)
else:
if interface.online: if interface.online:
online_interfaces += 1 online_interfaces += 1
if hasattr(interface, "autoconnect_down") and interface.autoconnect_down != None: if hasattr(interface, "autoconnect_down") and interface.autoconnect_down != None:
@ -729,10 +575,8 @@ class InterfaceDiscovery():
RNS.log(f"Error while de-registering auto-connected interface from transport: {e}", RNS.LOG_ERROR) RNS.log(f"Error while de-registering auto-connected interface from transport: {e}", RNS.LOG_ERROR)
def teardown_interface(self, interface): def teardown_interface(self, interface):
if interface in RNS.Transport.interfaces: interface.detach()
was_detached = RNS.Reticulum.get_instance()._detach_interface(interface.name, internal_forced=True) RNS.Transport.remove_interface(interface)
if not was_detached: RNS.log(f"Detach of auto-connected interface {interface} did not succeed", RNS.LOG_ERROR)
if interface in self.monitored_interfaces: self.monitored_interfaces.remove(interface) if interface in self.monitored_interfaces: self.monitored_interfaces.remove(interface)
def autoconnect_count(self): def autoconnect_count(self):
@ -779,123 +623,57 @@ class InterfaceDiscovery():
return exists return exists
def autoconnect_interface_name(self, name):
names = {interface.name for interface in RNS.Transport.interfaces}
if not name in names: return name
else:
n = 2; seq_name = f"{name} ({n})"
while seq_name in names:
n += 1; seq_name = f"{name} ({n})"
return seq_name
def autoconnect_qualified(self, info):
if not "transport" in info or info["transport"] != True: return False
if RNS.Reticulum.should_autoconnect_unverified_implementations(): return True
if not "impl_name" in info: return False
if not info["impl_name"] in self.AUTOCONNECT_IMPLS: return False
if not "version" in info or not info["version"]: return False
version = version_tuple(info["version"])
if version == None: return False
return version >= version_tuple(self.AUTOCONNECT_MIN_V)
def autoconnect(self, info): def autoconnect(self, info):
try: try:
if RNS.Reticulum.should_autoconnect_discovered_interfaces(): if RNS.Reticulum.should_autoconnect_discovered_interfaces():
autoconnected_count = self.autoconnect_count() autoconnected_count = self.autoconnect_count()
if autoconnected_count < RNS.Reticulum.max_autoconnected_interfaces(): if autoconnected_count < RNS.Reticulum.max_autoconnected_interfaces():
interface_type = info["type"] interface_type = info["type"]
has_cost_info = "dst_cost" in info and "pr_cost" in info
if interface_type in self.AUTOCONNECT_TYPES: if interface_type in self.AUTOCONNECT_TYPES:
impl = f"{info['impl_name']} {info['version']}" if ("impl_name" in info and "version" in info and info["impl_name"] and info["version"]) else "unknown implementation"
if not self.autoconnect_qualified(info):
RNS.log(f"Not auto-connecting discovered {interface_type} {info['name']} ({impl}), auto-connect criteria not satisfied", RNS.LOG_DEBUG)
return
if has_cost_info:
dst_value = RNS.Stamps.quantize_cost(RNS.Transport.DESTINATION_VALUE)
pr_value = RNS.Stamps.quantize_cost(RNS.Transport.PR_VALUE)
dst_cost = RNS.Stamps.quantize_cost(info["dst_cost"] or 0)
pr_cost = RNS.Stamps.quantize_cost(info["pr_cost"] or 0)
dst_cost_ok = dst_cost <= dst_value
pr_cost_ok = pr_cost <= pr_value
if not dst_cost_ok: RNS.log(f"Destination cost {dst_cost} higher than configured value {dst_value} for {interface_type} {info['name']} ({impl}), not auto-connecting", RNS.LOG_DEBUG)
if not pr_cost_ok: RNS.log(f"Path request cost {pr_cost} higher than configured value {pr_value} for {interface_type} {info['name']} ({impl}), not auto-connecting", RNS.LOG_DEBUG)
if not dst_cost_ok or not pr_cost_ok: return
endpoint_hash = self.endpoint_hash(info) endpoint_hash = self.endpoint_hash(info)
exists = self.interface_exists(info) exists = self.interface_exists(info)
if exists: RNS.log(f"Discovered {interface_type} already exists, not auto-connecting", RNS.LOG_DEBUG) if exists: RNS.log(f"Discovered {interface_type} already exists, not auto-connecting", RNS.LOG_DEBUG)
else: else:
if interface_type == "TCPClientInterface":
RNS.log(f"Your operating system does not support the Backbone interface type, and must degrade to using TCPClientInterface instead", RNS.LOG_WARNING)
RNS.log(f"Auto-connecting discovered TCPClient interfaces is not yet implemented, aborting auto-connect", RNS.LOG_WARNING)
RNS.log(f"You can obtain the configuration entry and add this interface manually instead using rnstatus -D", RNS.LOG_WARNING)
return
if interface_type == "I2PInterface": if interface_type == "I2PInterface":
RNS.log(f"Auto-connecting discovered I2P interfaces is not yet implemented, aborting auto-connect", RNS.LOG_DEBUG) RNS.log(f"Auto-connecting discovered I2P interfaces is not yet implemented, aborting auto-connect", RNS.LOG_WARNING)
RNS.log(f"You can obtain the configuration entry and add this interface manually instead using rnstatus -D", RNS.LOG_DEBUG) RNS.log(f"You can obtain the configuration entry and add this interface manually instead using rnstatus -D", RNS.LOG_WARNING)
return return
if is_ygg_ipv6(info["reachable_on"]): if is_ygg_ipv6(info["reachable_on"]):
# TODO: Somehow detect if yggdrasil is enabled on the system # TODO: Somehow detect if yggdrasil is enabled on the system
return return
if is_onion_address(info["reachable_on"]): interface_name = info["name"]
# TODO: Somehow detect if TOR is enabled on the system
return
if is_ip_address(info["reachable_on"]) and is_invalid_ip_address(info["reachable_on"]):
RNS.log(f"Not auto-connecting discovered interface with invalid IP address: {info['reachable_on']}", RNS.LOG_DEBUG)
return
with self.autoconnect_lock:
if self.interface_exists(info): return
backbone_support = (not RNS.vendor.platformutils.is_windows() and not RNS.vendor.platformutils.is_darwin())
interface_name = self.autoconnect_interface_name(info["name"])
if interface_name != info["name"]:
RNS.log(f"Auto-connect name collision for \"{info['name']}\", connecting as \"{interface_name}\"", RNS.LOG_NOTICE)
config_entry = info["config_entry"] config_entry = info["config_entry"]
interface_config = {} interface_config = {}
interface_config["name"] = f"{interface_name}" interface_config["name"] = f"{interface_name}"
ifac_netname = info["ifac_netname"] if "ifac_netname" in info else None
# TODO: Temporary measure to sanitize invalidly persisted "None" values ifac_netkey = info["ifac_netkey"] if "ifac_netkey" in info else None
# for incorrect IFAC parameters published due to unguarded nonsensical
# node-side configuration. Potentially remove once nodes upgrade to RNS
# versions that guard against the configuration mismatch.
if "ifac_netname" in info and info["ifac_netname"] == "None": info["ifac_netname"] = None
if "ifac_netkey" in info and info["ifac_netkey"] == "None": info["ifac_netkey"] = None
ifac_netname = info["ifac_netname"] if "ifac_netname" in info and info["ifac_netname"] else None
ifac_netkey = info["ifac_netkey"] if "ifac_netkey" in info and info["ifac_netkey"] else None
interface = None interface = None
if interface_type == "BackboneInterface": if interface_type == "BackboneInterface":
if backbone_support:
from RNS.Interfaces import BackboneInterface from RNS.Interfaces import BackboneInterface
interface_class = BackboneInterface.BackboneClientInterface
else:
from RNS.Interfaces import TCPInterface
interface_class = TCPInterface.TCPClientInterface
RNS.log(f"BackboneInterface is not yet supported on this operating system, auto-connecting discovered {interface_type} {interface_name} using TCPClientInterface", RNS.LOG_NOTICE)
interface_config["target_host"] = info["reachable_on"] interface_config["target_host"] = info["reachable_on"]
interface_config["target_port"] = info["port"] interface_config["target_port"] = info["port"]
interface = interface_class(RNS.Transport, interface_config) interface = BackboneInterface.BackboneClientInterface(RNS.Transport, interface_config)
if interface: if interface:
RNS.log(f"Auto-connecting discovered {interface_type} {interface_name}") RNS.log(f"Auto-connecting discovered {interface_type} {interface_name}")
interface.autoconnect_hash = endpoint_hash interface.autoconnect_hash = endpoint_hash
interface.autoconnect_source = info["network_id"] interface.autoconnect_source = info["network_id"]
if RNS.Reticulum.autoconnect_interface_mode(): mode = RNS.Reticulum.autoconnect_interface_mode() mode = RNS.Interfaces.Interface.Interface.MODE_GATEWAY if RNS.Reticulum.transport_enabled() else None
else: mode = self.AC_TRANSPORT_MODE if RNS.Reticulum.transport_enabled() else None
internal_a = True if RNS.Reticulum.autoconnect_announces_to_internal() else None
gravity = RNS.Reticulum.autoconnect_interface_gravity() or self.AC_GRAVITY
ar_target = RNS.Reticulum.get_instance()._default_ar_target() if RNS.Reticulum.transport_enabled() else None ar_target = RNS.Reticulum.get_instance()._default_ar_target() if RNS.Reticulum.transport_enabled() else None
ar_penalty = RNS.Reticulum.get_instance()._default_ar_penalty() if RNS.Reticulum.transport_enabled() else None ar_penalty = RNS.Reticulum.get_instance()._default_ar_penalty() if RNS.Reticulum.transport_enabled() else None
ar_grace = RNS.Reticulum.get_instance()._default_ar_grace() if RNS.Reticulum.transport_enabled() else None ar_grace = RNS.Reticulum.get_instance()._default_ar_grace() if RNS.Reticulum.transport_enabled() else None
RNS.Reticulum.get_instance()._add_interface(interface, mode=mode, ifac_netname=ifac_netname, ifac_netkey=ifac_netkey, configured_bitrate=5E6, RNS.Reticulum.get_instance()._add_interface(interface, mode=mode, ifac_netname=ifac_netname, ifac_netkey=ifac_netkey, configured_bitrate=5E6,
announce_rate_target=ar_target, announce_rate_grace=ar_grace, announce_rate_penalty=ar_penalty, announce_rate_target=ar_target, announce_rate_grace=ar_grace, announce_rate_penalty=ar_penalty)
announces_to_internal=internal_a, gravity=gravity)
self.monitor_interface(interface) self.monitor_interface(interface)
except Exception as e: except Exception as e:
@ -916,7 +694,6 @@ class BlackholeUpdater():
def start(self): def start(self):
if not self.should_run: if not self.should_run:
RNS.Reticulum.get_instance().wants_destination_type("rnstransport.info.blackhole")
source_count = len(RNS.Reticulum.blackhole_sources()) source_count = len(RNS.Reticulum.blackhole_sources())
ms = "" if source_count == 1 else "s" ms = "" if source_count == 1 else "s"
RNS.log(f"Starting blackhole updater with {source_count} source{ms}", RNS.LOG_DEBUG) RNS.log(f"Starting blackhole updater with {source_count} source{ms}", RNS.LOG_DEBUG)
@ -955,9 +732,8 @@ class BlackholeUpdater():
if os.path.isfile(sourcelistpath): os.unlink(sourcelistpath) if os.path.isfile(sourcelistpath): os.unlink(sourcelistpath)
os.rename(tmppath, sourcelistpath) os.rename(tmppath, sourcelistpath)
except Exception as e: RNS.log(f"Error while persisting blackhole list from {RNS.prettyhexrep(remote_identity.hash)}: {e}", RNS.LOG_ERROR) except Exception as e:
RNS.log(f"Error while persisting blackhole list from {RNS.prettyhexrep(remote_identity.hash)}: {e}", RNS.LOG_ERROR)
RNS.Identity._reset_validation_caches()
RNS.log(f"Blackhole list update from {RNS.prettyhexrep(remote_identity.hash)} completed", RNS.LOG_DEBUG) RNS.log(f"Blackhole list update from {RNS.prettyhexrep(remote_identity.hash)} completed", RNS.LOG_DEBUG)
@ -970,7 +746,7 @@ class BlackholeUpdater():
if identity_hash in self.last_updates: last_update = self.last_updates[identity_hash] if identity_hash in self.last_updates: last_update = self.last_updates[identity_hash]
else: last_update = 0 else: last_update = 0
if now > last_update+RNS.Reticulum.blackhole_update_interval(): if now > last_update+self.UPDATE_INTERVAL:
try: try:
destination_hash = RNS.Destination.hash_from_name_and_identity("rnstransport.info.blackhole", identity_hash) destination_hash = RNS.Destination.hash_from_name_and_identity("rnstransport.info.blackhole", identity_hash)
RNS.log(f"Attempting blackhole list update from {RNS.prettyhexrep(identity_hash)}...", RNS.LOG_DEBUG) RNS.log(f"Attempting blackhole list update from {RNS.prettyhexrep(identity_hash)}...", RNS.LOG_DEBUG)
@ -1000,15 +776,6 @@ def is_ygg_ipv6(address_string):
try: return ipaddress.ip_address(address_string) in ipaddress.IPv6Network("200::/7") try: return ipaddress.ip_address(address_string) in ipaddress.IPv6Network("200::/7")
except: return False except: return False
def is_onion_address(address_string):
try: return address_string.lower().endswith(".onion")
except: return False
INVALID_IP_ADDRESSES = ["127.0.0.1", "0.0.0.0"]
def is_invalid_ip_address(address_string):
if not address_string in INVALID_IP_ADDRESSES: return False
else: return True
def is_hostname(hostname): def is_hostname(hostname):
if hostname[-1] == ".": hostname = hostname[:-1] if hostname[-1] == ".": hostname = hostname[:-1]
if len(hostname) > 253: return False if len(hostname) > 253: return False
@ -1017,18 +784,6 @@ def is_hostname(hostname):
allowed = re.compile(r"(?!-)[a-z0-9-]{1,63}(?<!-)$", re.IGNORECASE) allowed = re.compile(r"(?!-)[a-z0-9-]{1,63}(?<!-)$", re.IGNORECASE)
return all(allowed.match(label) for label in components) return all(allowed.match(label) for label in components)
def version_tuple(version_string):
if type(version_string) != str: return None
try:
components = []
for component in version_string.strip().split("."):
match = re.match(r"[0-9]+", component)
if match == None: break
components.append(int(match.group(0)))
if len(components) == 0: return None
return tuple(components)
except Exception as e: return None
san_map = "" san_map = ""
for i in range(48, 58): san_map += bytes([i]).decode("ascii") for i in range(48, 58): san_map += bytes([i]).decode("ascii")
for i in range(65, 91): san_map += bytes([i]).decode("ascii") for i in range(65, 91): san_map += bytes([i]).decode("ascii")

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -40,9 +40,6 @@ from .vendor import umsgpack as umsgpack
from RNS.Cryptography import X25519PrivateKey, X25519PublicKey, Ed25519PrivateKey, Ed25519PublicKey from RNS.Cryptography import X25519PrivateKey, X25519PublicKey, Ed25519PrivateKey, Ed25519PublicKey
from RNS.Cryptography import Token from RNS.Cryptography import Token
from RNS.Storage import KnownDestinationsStore
from RNS.Stamps import stamp_workblock, stamp_valid, stamp_value, generate_stamped_key
from RNS.Stamps import cost_from_bytes, quantize_cost
class Identity: class Identity:
@ -81,14 +78,9 @@ class Identity:
AES128_BLOCKSIZE = 16 # In bytes AES128_BLOCKSIZE = 16 # In bytes
AES256_BLOCKSIZE = 16 # In bytes AES256_BLOCKSIZE = 16 # In bytes
HASHLENGTH = 256 # In bits HASHLENGTH = 256 # In bits
NAME_HASH_LENGTH = 80 # In bits
SIGLENGTH = KEYSIZE # In bits SIGLENGTH = KEYSIZE # In bits
STAMPLENGTH = 14 # In bytes
EXTHEADERLENGTH = 2 # In bytes
LRCOSTLENGTH = 2 # In bytes
EXTPAYLOADLENGTH = EXTHEADERLENGTH+STAMPLENGTH+LRCOSTLENGTH
EXTPAYLOADMARKER = 0xC1 # Never used at msgpack object boundaries
NAME_HASH_LENGTH = 80
TRUNCATED_HASHLENGTH = RNS.Reticulum.TRUNCATED_HASHLENGTH TRUNCATED_HASHLENGTH = RNS.Reticulum.TRUNCATED_HASHLENGTH
""" """
Constant specifying the truncated hash length (in bits) used by Reticulum Constant specifying the truncated hash length (in bits) used by Reticulum
@ -96,34 +88,29 @@ class Identity:
""" """
DERIVED_KEY_LENGTH = 512//8 DERIVED_KEY_LENGTH = 512//8
DERIVED_KEY_LENGTH_LEGACY = 256//8
# Storage # Storage
known_destinations = {} known_destinations = {}
known_destinations_store = None
known_ratchets = {} known_ratchets = {}
ratchet_persist_lock = threading.Lock() ratchet_persist_lock = threading.Lock()
known_destinations_lock = threading.Lock() known_destinations_lock = threading.Lock()
@staticmethod @staticmethod
def remember(packet_hash, destination_hash, public_key, app_data=None, stamp=None, stamp_value=0, lr_cost=None): def remember(packet_hash, destination_hash, public_key, app_data = None):
if stamp_value == None: stamp_value = 0 if len(public_key) != Identity.KEYSIZE//8:
if len(public_key) != Identity.KEYSIZE//8: raise TypeError(f"Can't remember {RNS.prettyhexrep(destination_hash)}, the public key size of {len(public_key)} is not valid.", RNS.LOG_ERROR) raise TypeError("Can't remember "+RNS.prettyhexrep(destination_hash)+", the public key size of "+str(len(public_key))+" is not valid.", RNS.LOG_ERROR)
else: else:
with Identity.known_destinations_lock: with Identity.known_destinations_lock:
if not destination_hash in Identity.known_destinations: if not destination_hash in Identity.known_destinations:
Identity.known_destinations[destination_hash] = [time.time(), packet_hash, public_key, app_data, 0, stamp, stamp_value, lr_cost] Identity.known_destinations[destination_hash] = [time.time(), packet_hash, public_key, app_data, 0]
else: else:
entry = Identity.known_destinations[destination_hash] entry = Identity.known_destinations[destination_hash]
entry[0] = time.time() entry[0] = time.time()
entry[1] = packet_hash entry[1] = packet_hash
entry[2] = public_key entry[2] = public_key
entry[3] = app_data entry[3] = app_data
entry[5] = stamp
entry[6] = stamp_value
entry[7] = lr_cost
Identity._kd_mark_dirty(destination_hash)
@staticmethod @staticmethod
def recall(target_hash, from_identity_hash=False, _no_use=False): def recall(target_hash, from_identity_hash=False, _no_use=False):
@ -140,15 +127,13 @@ class Identity:
:returns: An :ref:`RNS.Identity<api-identity>` instance that can be used to create an outgoing :ref:`RNS.Destination<api-destination>`, or *None* if the destination is unknown. :returns: An :ref:`RNS.Identity<api-identity>` instance that can be used to create an outgoing :ref:`RNS.Destination<api-destination>`, or *None* if the destination is unknown.
""" """
if from_identity_hash: if from_identity_hash:
with Identity.known_destinations_lock: destination_hashes = list(Identity.known_destinations.keys()) for destination_hash in Identity.known_destinations:
for destination_hash in destination_hashes: if target_hash == Identity.truncated_hash(Identity.known_destinations[destination_hash][2]):
entry = Identity.known_destinations.get(destination_hash)
if not entry: continue
if target_hash == Identity.truncated_hash(entry[2]):
if not _no_use: RNS.Reticulum.get_instance()._used_destination_data(destination_hash) if not _no_use: RNS.Reticulum.get_instance()._used_destination_data(destination_hash)
identity_data = Identity.known_destinations[destination_hash]
identity = Identity(create_keys=False) identity = Identity(create_keys=False)
identity.load_public_key(entry[2]) identity.load_public_key(identity_data[2])
identity.app_data = entry[3] identity.app_data = identity_data[3]
return identity return identity
return None return None
@ -187,39 +172,13 @@ class Identity:
else: return None else: return None
@staticmethod @staticmethod
def recall_stamp_value(destination_hash): def save_known_destinations(background=False, recombine=True):
""" # TODO: Improve the storage method so we don't have to
Recall last heard announce stamp value for a destination hash. # deserialize and serialize the entire table on every
# save, but the only changes. It might be possible to
# simply overwrite on exit now that every local client
# disconnect triggers a data persist.
:param destination_hash: Destination hash as *bytes*.
:returns: *Float*, or *None* if the destination is unknown.
"""
known_destination = Identity.known_destinations.get(destination_hash)
if not known_destination: return None
else: return known_destination[6]
@staticmethod
def recall_link_cost(destination_hash):
"""
Recall the link request stamp cost announced by a destination.
:param destination_hash: Destination hash as *bytes*.
:returns: *Float*, or *None* if the destination is unknown or has not announced a cost.
"""
known_destination = Identity.known_destinations.get(destination_hash)
if not known_destination: return None
else: return known_destination[7]
@staticmethod
def save_known_destinations(background=False, recombine=False):
if recombine: RNS.log(f"Recombining known destinations from disk cache on persist is deprecated, argument ignored", RNS.LOG_WARNING)
if RNS.Transport.owner.is_connected_to_shared_instance: return
if Identity.known_destinations_store != None: Identity.flush_known_destinations()
else: Identity.save_known_destinations_legacy(background=background)
@staticmethod
def save_known_destinations_legacy(background=False):
try: try:
if hasattr(Identity, "saving_known_destinations"): if hasattr(Identity, "saving_known_destinations"):
wait_interval = 0.2 wait_interval = 0.2
@ -233,103 +192,43 @@ class Identity:
Identity.saving_known_destinations = True Identity.saving_known_destinations = True
save_start = time.time() save_start = time.time()
RNS.log("Saving "+str(len(Identity.known_destinations))+" known destinations to storage...", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
temp_file = RNS.Reticulum.storagepath+f"/known_destinations.tmp.{time.time()}" if recombine:
storage_known_destinations = {}
if os.path.isfile(RNS.Reticulum.storagepath+"/known_destinations"):
try: try:
with open(temp_file,"wb") as file: umsgpack.dump(Identity.known_destinations.copy(), file) with open(RNS.Reticulum.storagepath+"/known_destinations","rb") as file:
os.replace(temp_file, RNS.Reticulum.storagepath+f"/known_destinations") storage_known_destinations = umsgpack.load(file)
except: pass
try:
for destination_hash in storage_known_destinations:
if not destination_hash in Identity.known_destinations:
with Identity.known_destinations_lock:
Identity.known_destinations[destination_hash] = storage_known_destinations[destination_hash]
except Exception as e: except Exception as e:
RNS.log(f"Error while serializing and writing known destinations: {e}", RNS.LOG_ERROR) RNS.log("Skipped recombining known destinations from disk, since an error occurred: "+str(e), RNS.LOG_WARNING)
try: os.unlink(temp_file)
except Exception as exc: RNS.log(f"Could not clean up temporary file {temp_file}: {exc}", RNS.LOG_WARNING)
raise e
RNS.log(f"Saved known destinations to storage in {RNS.prettyshorttime(time.time()-save_start)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log("Saving "+str(len(Identity.known_destinations))+" known destinations to storage...", RNS.LOG_VERBOSE)
with open(RNS.Reticulum.storagepath+"/known_destinations","wb") as file:
umsgpack.dump(Identity.known_destinations.copy(), file)
save_time = time.time() - save_start
if save_time < 1: time_str = str(round(save_time*1000,2))+"ms"
else: time_str = str(round(save_time,2))+"s"
RNS.log("Saved known destinations to storage in "+time_str, RNS.LOG_VERBOSE)
except Exception as e: except Exception as e:
RNS.log("Error while saving known destinations to disk, the contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("Error while saving known destinations to disk, the contained exception was: "+str(e), RNS.LOG_ERROR)
RNS.trace_exception(e) RNS.trace_exception(e)
finally: Identity.saving_known_destinations = False Identity.saving_known_destinations = False
@staticmethod @staticmethod
def load_known_destinations(): def load_known_destinations():
if RNS.Reticulum.get_instance().is_connected_to_shared_instance:
try:
store = KnownDestinationsStore(RNS.Reticulum.storagepath+"/known_destinations.db", hash_width=RNS.Reticulum.TRUNCATED_HASHLENGTH//8, read_only=True)
count = 0
with Identity.known_destinations_lock:
for destination_hash, entry in store.load_all():
Identity.known_destinations[destination_hash] = entry
count += 1
store.close()
RNS.log(f"Loaded {count} known destinations from shared instance", RNS.LOG_VERBOSE)
except Exception as e:
try:
RNS.log(f"Could not load known destinations from shared instance database: {e}", RNS.LOG_DEBUG)
RNS.log(f"Potentially running in split mode, attempting to load known destinations over RPC...", RNS.LOG_DEBUG)
known_destinations = RNS.Reticulum.get_instance().get_known_destinations() or {}
with Identity.known_destinations_lock: Identity.known_destinations = known_destinations
RNS.log(f"Loaded {len(Identity.known_destinations)} known destinations from shared instance", RNS.LOG_VERBOSE)
except Exception as e:
with Identity.known_destinations_lock: Identity.known_destinations = {}
RNS.log(f"Could not load known destinations from shared instance: {e}", RNS.LOG_ERROR)
return
store = None
try: store = KnownDestinationsStore(RNS.Reticulum.storagepath+"/known_destinations.db", hash_width=RNS.Reticulum.TRUNCATED_HASHLENGTH//8)
except Exception as e:
RNS.log("Could not initialise known destinations database, falling back to legacy snapshot persistence. The contained exception was: "+str(e), RNS.LOG_ERROR)
Identity.known_destinations_store = None
if store != None:
Identity.known_destinations_store = store
legacy_path = RNS.Reticulum.storagepath+"/known_destinations"
if os.path.isfile(legacy_path):
try:
imported = store.import_legacy(legacy_path)
os.rename(legacy_path, legacy_path+".imported")
RNS.log(f"Imported {imported} known destinations from legacy storage into the known destinations database", RNS.LOG_VERBOSE)
except Exception as e:
RNS.log("Could not import legacy known destinations file into the known destinations database. The contained exception was: "+str(e), RNS.LOG_ERROR)
RNS.trace_exception(e)
st = time.time()
try:
count = 0
with Identity.known_destinations_lock:
for destination_hash, entry in store.load_all():
Identity.known_destinations[destination_hash] = entry
count += 1
RNS.log(f"Loaded {count} known destinations from storage in {RNS.prettyshorttime(time.time()-st)}", RNS.LOG_VERBOSE)
except Exception as e:
RNS.log("Error loading known destinations from storage, continuing with an empty table", RNS.LOG_ERROR)
RNS.trace_exception(e)
ratchetdir = RNS.Reticulum.storagepath+"/ratchets"
if os.path.isdir(ratchetdir) and len(os.listdir(ratchetdir)) > 0:
try:
imported = store.import_legacy_ratchets(ratchetdir, time.time()-Identity.RATCHET_EXPIRY, Identity.known_destinations)
RNS.log(f"Imported {imported} ratchets from legacy file storage", RNS.LOG_VERBOSE)
except Exception as e:
RNS.log(f"Could not import legacy ratchet files into the known destinations database. The contained exception was: {e}", RNS.LOG_ERROR)
RNS.trace_exception(e)
else: Identity.load_known_destinations_legacy()
@staticmethod
def load_known_destinations_legacy():
if os.path.isfile(RNS.Reticulum.storagepath+"/known_destinations"): if os.path.isfile(RNS.Reticulum.storagepath+"/known_destinations"):
st = time.time() st = time.time()
try: try:
@ -341,11 +240,7 @@ class Identity:
if len(known_destination) == RNS.Reticulum.TRUNCATED_HASHLENGTH//8: if len(known_destination) == RNS.Reticulum.TRUNCATED_HASHLENGTH//8:
if len(loaded_known_destinations[known_destination]) < 5: if len(loaded_known_destinations[known_destination]) < 5:
e = loaded_known_destinations[known_destination] e = loaded_known_destinations[known_destination]
loaded_known_destinations[known_destination] = [e[0], e[1], e[2], e[3], 0, None, 0, None] loaded_known_destinations[known_destination] = [e[0], e[1], e[2], e[3], 0]
elif len(loaded_known_destinations[known_destination]) < 8:
e = loaded_known_destinations[known_destination]
while len(e) < 8: e.append(None if len(e) in (5, 7) else 0)
loaded_known_destinations[known_destination] = e
with Identity.known_destinations_lock: with Identity.known_destinations_lock:
Identity.known_destinations[known_destination] = loaded_known_destinations[known_destination] Identity.known_destinations[known_destination] = loaded_known_destinations[known_destination]
@ -364,7 +259,6 @@ class Identity:
if destination_hash in Identity.known_destinations: if destination_hash in Identity.known_destinations:
if not Identity.known_destinations[destination_hash][4] < 0: if not Identity.known_destinations[destination_hash][4] < 0:
Identity.known_destinations[destination_hash][4] = time.time() Identity.known_destinations[destination_hash][4] = time.time()
Identity._kd_mark_dirty(destination_hash)
return True return True
return False return False
@ -374,7 +268,6 @@ class Identity:
with Identity.known_destinations_lock: with Identity.known_destinations_lock:
if destination_hash in Identity.known_destinations: if destination_hash in Identity.known_destinations:
Identity.known_destinations[destination_hash][4] = -1 Identity.known_destinations[destination_hash][4] = -1
Identity._kd_mark_dirty(destination_hash)
return True return True
return False return False
@ -384,7 +277,6 @@ class Identity:
with Identity.known_destinations_lock: with Identity.known_destinations_lock:
if destination_hash in Identity.known_destinations: if destination_hash in Identity.known_destinations:
Identity.known_destinations[destination_hash][4] = time.time() Identity.known_destinations[destination_hash][4] = time.time()
Identity._kd_mark_dirty(destination_hash)
return True return True
return False return False
@ -393,11 +285,8 @@ class Identity:
def _retain_identity(identity_hash): def _retain_identity(identity_hash):
try: try:
retained = False retained = False
with Identity.known_destinations_lock: destination_hashes = list(Identity.known_destinations.keys()) for destination_hash in Identity.known_destinations:
for destination_hash in destination_hashes: if identity_hash == Identity.truncated_hash(Identity.known_destinations[destination_hash][2]):
entry = Identity.known_destinations.get(destination_hash)
if not entry: continue
if identity_hash == Identity.truncated_hash(entry[2]):
if Identity._retain_destination_data(destination_hash): retained = True if Identity._retain_destination_data(destination_hash): retained = True
return retained return retained
@ -405,7 +294,7 @@ class Identity:
except Exception as e: RNS.log(f"Error while retaining identity {RNS.prettyhexrep(identity_hash)}: {e}", RNS.LOG_ERROR) except Exception as e: RNS.log(f"Error while retaining identity {RNS.prettyhexrep(identity_hash)}: {e}", RNS.LOG_ERROR)
@staticmethod @staticmethod
def clean_known_destinations(background=False): def clean_known_destinations():
now = time.time() now = time.time()
st = now st = now
total = len(Identity.known_destinations) total = len(Identity.known_destinations)
@ -413,15 +302,8 @@ class Identity:
no_path = 0 no_path = 0
retained = 0 retained = 0
never_used = 0 never_used = 0
ratchetdir = RNS.Reticulum.storagepath+"/ratchets" for destination_hash in Identity.known_destinations:
RNS.log(f"Cleaning known destinations{' at background priority' if background else ''}...", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
with Identity.known_destinations_lock: destination_hashes = list(Identity.known_destinations.keys())
for destination_hash in destination_hashes:
try: try:
if background: time.sleep(0.001) # Low priority, yield thread
RNS.Transport.destinations_last_cleaned = time.time()
if RNS.Transport.has_path(destination_hash): has_path = True if RNS.Transport.has_path(destination_hash): has_path = True
else: else:
has_path = False has_path = False
@ -450,7 +332,7 @@ class Identity:
if not is_retained and not has_path: if not is_retained and not has_path:
if not was_used and now - last_announce > RNS.Transport.UNUSED_DESTINATION_LINGER: stale.append(destination_hash) if not was_used and now - last_announce > RNS.Transport.UNUSED_DESTINATION_LINGER: stale.append(destination_hash)
elif unused_for > RNS.Transport.PATH_TIMEOUT*1.25: stale.append(destination_hash) elif unused_for > RNS.Transport.DESTINATION_TIMEOUT*1.25: stale.append(destination_hash)
except Exception as e: RNS.log(f"Faulty entry for {RNS.prettyhexrep(destination_hash)} while cleaning known destinations: {e}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None except Exception as e: RNS.log(f"Faulty entry for {RNS.prettyhexrep(destination_hash)} while cleaning known destinations: {e}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
@ -459,26 +341,9 @@ class Identity:
with Identity.known_destinations_lock: with Identity.known_destinations_lock:
if destination_hash in Identity.known_destinations: if destination_hash in Identity.known_destinations:
Identity.known_destinations.pop(destination_hash) Identity.known_destinations.pop(destination_hash)
Identity._kd_mark_deleted(destination_hash)
if destination_hash in Identity.known_ratchets: Identity.known_ratchets.pop(destination_hash)
removed += 1 removed += 1
if not Identity.known_destinations_store: # RNS.log(f"Total destinations: {total}, stale: {len(stale)}, removed: {removed}, no path: {no_path}, never used: {never_used}, with path: {total-no_path}, used: {total-never_used}, retained: {retained}. Completed in {RNS.prettyshorttime(time.time()-st)}", RNS.LOG_WARNING) # TODO: Remove
try:
hexhash = RNS.hexrep(destination_hash, delimit=False)
ratchet_path = f"{ratchetdir}/{hexhash}"
if os.path.isfile(ratchet_path): os.unlink(ratchet_path)
except Exception as e: RNS.log(f"Could not clean stale ratchets for {RNS.prettyhexrep(destination_hash)}: {e}", RNS.LOG_WARNING)
if Identity.known_destinations_store != None:
expired_ratchets = Identity.known_destinations_store.expire_ratchets(time.time()-Identity.RATCHET_EXPIRY)
if len(expired_ratchets) > 0:
for destination_hash in expired_ratchets:
with Identity.known_destinations_lock:
if destination_hash in Identity.known_ratchets: Identity.known_ratchets.pop(destination_hash)
RNS.log(f"Cleaned known destinations in {RNS.prettyshorttime(time.time()-st)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
RNS.log(f"Total: {total}, stale: {len(stale)}, removed: {removed}, no path: {no_path}, never used: {never_used}, with path: {total-no_path}, used: {total-never_used}, retained: {retained}", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
if not RNS.Transport.owner.is_connected_to_shared_instance: Identity.save_known_destinations(recombine=False) if not RNS.Transport.owner.is_connected_to_shared_instance: Identity.save_known_destinations(recombine=False)
@staticmethod @staticmethod
@ -542,21 +407,11 @@ class Identity:
@staticmethod @staticmethod
def _remember_ratchet(destination_hash, ratchet): def _remember_ratchet(destination_hash, ratchet):
try: try:
# Don't re-insert existing ratchet if destination_hash in Identity.known_ratchets and Identity.known_ratchets[destination_hash] == ratchet:
if Identity.known_ratchets.get(destination_hash) == ratchet: return ratchet_exists = True
store = Identity.known_destinations_store
if store != None:
# Persist to store, only update in-memory cache if this
# destination already exists there.
store.remember_ratchet(destination_hash, ratchet, time.time())
if destination_hash in Identity.known_ratchets:
Identity.known_ratchets[destination_hash] = ratchet
# Legacy store: Use file system
# Shared instance client: Just keep in memory cache
else: else:
ratchet_exists = destination_hash in Identity.known_ratchets and Identity.known_ratchets[destination_hash] == ratchet ratchet_exists = False
if not ratchet_exists: if not ratchet_exists:
RNS.log(f"Remembering ratchet {RNS.prettyhexrep(Identity._get_ratchet_id(ratchet))} for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None RNS.log(f"Remembering ratchet {RNS.prettyhexrep(Identity._get_ratchet_id(ratchet))} for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
Identity.known_ratchets[destination_hash] = ratchet Identity.known_ratchets[destination_hash] = ratchet
@ -568,13 +423,16 @@ class Identity:
ratchetdir = RNS.Reticulum.storagepath+"/ratchets" ratchetdir = RNS.Reticulum.storagepath+"/ratchets"
if not os.path.isdir(ratchetdir): os.makedirs(ratchetdir) if not os.path.isdir(ratchetdir):
os.makedirs(ratchetdir)
outpath = f"{ratchetdir}/{hexhash}.out" outpath = f"{ratchetdir}/{hexhash}.out"
finalpath = f"{ratchetdir}/{hexhash}" finalpath = f"{ratchetdir}/{hexhash}"
with open(outpath, "wb") as ratchet_file: ratchet_file.write(umsgpack.packb(ratchet_data)) with open(outpath, "wb") as ratchet_file:
ratchet_file.write(umsgpack.packb(ratchet_data))
os.replace(outpath, finalpath) os.replace(outpath, finalpath)
threading.Thread(target=persist_job, daemon=True).start() threading.Thread(target=persist_job, daemon=True).start()
except Exception as e: except Exception as e:
@ -584,7 +442,6 @@ class Identity:
@staticmethod @staticmethod
def _clean_ratchets(): def _clean_ratchets():
if Identity.known_destinations_store != None: return
RNS.log("Cleaning ratchets...", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log("Cleaning ratchets...", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
try: try:
count = 0 count = 0
@ -624,43 +481,7 @@ class Identity:
@staticmethod @staticmethod
def get_ratchet(destination_hash): def get_ratchet(destination_hash):
if destination_hash in Identity.known_ratchets: return Identity.known_ratchets[destination_hash] if not destination_hash in Identity.known_ratchets:
store = Identity.known_destinations_store
if store != None:
ratchet_data = store.fetch_ratchet(destination_hash)
if ratchet_data != None:
ratchet, received = ratchet_data
if time.time() < received+Identity.RATCHET_EXPIRY:
Identity.known_ratchets[destination_hash] = ratchet
return ratchet
else: return None
else:
RNS.log(f"Could not load ratchet for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return None
elif RNS.Reticulum.get_instance().is_connected_to_shared_instance:
try:
# TODO: Ideally, refactor this to an RPC method instead
store = KnownDestinationsStore(RNS.Reticulum.storagepath+"/known_destinations.db", hash_width=RNS.Reticulum.TRUNCATED_HASHLENGTH//8, read_only=True)
ratchet_data = store.fetch_ratchet(destination_hash)
store.close()
if ratchet_data != None:
ratchet, received = ratchet_data
if time.time() < received+Identity.RATCHET_EXPIRY:
Identity.known_ratchets[destination_hash] = ratchet
return ratchet
except Exception as e: RNS.log(f"Could not fetch ratchet for {RNS.prettyhexrep(destination_hash)} from shared instance database. The contained exception was: {e}", RNS.LOG_ERROR)
return None
else:
if not os.path.isfile(RNS.Reticulum.storagepath+"/ratchets/"+RNS.hexrep(destination_hash, delimit=False)): return None
ratchetdir = RNS.Reticulum.storagepath+"/ratchets" ratchetdir = RNS.Reticulum.storagepath+"/ratchets"
hexhash = RNS.hexrep(destination_hash, delimit=False) hexhash = RNS.hexrep(destination_hash, delimit=False)
ratchet_path = f"{ratchetdir}/{hexhash}" ratchet_path = f"{ratchetdir}/{hexhash}"
@ -670,138 +491,22 @@ class Identity:
ratchet_data = umsgpack.unpackb(ratchet_file.read()) ratchet_data = umsgpack.unpackb(ratchet_file.read())
if time.time() < ratchet_data["received"]+Identity.RATCHET_EXPIRY and len(ratchet_data["ratchet"]) == Identity.RATCHETSIZE//8: if time.time() < ratchet_data["received"]+Identity.RATCHET_EXPIRY and len(ratchet_data["ratchet"]) == Identity.RATCHETSIZE//8:
Identity.known_ratchets[destination_hash] = ratchet_data["ratchet"] Identity.known_ratchets[destination_hash] = ratchet_data["ratchet"]
else: return None else:
return None
except Exception as e: except Exception as e:
RNS.log(f"An error occurred while loading ratchet data for {RNS.prettyhexrep(destination_hash)} from storage.", RNS.LOG_ERROR) RNS.log(f"An error occurred while loading ratchet data for {RNS.prettyhexrep(destination_hash)} from storage.", RNS.LOG_ERROR)
RNS.log(f"The contained exception was: {e}", RNS.LOG_ERROR) RNS.log(f"The contained exception was: {e}", RNS.LOG_ERROR)
return None return None
if destination_hash in Identity.known_ratchets: return Identity.known_ratchets[destination_hash] if destination_hash in Identity.known_ratchets:
return Identity.known_ratchets[destination_hash]
else: else:
RNS.log(f"Could not load ratchet for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log(f"Could not load ratchet for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return None return None
@staticmethod @staticmethod
def _reset_validation_caches(): def validate_announce(packet, only_validate_signature=False):
Identity.announce_valid_cache = {}
Identity.announce_invalid_cache = {}
__valid_cache_count = 0
__valid_cache_max = 384
@staticmethod
def __cache_valid_announce(result):
Identity.__valid_cache_count += 1
result["id"] = Identity.__valid_cache_count
packet_hash = result["packet_hash"]
if packet_hash and not packet_hash in Identity.announce_valid_cache:
Identity.announce_valid_cache[packet_hash] = result
Identity.announce_invalid_cache.pop(packet_hash, None)
try:
while len(Identity.announce_valid_cache) > Identity.__valid_cache_max:
Identity.announce_valid_cache.pop(next(iter(Identity.announce_valid_cache)))
except Exception as e: RNS.log(f"Error while culling announce validation cache: {e}", RNS.LOG_DEBUG)
return True
__invalid_cache_count = 0
__invalid_cache_max = 384
@staticmethod
def __cache_invalid_announce(packet, signal=None, stamp_value=None):
Identity.__invalid_cache_count += 1
result = (signal or False, stamp_value)
if packet.packet_hash: Identity.announce_invalid_cache[packet.packet_hash] = result
try:
while len(Identity.announce_invalid_cache) > Identity.__invalid_cache_max:
Identity.announce_invalid_cache.pop(next(iter(Identity.announce_invalid_cache)))
except Exception as e: RNS.log(f"Error while culling announce invalidation cache: {e}", RNS.LOG_DEBUG)
return result[0]
@staticmethod
def __finalize_av(r, cache_hit=False):
# We are also re-persisting here on cache hit in the edge (but very possible) case
# that a path was culled (for example because of interface teardown), and its
# associated destination data was removed. Known destinations must always cover
# what is in the path table, anything else is state inconsintency.
if not cache_hit or not r["destination_hash"] in Identity.known_destinations:
RNS.Identity.remember(r["packet_hash"], r["destination_hash"], r["public_key"], app_data=r["app_data"],
stamp=r["stamp"], stamp_value=r["stamp_value"], lr_cost=r["lr_cost"])
if r["ratchet"]: Identity._remember_ratchet(r["destination_hash"], r["ratchet"])
valid_announce_loglevel = RNS.LOG_EXTREME
if RNS.sl(valid_announce_loglevel):
if r["rssi"] != None or r["snr"] != None:
signal_str = " ["
if r["rssi"] != None:
signal_str += "RSSI "+str(r["rssi"])+"dBm"
if r["snr"] != None: signal_str += ", "
if r["snr"] != None: signal_str += "SNR "+str(r["snr"])+"dB"
signal_str += "]"
else: signal_str = ""
via_str = f" via {RNS.prettyhexrep(r['transport_id'])}" if r["transport_id"] else ""
val_str = f" (v:{RNS.prettycost(r['stamp_value'])})"
RNS.log(f"Valid announce for {RNS.prettyhexrep(r['destination_hash'])}{val_str} {r['hops']} hops away, received{via_str} on {r['receiving_interface']}{signal_str}", valid_announce_loglevel)
return True
announce_valid_cache = {}
announce_invalid_cache = {}
@staticmethod
def validate_announce(packet, only_validate_signature=False, signal_blackholed=False, signal_stamp=False, required_value=None):
# Check cached announce invalidations
cached_invalidation = Identity.announce_invalid_cache.get(packet.packet_hash, None)
if cached_invalidation:
try:
if cached_invalidation[0] != "invalid_stamp":
if cached_invalidation[0] == "no_stamp": return cached_invalidation[0] if signal_stamp else False
elif cached_invalidation[0] == "blackholed": return cached_invalidation[0] if signal_blackholed else False
else: return False
elif cached_invalidation[1] != None:
rcvd_value = quantize_cost(cached_invalidation[1])
if packet.receiving_interface: cost_target = packet.receiving_interface.destination_cost
else: cost_target = RNS.Transport.DESTINATION_COST
if quantize_cost(cost_target) > quantize_cost(rcvd_value): return "invalid_stamp" if signal_stamp else False
else: pass # Stamp value above target for this interface, allow re-validation
else:
RNS.log(f"Incoherent state in cached announce invalidation, rejecting announce", RNS.LOG_WARNING)
return False
except Exception as e:
RNS.log(f"Error while processing cached invalid announce: {e}", RNS.LOG_DEBUG)
return False
# Check cached announce validations
cached_validation = Identity.announce_valid_cache.get(packet.packet_hash, None)
if cached_validation:
try:
if RNS.Transport.STAMPS_ENABLED and RNS.Transport.STAMPS_ENFORCED:
if required_value == None:
if packet.receiving_interface: cost_target = packet.receiving_interface.destination_cost
else: cost_target = RNS.Transport.DESTINATION_COST
else: cost_target = required_value
cached_stamp_value = cached_validation.get("stamp_value", None)
if cached_stamp_value == None: return "no_stamp" if signal_stamp else False
elif quantize_cost(cost_target) > quantize_cost(cached_stamp_value):
return "invalid_stamp" if signal_stamp else False
except Exception as e:
RNS.log(f"Error while processing cached valid announce: {e}", RNS.LOG_DEBUG)
return False
return Identity.__finalize_av(cached_validation, cache_hit=True)
if required_value == None:
if packet.receiving_interface: required_value = packet.receiving_interface.destination_cost
else: required_value = RNS.Transport.DESTINATION_COST
try: try:
if packet.packet_type == RNS.Packet.ANNOUNCE: if packet.packet_type == RNS.Packet.ANNOUNCE:
keysize = Identity.KEYSIZE//8 keysize = Identity.KEYSIZE//8
@ -813,14 +518,11 @@ class Identity:
# Get public key bytes from announce # Get public key bytes from announce
public_key = packet.data[:keysize] public_key = packet.data[:keysize]
# Shared format structure
ratchet = b""
name_hash = packet.data[keysize:keysize+name_hash_len]
random_hash = packet.data[keysize+name_hash_len:keysize+name_hash_len+10]
# If the packet context flag is set, # If the packet context flag is set,
# this announce contains a new ratchet # this announce contains a new ratchet
if packet.context_flag == RNS.Packet.FLAG_SET: if packet.context_flag == RNS.Packet.FLAG_SET:
name_hash = packet.data[keysize:keysize+name_hash_len ]
random_hash = packet.data[keysize+name_hash_len:keysize+name_hash_len+10]
ratchet = packet.data[keysize+name_hash_len+10:keysize+name_hash_len+10+ratchetsize] ratchet = packet.data[keysize+name_hash_len+10:keysize+name_hash_len+10+ratchetsize]
signature = packet.data[keysize+name_hash_len+10+ratchetsize:keysize+name_hash_len+10+ratchetsize+sig_len] signature = packet.data[keysize+name_hash_len+10+ratchetsize:keysize+name_hash_len+10+ratchetsize+sig_len]
app_data = b"" app_data = b""
@ -830,6 +532,9 @@ class Identity:
# If the packet context flag is not set, # If the packet context flag is not set,
# this announce does not contain a ratchet # this announce does not contain a ratchet
else: else:
ratchet = b""
name_hash = packet.data[keysize:keysize+name_hash_len]
random_hash = packet.data[keysize+name_hash_len:keysize+name_hash_len+10]
signature = packet.data[keysize+name_hash_len+10:keysize+name_hash_len+10+sig_len] signature = packet.data[keysize+name_hash_len+10:keysize+name_hash_len+10+sig_len]
app_data = b"" app_data = b""
if len(packet.data) > keysize+name_hash_len+10+sig_len: if len(packet.data) > keysize+name_hash_len+10+sig_len:
@ -840,80 +545,21 @@ class Identity:
if not len(packet.data) > Identity.KEYSIZE//8+Identity.NAME_HASH_LENGTH//8+10+Identity.SIGLENGTH//8: if not len(packet.data) > Identity.KEYSIZE//8+Identity.NAME_HASH_LENGTH//8+10+Identity.SIGLENGTH//8:
app_data = None app_data = None
stamp = None
dst_stamped = None
dst_value = None
stamp_validated = None
dst_lr_cost = None
if RNS.Transport.STAMPS_ENABLED and app_data and len(app_data) >= Identity.EXTPAYLOADLENGTH:
ext_payload = app_data[-Identity.EXTPAYLOADLENGTH:]
ext_header = ext_payload[:Identity.EXTHEADERLENGTH]
ext_flags = ext_header[1] # Reserved for future extensions
ext_anchor = ext_payload[-1]
# The ext_anchor carries the lr_cost integer part, and a
# reserved flag in the MSB. The fractional part of the
# lr_cost is stored in the immediately preceding byte.
ext_extended = (ext_anchor & 0x80) != 0
ext_lr_cost_i = ext_anchor & 0x7F
ext_lr_cost_m = ext_payload[-2]
cost_bytes = bytes([ext_lr_cost_i, ext_lr_cost_m])
ext_lr_cost = cost_from_bytes(cost_bytes)
known_format = (not ext_extended) and (ext_lr_cost <= RNS.Transport.STAMP_COST_CAP)
if (not ext_header[0] == Identity.EXTPAYLOADMARKER) or (not known_format): dst_stamped = False
else:
# RNS.log(f"Checking potential stamp in announce with app_data len {len(app_data)} for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) # TODO: Remove after 1.6.1
dst_stamped = True
try:
stamp = ext_payload[Identity.EXTHEADERLENGTH:Identity.EXTHEADERLENGTH+Identity.STAMPLENGTH]
material = destination_hash+public_key+name_hash
wb = stamp_workblock(material, expand_rounds=0)
valid = stamp_valid(stamp, required_value, wb)
dst_value = stamp_value(wb, stamp)
if valid:
app_data = app_data[:-Identity.EXTPAYLOADLENGTH]
if not len(app_data): app_data = None
stamp_validated = True
dst_lr_cost = ext_lr_cost
except Exception as e: RNS.log(f"Announce stamp parsing failed: {e}", RNS.LOG_ERROR)
# TODO: Initial debug only, remove ######
# if RNS.sl(RNS.LOG_DEBUG):
# if not dst_stamped: RNS.log(f"No stamp on announce from {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG)
# else:
# if not stamp_validated: RNS.log(f"Invalid or insufficient-value stamp for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG)
# else: RNS.log(f"Valid stamp with value {dst_value} for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG)
#########################################
if RNS.Transport.STAMPS_ENABLED and RNS.Transport.STAMPS_ENFORCED:
if not dst_stamped:
if signal_stamp: return Identity.__cache_invalid_announce(packet, "no_stamp")
else: return Identity.__cache_invalid_announce(packet)
elif not stamp_validated:
if signal_stamp: return Identity.__cache_invalid_announce(packet, "invalid_stamp", dst_value)
else: return Identity.__cache_invalid_announce(packet)
announced_identity = Identity(create_keys=False) announced_identity = Identity(create_keys=False)
announced_identity.load_public_key(public_key) announced_identity.load_public_key(public_key)
if len(RNS.Transport.blackholed_identities) > 0: if len(RNS.Transport.blackholed_identities) > 0:
if announced_identity.hash in RNS.Transport.blackholed_identities: if announced_identity.hash in RNS.Transport.blackholed_identities:
RNS.log(f"Invalidated and dropped announce from blackholed identity {RNS.prettyhexrep(announced_identity.hash)}", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None RNS.log(f"Invalidated and dropped announce from blackholed identity {RNS.prettyhexrep(announced_identity.hash)}", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
if signal_blackholed: return Identity.__cache_invalid_announce(packet, "blackholed") return False
else: return Identity.__cache_invalid_announce(packet)
if announced_identity.pub != None and (packet.announce_signature_validated or announced_identity.validate(signature, signed_data)): if announced_identity.pub != None and announced_identity.validate(signature, signed_data):
packet.announce_signature_validated = True
if only_validate_signature: if only_validate_signature:
del announced_identity del announced_identity
return True return True
hash_material = name_hash+announced_identity.hash hash_material = name_hash+announced_identity.hash
expected_hash = RNS.Identity.full_hash(hash_material)[:RNS.Reticulum.TRUNCATED_HASHLENGTH//8] expected_hash = RNS.Identity.full_hash(hash_material)[:RNS.Reticulum.TRUNCATED_HASHLENGTH//8]
del announced_identity
if destination_hash == expected_hash: if destination_hash == expected_hash:
# Check if we already have a public key for this destination # Check if we already have a public key for this destination
@ -924,63 +570,55 @@ class Identity:
# that someone manages a hash collision, we reject the announce. # that someone manages a hash collision, we reject the announce.
RNS.log("Received announce with valid signature and destination hash, but announced public key does not match already known public key.", RNS.LOG_CRITICAL) RNS.log("Received announce with valid signature and destination hash, but announced public key does not match already known public key.", RNS.LOG_CRITICAL)
RNS.log("This may indicate an attempt to modify network paths, or a random hash collision. The announce was rejected.", RNS.LOG_CRITICAL) RNS.log("This may indicate an attempt to modify network paths, or a random hash collision. The announce was rejected.", RNS.LOG_CRITICAL)
return Identity.__cache_invalid_announce(packet) return False
validation_result = { "packet_hash": packet.packet_hash, "destination_hash": destination_hash, RNS.Identity.remember(packet.get_hash(), destination_hash, public_key, app_data)
"public_key": public_key, "ratchet": ratchet, "app_data": app_data, "stamp": stamp, del announced_identity
"stamp_value": dst_value, "lr_cost": dst_lr_cost, "transport_id": packet.transport_id,
"hops": packet.hops, "receiving_interface": str(packet.receiving_interface),
"rssi": packet.rssi, "snr": packet.snr, "q": packet.q }
Identity.__finalize_av(validation_result) if packet.rssi != None or packet.snr != None:
return Identity.__cache_valid_announce(validation_result) signal_str = " ["
if packet.rssi != None:
signal_str += "RSSI "+str(packet.rssi)+"dBm"
if packet.snr != None:
signal_str += ", "
if packet.snr != None:
signal_str += "SNR "+str(packet.snr)+"dB"
signal_str += "]"
else:
signal_str = ""
if hasattr(packet, "transport_id") and packet.transport_id != None:
RNS.log("Valid announce for "+RNS.prettyhexrep(destination_hash)+" "+str(packet.hops)+" hops away, received via "+RNS.prettyhexrep(packet.transport_id)+" on "+str(packet.receiving_interface)+signal_str, RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
else:
RNS.log("Valid announce for "+RNS.prettyhexrep(destination_hash)+" "+str(packet.hops)+" hops away, received on "+str(packet.receiving_interface)+signal_str, RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
if ratchet:
Identity._remember_ratchet(destination_hash, ratchet)
return True
else: else:
RNS.log("Received invalid announce for "+RNS.prettyhexrep(destination_hash)+": Destination mismatch.", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log("Received invalid announce for "+RNS.prettyhexrep(destination_hash)+": Destination mismatch.", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return Identity.__cache_invalid_announce(packet) return False
else: else:
RNS.log("Received invalid announce for "+RNS.prettyhexrep(destination_hash)+": Invalid signature.", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log("Received invalid announce for "+RNS.prettyhexrep(destination_hash)+": Invalid signature.", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
del announced_identity del announced_identity
return Identity.__cache_invalid_announce(packet) return False
except Exception as e: except Exception as e:
RNS.log("Error occurred while validating announce. The contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("Error occurred while validating announce. The contained exception was: "+str(e), RNS.LOG_ERROR)
RNS.trace_exception(e) # TODO: Remove debug
return False return False
@staticmethod @staticmethod
def persist_data(background=False): def persist_data(background=False):
if hasattr(RNS.Transport, "owner") and not RNS.Transport.owner.is_connected_to_shared_instance: if not RNS.Transport.owner.is_connected_to_shared_instance:
if Identity.known_destinations_store != None: Identity.flush_known_destinations() Identity.save_known_destinations(background=background)
else: Identity.save_known_destinations_legacy(background=background)
@staticmethod @staticmethod
def flush_known_destinations(): def exit_handler():
store = Identity.known_destinations_store Identity.persist_data()
if store == None: return False
with Identity.known_destinations_lock: upserts, deletes, ratchet_upserts, ratchet_deletes = store.take_pending()
try: return store.apply(upserts, deletes, ratchet_upserts, ratchet_deletes, Identity.known_destinations)
except Exception as e:
RNS.log(f"Could not flush known destinations to storage. The contained exception was: {e}", RNS.LOG_ERROR)
return False
@staticmethod
def _kd_mark_dirty(destination_hash):
store = Identity.known_destinations_store
if store != None: store.mark_dirty(destination_hash)
@staticmethod
def _kd_mark_deleted(destination_hash):
store = Identity.known_destinations_store
if store != None:
store.mark_deleted(destination_hash)
store.mark_ratchet_deleted(destination_hash)
@staticmethod
def exit_handler(): Identity.persist_data()
@staticmethod @staticmethod
def from_bytes(prv_bytes): def from_bytes(prv_bytes):
@ -1047,7 +685,7 @@ class Identity:
RNS.log("Error while saving identity to "+str(path), RNS.LOG_ERROR) RNS.log("Error while saving identity to "+str(path), RNS.LOG_ERROR)
RNS.log("The contained exception was: "+str(e)) RNS.log("The contained exception was: "+str(e))
def __init__(self, create_keys=True, value=None): def __init__(self,create_keys=True):
# Initialize keys to none # Initialize keys to none
self.prv = None self.prv = None
self.prv_bytes = None self.prv_bytes = None
@ -1063,43 +701,25 @@ class Identity:
self.hexhash = None self.hexhash = None
if create_keys: if create_keys:
self.create_keys(value=value) self.create_keys()
def create_keys(self, value=None): def create_keys(self):
self.sig_prv = Ed25519PrivateKey.generate()
self.sig_pub = self.sig_prv.public_key()
self.sig_prv_bytes = self.sig_prv.private_bytes()
self.sig_pub_bytes = self.sig_pub.public_bytes()
if not value:
self.prv = X25519PrivateKey.generate() self.prv = X25519PrivateKey.generate()
self.prv_bytes = self.prv.private_bytes() self.prv_bytes = self.prv.private_bytes()
else: self.sig_prv = Ed25519PrivateKey.generate()
stamped_key, v = generate_stamped_key(material=self.sig_pub_bytes, stamp_cost=value) self.sig_prv_bytes = self.sig_prv.private_bytes()
if not stamped_key: raise RuntimeError("Stamped identity key generation failed or was cancelled")
stamped_prv = stamped_key[0]
self.prv = X25519PrivateKey.from_private_bytes(stamped_prv)
self.prv_bytes = self.prv.private_bytes()
assert self.prv_bytes == stamped_prv
assert self.prv.public_key().public_bytes() == stamped_key[1]
self.pub = self.prv.public_key() self.pub = self.prv.public_key()
self.pub_bytes = self.pub.public_bytes() self.pub_bytes = self.pub.public_bytes()
self.sig_pub = self.sig_prv.public_key()
self.sig_pub_bytes = self.sig_pub.public_bytes()
self.update_hashes() self.update_hashes()
RNS.log("Identity keys created for "+RNS.prettyhexrep(self.hash), RNS.LOG_VERBOSE) RNS.log("Identity keys created for "+RNS.prettyhexrep(self.hash), RNS.LOG_VERBOSE)
def get_value(self):
"""
:returns: The stamp value of this identity as a *float*
"""
if not self.get_public_key(): return None
else: return stamp_value(self.sig_pub_bytes, self.pub_bytes)
def get_private_key(self): def get_private_key(self):
""" """
:returns: The private key as *bytes* :returns: The private key as *bytes*
@ -1138,7 +758,9 @@ class Identity:
return True return True
except Exception as e: except Exception as e:
RNS.log(f"Failed to load identity key, the contained exception was: {e}", RNS.LOG_ERROR) raise e
RNS.log("Failed to load identity key", RNS.LOG_ERROR)
RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
return False return False
def load_public_key(self, pub_bytes): def load_public_key(self, pub_bytes):
@ -1156,12 +778,8 @@ class Identity:
self.sig_pub = Ed25519PublicKey.from_public_bytes(self.sig_pub_bytes) self.sig_pub = Ed25519PublicKey.from_public_bytes(self.sig_pub_bytes)
self.update_hashes() self.update_hashes()
return True
except Exception as e: except Exception as e:
RNS.log(f"Error while loading public key, the contained exception was: {e}", RNS.LOG_ERROR) RNS.log("Error while loading public key, the contained exception was: "+str(e), RNS.LOG_ERROR)
return False
def update_hashes(self): def update_hashes(self):
self.hash = Identity.truncated_hash(self.get_public_key()) self.hash = Identity.truncated_hash(self.get_public_key())

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -128,9 +128,7 @@ class AX25KISSInterface(Interface):
self.packet_queue = [] self.packet_queue = []
self.flow_control = flow_control self.flow_control = flow_control
self.shared_medium = True
self.interface_ready = False self.interface_ready = False
self.reconnecting = False
self.flow_control_timeout = 5 self.flow_control_timeout = 5
self.flow_control_locked = time.time() self.flow_control_locked = time.time()
@ -153,18 +151,14 @@ class AX25KISSInterface(Interface):
try: try:
self.open_port() self.open_port()
if self.serial.is_open: self.configure_device()
else: raise IOError("Could not open serial port")
except Exception as e: except Exception as e:
RNS.log(f"Could not open serial port for {self}", RNS.LOG_ERROR) RNS.log("Could not open serial port for interface "+str(self), RNS.LOG_ERROR)
RNS.log(f"The contained exception was: {e}", RNS.LOG_ERROR) raise e
RNS.log(f"Reticulum will attempt to bring up this interface periodically", RNS.LOG_ERROR)
if not self.detached and not self.reconnecting:
thread = threading.Thread(target=self.reconnect_port)
thread.daemon = True
thread.start()
if self.serial.is_open:
self.configure_device()
else:
raise IOError("Could not open serial port")
def open_port(self): def open_port(self):
RNS.log("Opening serial port "+self.port+"...", RNS.LOG_VERBOSE) RNS.log("Opening serial port "+self.port+"...", RNS.LOG_VERBOSE)
@ -182,12 +176,6 @@ class AX25KISSInterface(Interface):
dsrdtr = False, dsrdtr = False,
) )
def detach(self):
self.detached = True
self.online = False
try: self.serial.close()
except Exception as e: RNS.log(f"Could not close port for {self} at detach: {e}", RNS.LOG_ERROR)
def configure_device(self): def configure_device(self):
# Allow time for interface to initialise before config # Allow time for interface to initialise before config
sleep(2.0) sleep(2.0)
@ -266,7 +254,9 @@ class AX25KISSInterface(Interface):
def process_incoming(self, data): def process_incoming(self, data):
if (len(data) > AX25.HEADER_SIZE): self.owner.inbound(data[AX25.HEADER_SIZE:], self) if (len(data) > AX25.HEADER_SIZE):
self.rxb += len(data)
self.owner.inbound(data[AX25.HEADER_SIZE:], self)
def process_outgoing(self,data): def process_outgoing(self,data):
@ -382,28 +372,30 @@ class AX25KISSInterface(Interface):
RNS.log("A serial port error occurred, the contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("A serial port error occurred, the contained exception was: "+str(e), RNS.LOG_ERROR)
RNS.log("The interface "+str(self)+" experienced an unrecoverable error and is now offline.", RNS.LOG_ERROR) RNS.log("The interface "+str(self)+" experienced an unrecoverable error and is now offline.", RNS.LOG_ERROR)
if RNS.Reticulum.panic_on_interface_error: RNS.panic() if RNS.Reticulum.panic_on_interface_error:
RNS.panic()
RNS.log("Reticulum will attempt to reconnect the interface periodically.", RNS.LOG_ERROR) RNS.log("Reticulum will attempt to reconnect the interface periodically.", RNS.LOG_ERROR)
self.online = False self.online = False
try: self.serial.close() self.serial.close()
except Exception as e: RNS.log(f"Could not close port for {self}", RNS.LOG_ERROR) self.reconnect_port()
if not self.detached and not self.reconnecting: self.reconnect_port()
def reconnect_port(self): def reconnect_port(self):
self.reconnecting = True while not self.online:
while not self.online and not self.detached:
try: try:
time.sleep(5) time.sleep(5)
RNS.log("Attempting to reconnect serial port "+str(self.port)+" for "+str(self)+"...", RNS.LOG_VERBOSE) RNS.log("Attempting to reconnect serial port "+str(self.port)+" for "+str(self)+"...", RNS.LOG_VERBOSE)
self.open_port() self.open_port()
if self.serial.is_open: self.configure_device() if self.serial.is_open:
except Exception as e: RNS.log("Error while reconnecting port, the contained exception was: "+str(e), RNS.LOG_ERROR) self.configure_device()
except Exception as e:
RNS.log("Error while reconnecting port, the contained exception was: "+str(e), RNS.LOG_ERROR)
self.reconnecting = False RNS.log("Reconnected serial port for "+str(self))
if self.online: RNS.log("Reconnected serial port for "+str(self))
def should_ingress_limit(self): return False def should_ingress_limit(self):
return False
def __str__(self): return "AX25KISSInterface["+self.name+"]" def __str__(self):
return "AX25KISSInterface["+self.name+"]"

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -128,11 +128,9 @@ class KISSInterface(Interface):
self.beacon_d = beacon_data.encode("utf-8") self.beacon_d = beacon_data.encode("utf-8")
self.first_tx = None self.first_tx = None
self.bitrate = KISSInterface.BITRATE_GUESS self.bitrate = KISSInterface.BITRATE_GUESS
self.reconnecting = False
self.packet_queue = [] self.packet_queue = []
self.flow_control = flow_control self.flow_control = flow_control
self.shared_medium = True
self.interface_ready = False self.interface_ready = False
self.flow_control_timeout = 5 self.flow_control_timeout = 5
self.flow_control_locked = time.time() self.flow_control_locked = time.time()
@ -150,23 +148,15 @@ class KISSInterface(Interface):
try: try:
self.open_port() self.open_port()
if self.serial.is_open: self.configure_device()
else: raise IOError("Could not open serial port")
except Exception as e: except Exception as e:
RNS.log(f"Could not open serial port for {self}", RNS.LOG_ERROR) RNS.log("Could not open serial port "+self.port, RNS.LOG_ERROR)
RNS.log(f"The contained exception was: {e}", RNS.LOG_ERROR) raise e
RNS.log(f"Reticulum will attempt to bring up this interface periodically", RNS.LOG_ERROR)
if not self.detached and not self.reconnecting: if self.serial.is_open:
thread = threading.Thread(target=self.reconnect_port) self.configure_device()
thread.daemon = True else:
thread.start() raise IOError("Could not open serial port")
def detach(self):
self.detached = True
self.online = False
try: self.serial.close()
except Exception as e: RNS.log(f"Could not close port for {self} at detach: {e}", RNS.LOG_ERROR)
def open_port(self): def open_port(self):
RNS.log("Opening serial port "+self.port+"...") RNS.log("Opening serial port "+self.port+"...")
@ -426,23 +416,24 @@ class KISSInterface(Interface):
RNS.log("Reticulum will attempt to reconnect the interface periodically.", RNS.LOG_ERROR) RNS.log("Reticulum will attempt to reconnect the interface periodically.", RNS.LOG_ERROR)
self.online = False self.online = False
try: self.serial.close() self.serial.close()
except Exception as e: RNS.log(f"Could not close port for {self}", RNS.LOG_ERROR) self.reconnect_port()
if not self.detached and not self.reconnecting: self.reconnect_port()
def reconnect_port(self): def reconnect_port(self):
self.reconnecting = True
while not self.online: while not self.online:
try: try:
time.sleep(5) time.sleep(5)
RNS.log("Attempting to reconnect serial port "+str(self.port)+" for "+str(self)+"...", RNS.LOG_VERBOSE) RNS.log("Attempting to reconnect serial port "+str(self.port)+" for "+str(self)+"...", RNS.LOG_VERBOSE)
self.open_port() self.open_port()
if self.serial.is_open: self.configure_device() if self.serial.is_open:
except Exception as e: RNS.log("Error while reconnecting port, the contained exception was: "+str(e), RNS.LOG_ERROR) self.configure_device()
except Exception as e:
RNS.log("Error while reconnecting port, the contained exception was: "+str(e), RNS.LOG_ERROR)
self.reconnecting = False RNS.log("Reconnected serial port for "+str(self))
if self.online: RNS.log("Reconnected serial port for "+str(self))
def should_ingress_limit(self): return False def should_ingress_limit(self):
return False
def __str__(self): return "KISSInterface["+self.name+"]" def __str__(self):
return "KISSInterface["+self.name+"]"

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -282,9 +282,12 @@ class RNodeInterface(Interface):
if (port != None or device_serial != None) and (enable_bluetooth or disable_bluetooth or pairing_mode): if (port != None or device_serial != None) and (enable_bluetooth or disable_bluetooth or pairing_mode):
serial = None serial = None
bluetooth_state = None bluetooth_state = None
if pairing_mode: bluetooth_state = 0x01 if pairing_mode:
elif enable_bluetooth: bluetooth_state = 0x01 bluetooth_state = 0x01
elif disable_bluetooth: bluetooth_state = 0x00 elif enable_bluetooth:
bluetooth_state = 0x01
elif disable_bluetooth:
bluetooth_state = 0x00
if port != None: if port != None:
RNS.log("Opening serial port "+port+"...") RNS.log("Opening serial port "+port+"...")
@ -500,7 +503,6 @@ class RNodeInterface(Interface):
self.r_interference = None self.r_interference = None
self.r_interference_l = None self.r_interference_l = None
self.r_temperature = None self.r_temperature = None
self.reports_phy_stats = True
self.r_battery_state = RNodeInterface.BATTERY_STATE_UNKNOWN self.r_battery_state = RNodeInterface.BATTERY_STATE_UNKNOWN
self.r_battery_percent = 0 self.r_battery_percent = 0
@ -516,7 +518,6 @@ class RNodeInterface(Interface):
self.packet_queue = [] self.packet_queue = []
self.flow_control = flow_control self.flow_control = flow_control
self.shared_medium = True
self.interface_ready = False self.interface_ready = False
self.announce_rate_target = None self.announce_rate_target = None
self.last_port_io = 0 self.last_port_io = 0
@ -1505,11 +1506,11 @@ class RNodeInterface(Interface):
except Exception as e: except Exception as e:
self.online = False self.online = False
if not self.detached:
RNS.log("A serial port error occurred, the contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("A serial port error occurred, the contained exception was: "+str(e), RNS.LOG_ERROR)
RNS.log("The interface "+str(self)+" experienced an unrecoverable error and is now offline.", RNS.LOG_ERROR) RNS.log("The interface "+str(self)+" experienced an unrecoverable error and is now offline.", RNS.LOG_ERROR)
if RNS.Reticulum.panic_on_interface_error: RNS.panic() if RNS.Reticulum.panic_on_interface_error:
RNS.panic()
RNS.log("Reticulum will attempt to reconnect the interface periodically.", RNS.LOG_ERROR) RNS.log("Reticulum will attempt to reconnect the interface periodically.", RNS.LOG_ERROR)
@ -1571,14 +1572,10 @@ class RNodeInterface(Interface):
except Exception as e: except Exception as e:
RNS.log(f"An error occurred while detaching {self}: {e}", RNS.LOG_ERROR) RNS.log(f"An error occurred while detaching {self}: {e}", RNS.LOG_ERROR)
if self.use_ble: if self.use_ble: self.ble.close()
self.ble.close() if self.use_tcp:
self.ble.cleanup()
elif self.use_tcp:
time.sleep(0.5) time.sleep(0.5)
self.tcp.close() self.tcp.close()
else:
self.serial.close()
def should_ingress_limit(self): def should_ingress_limit(self):
return False return False
@ -1657,9 +1654,6 @@ class BLEConnection(BluetoothDispatcher):
self.owner.ble_rx_queue = b"" self.owner.ble_rx_queue = b""
return data return data
def cleanup(self):
self.should_run = False
def close(self): def close(self):
try: try:
if self.connected: if self.connected:
@ -1748,7 +1742,7 @@ class BLEConnection(BluetoothDispatcher):
ble_devices = self.find_target_devices() ble_devices = self.find_target_devices()
if len(ble_devices) > 0: self.ble_device = ble_devices.pop() if len(ble_devices) > 0: self.ble_device = ble_devices.pop()
else: self.ble_device = None else: self.ble_device == None
if self.ble_device != None: if self.ble_device != None:
if self.was_connected: if self.was_connected:

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -95,7 +95,6 @@ class SerialInterface(Interface):
raise ValueError("No port specified for serial interface") raise ValueError("No port specified for serial interface")
self.HW_MTU = 564 self.HW_MTU = 564
self.shared_medium = True
self.pyserial = serial self.pyserial = serial
self.serial = None self.serial = None
@ -109,7 +108,6 @@ class SerialInterface(Interface):
self.timeout = 100 self.timeout = 100
self.online = False self.online = False
self.bitrate = self.speed self.bitrate = self.speed
self.reconnecting = False
if parity.lower() == "e" or parity.lower() == "even": if parity.lower() == "e" or parity.lower() == "even":
self.parity = "E" self.parity = "E"
@ -119,23 +117,15 @@ class SerialInterface(Interface):
try: try:
self.open_port() self.open_port()
if self.serial.is_open: self.configure_device()
else: raise IOError("Could not open serial port")
except Exception as e: except Exception as e:
RNS.log(f"Could not open serial port for {self}", RNS.LOG_ERROR) RNS.log("Could not open serial port for interface "+str(self), RNS.LOG_ERROR)
RNS.log(f"The contained exception was: {e}", RNS.LOG_ERROR) raise e
RNS.log(f"Reticulum will attempt to bring up this interface periodically", RNS.LOG_ERROR)
if not self.detached and not self.reconnecting: if self.serial.is_open:
thread = threading.Thread(target=self.reconnect_port) self.configure_device()
thread.daemon = True else:
thread.start() raise IOError("Could not open serial port")
def detach(self):
self.detached = True
self.online = False
try: self.serial.close()
except Exception as e: RNS.log(f"Could not close port for {self} at detach: {e}", RNS.LOG_ERROR)
def open_port(self): def open_port(self):
RNS.log("Opening serial port "+self.port+"...") RNS.log("Opening serial port "+self.port+"...")
@ -267,23 +257,24 @@ class SerialInterface(Interface):
RNS.log("Reticulum will attempt to reconnect the interface periodically.", RNS.LOG_ERROR) RNS.log("Reticulum will attempt to reconnect the interface periodically.", RNS.LOG_ERROR)
self.online = False self.online = False
try: self.serial.close() self.serial.close()
except Exception as e: RNS.log(f"Could not close port for {self}", RNS.LOG_ERROR) self.reconnect_port()
if not self.detached and not self.reconnecting: self.reconnect_port()
def reconnect_port(self): def reconnect_port(self):
self.reconnecting = True while not self.online:
while not self.online and not self.detached:
try: try:
time.sleep(5) time.sleep(5)
RNS.log("Attempting to reconnect serial port "+str(self.port)+" for "+str(self)+"...", RNS.LOG_VERBOSE) RNS.log("Attempting to reconnect serial port "+str(self.port)+" for "+str(self)+"...", RNS.LOG_VERBOSE)
self.open_port() self.open_port()
if self.serial.is_open: self.configure_device() if self.serial.is_open:
except Exception as e: RNS.log("Error while reconnecting port, the contained exception was: "+str(e), RNS.LOG_ERROR) self.configure_device()
except Exception as e:
RNS.log("Error while reconnecting port, the contained exception was: "+str(e), RNS.LOG_ERROR)
self.reconnecting = False RNS.log("Reconnected serial port for "+str(self))
if self.online: RNS.log("Reconnected serial port for "+str(self))
def should_ingress_limit(self): return False def should_ingress_limit(self):
return False
def __str__(self): return "SerialInterface["+self.name+"]" def __str__(self):
return "SerialInterface["+self.name+"]"

View file

@ -33,7 +33,5 @@ import glob
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py") py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc") pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
so_modules = glob.glob(os.path.dirname(__file__)+"/*.so") modules = py_modules+pyc_modules
pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd") __all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
modules = py_modules+pyc_modules+so_modules+pyd_modules
__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -127,7 +127,6 @@ class AutoInterface(Interface):
self.link_local_addresses = [] self.link_local_addresses = []
self.adopted_interfaces = {} self.adopted_interfaces = {}
self.interface_servers = {} self.interface_servers = {}
self.discovery_sockets = []
self.multicast_echoes = {} self.multicast_echoes = {}
self.initial_echoes = {} self.initial_echoes = {}
self.timed_out_interfaces = {} self.timed_out_interfaces = {}
@ -269,8 +268,6 @@ class AutoInterface(Interface):
addr_info = socket.getaddrinfo(link_local_addr+"%"+ifname, self.unicast_discovery_port, socket.AF_INET6, socket.SOCK_DGRAM) addr_info = socket.getaddrinfo(link_local_addr+"%"+ifname, self.unicast_discovery_port, socket.AF_INET6, socket.SOCK_DGRAM)
unicast_discovery_socket.bind(addr_info[0][4]) unicast_discovery_socket.bind(addr_info[0][4])
self.discovery_sockets.append(unicast_discovery_socket)
mcast_addr = self.mcast_discovery_address mcast_addr = self.mcast_discovery_address
RNS.log(str(self)+" Creating multicast discovery listener on "+str(ifname)+" with address "+str(mcast_addr), RNS.LOG_EXTREME) RNS.log(str(self)+" Creating multicast discovery listener on "+str(ifname)+" with address "+str(mcast_addr), RNS.LOG_EXTREME)
@ -299,8 +296,6 @@ class AutoInterface(Interface):
discovery_socket.bind(addr_info[0][4]) discovery_socket.bind(addr_info[0][4])
self.discovery_sockets.append(discovery_socket)
# Set up thread for multicast discovery packets # Set up thread for multicast discovery packets
def discovery_loop(): self.discovery_handler(discovery_socket, ifname) def discovery_loop(): self.discovery_handler(discovery_socket, ifname)
thread = threading.Thread(target=discovery_loop, daemon=True).start() thread = threading.Thread(target=discovery_loop, daemon=True).start()
@ -363,9 +358,9 @@ class AutoInterface(Interface):
thread.daemon = True thread.daemon = True
thread.start() thread.start()
while not self.detached: while True:
data, ipv6_src = socket.recvfrom(1024) data, ipv6_src = socket.recvfrom(1024)
if self.final_init_done and data and ipv6_src: if self.final_init_done:
peering_hash = data[:RNS.Identity.HASHLENGTH//8] peering_hash = data[:RNS.Identity.HASHLENGTH//8]
expected_hash = RNS.Identity.full_hash(self.group_id+ipv6_src[0].encode("utf-8")) expected_hash = RNS.Identity.full_hash(self.group_id+ipv6_src[0].encode("utf-8"))
if peering_hash == expected_hash: if peering_hash == expected_hash:
@ -374,7 +369,7 @@ class AutoInterface(Interface):
RNS.log(str(self)+" received peering packet on "+str(ifname)+" from "+str(ipv6_src[0])+", but authentication hash was incorrect.", RNS.LOG_DEBUG) RNS.log(str(self)+" received peering packet on "+str(ifname)+" from "+str(ipv6_src[0])+", but authentication hash was incorrect.", RNS.LOG_DEBUG)
def peer_jobs(self): def peer_jobs(self):
while not self.detached: while True:
time.sleep(self.peer_job_interval) time.sleep(self.peer_job_interval)
now = time.time() now = time.time()
timed_out_peers = [] timed_out_peers = []
@ -433,22 +428,14 @@ class AutoInterface(Interface):
if ifname in self.interface_servers: if ifname in self.interface_servers:
RNS.log("Shutting down previous UDP listener for "+str(self)+" "+str(ifname), RNS.LOG_DEBUG) RNS.log("Shutting down previous UDP listener for "+str(self)+" "+str(ifname), RNS.LOG_DEBUG)
previous_server = self.interface_servers[ifname] previous_server = self.interface_servers[ifname]
def shutdown_server(): previous_server.shutdown() def shutdown_server():
previous_server.shutdown()
threading.Thread(target=shutdown_server, daemon=True).start() threading.Thread(target=shutdown_server, daemon=True).start()
RNS.log("Starting new UDP listener for "+str(self)+" "+str(ifname), RNS.LOG_DEBUG) RNS.log("Starting new UDP listener for "+str(self)+" "+str(ifname), RNS.LOG_DEBUG)
retry_delay = 1.25
listener_started = False
while not listener_started:
try:
time.sleep(retry_delay)
udp_server = socketserver.UDPServer(listen_address, self.handler_factory(self.process_incoming)) udp_server = socketserver.UDPServer(listen_address, self.handler_factory(self.process_incoming))
self.interface_servers[ifname] = udp_server self.interface_servers[ifname] = udp_server
listener_started = True
except Exception as e:
RNS.log(f"Could not start new UDP listener for {self} on {listen_address}: {e}", RNS.LOG_WARNING)
RNS.log(f"Retrying in {retry_delay} seconds", RNS.LOG_WARNING)
thread = threading.Thread(target=udp_server.serve_forever) thread = threading.Thread(target=udp_server.serve_forever)
thread.daemon = True thread.daemon = True
@ -483,7 +470,7 @@ class AutoInterface(Interface):
def announce_handler(self, ifname): def announce_handler(self, ifname):
while not self.detached: while True:
self.peer_announce(ifname) self.peer_announce(ifname)
time.sleep(self.announce_interval) time.sleep(self.announce_interval)
@ -558,9 +545,6 @@ class AutoInterface(Interface):
spawned_interface.ic_pr_burst_freq_new = self.ic_pr_burst_freq_new spawned_interface.ic_pr_burst_freq_new = self.ic_pr_burst_freq_new
spawned_interface.ic_pr_burst_freq = self.ic_pr_burst_freq spawned_interface.ic_pr_burst_freq = self.ic_pr_burst_freq
spawned_interface._destination_cost = self._destination_cost
spawned_interface._pr_cost = self._pr_cost
spawned_interface.parent_interface = self spawned_interface.parent_interface = self
spawned_interface.bitrate = self.bitrate spawned_interface.bitrate = self.bitrate
@ -588,11 +572,6 @@ class AutoInterface(Interface):
spawned_interface.announce_rate_grace = self.announce_rate_grace spawned_interface.announce_rate_grace = self.announce_rate_grace
spawned_interface.announce_rate_penalty = self.announce_rate_penalty spawned_interface.announce_rate_penalty = self.announce_rate_penalty
spawned_interface.mode = self.mode spawned_interface.mode = self.mode
spawned_interface.gravity = self.gravity
spawned_interface.recursive_prs = self.recursive_prs
spawned_interface.announces_from_internal = self.announces_from_internal
spawned_interface.announces_to_internal = self.announces_to_internal
spawned_interface.announce_cap = self.announce_cap
spawned_interface.HW_MTU = self.HW_MTU spawned_interface.HW_MTU = self.HW_MTU
spawned_interface.online = True spawned_interface.online = True
RNS.Transport.add_interface(spawned_interface) RNS.Transport.add_interface(spawned_interface)
@ -616,20 +595,7 @@ class AutoInterface(Interface):
def process_outgoing(self, data): pass def process_outgoing(self, data): pass
def detach(self): def detach(self): self.online = False
self.online = False
self.detached = True
RNS.log(f"Shutting down all UDP listeners for {self}", RNS.LOG_NOTICE)
for ifname in self.interface_servers:
try:
self.interface_servers[ifname].server_close()
self.interface_servers[ifname].shutdown()
except Exception as e: RNS.log(f"Could not shut down UDP listener {ifname} for {self}: {e}", RNS.LOG_ERROR)
for s in self.discovery_sockets:
try: s.close()
except Exception as e: RNS.log(f"Error while shutting down discovery socket {s} for {self}: {e}", RNS.LOG_ERROR)
def __str__(self): return f"AutoInterface[{self.name}]" def __str__(self): return f"AutoInterface[{self.name}]"
@ -663,6 +629,8 @@ class AutoInterfacePeer(Interface):
self.owner.refresh_peer(self.addr) self.owner.refresh_peer(self.addr)
self.owner.mif_deque.append(data_hash) self.owner.mif_deque.append(data_hash)
self.owner.mif_deque_times.append([data_hash, time.time()]) self.owner.mif_deque_times.append([data_hash, time.time()])
self.rxb += len(data)
self.owner.rxb += len(data)
self.owner.owner.inbound(data, self) self.owner.owner.inbound(data, self)
def process_outgoing(self, data): def process_outgoing(self, data):

View file

@ -29,8 +29,6 @@
# SOFTWARE. # SOFTWARE.
from RNS.Interfaces.Interface import Interface from RNS.Interfaces.Interface import Interface
from RNS.Interfaces.util.TransmitBuffer import TransmitBuffer
from RNS.Interfaces.util.HDLC import HDLC, ReceiveBuffer
import threading import threading
import socket import socket
import select import select
@ -39,49 +37,29 @@ import sys
import os import os
import RNS import RNS
class HDLC():
FLAG = 0x7E
ESC = 0x7D
ESC_MASK = 0x20
@staticmethod
def escape(data):
data = data.replace(bytes([HDLC.ESC]), bytes([HDLC.ESC, HDLC.ESC^HDLC.ESC_MASK]))
data = data.replace(bytes([HDLC.FLAG]), bytes([HDLC.ESC, HDLC.FLAG^HDLC.ESC_MASK]))
return data
class BackboneInterface(Interface): class BackboneInterface(Interface):
HW_MTU = 1048576 HW_MTU = 1048576
BITRATE_GUESS = 100_000_000 BITRATE_GUESS = 1_000_000_000
DEFAULT_IFAC_SIZE = 16 DEFAULT_IFAC_SIZE = 16
AUTOCONFIGURE_MTU = True AUTOCONFIGURE_MTU = True
BLOCK_FAST_FLAPPING = True
FAST_FLAP_THRESHOLD = 20
FAST_FLAP_GRACE = 5
FAST_FLAP_EXPIRY = 12*60*60
fast_flapping_lock = threading.Lock()
fast_flapping = {}
CLIENT_LIMIT = 500
DP_IC_HIGH_WM_PCT = 90
DP_IC_MID_WM_PCT = 68
DP_IC_LOW_WM_PCT = 10
DP_IC_INTERVAL = 0.250
DP_IC_TRIGGER = 1.5
DP_IC_RCVBUF = 32768
DP_IC_IF_HEADROOM = 32
DP_IC_PENALTY = 1.5
DP_EC_INTERVAL = 1.0
DP_EC_MID_WM = 128*1024
DP_EC_HIGH_WM = 4*1024*1024
DP_EC_STALL_TICKS = 3
DP_EC_MAX_ETA = 10.0
DP_EC_RELEASE_ETA = 5.0 # Gate release hysterisis
DP_EC_DEAD_TIME = 12.0
epoll = None epoll = None
listener_filenos = {} listener_filenos = {}
spawned_interface_filenos = {} spawned_interface_filenos = {}
epoll = None
_job_active = False _job_active = False
_ic_job_active = False
_ec_job_active = False
_job_lock = threading.Lock() _job_lock = threading.Lock()
_ec_job_lock = threading.Lock()
_ic_job_lock = threading.Lock()
_dp_ic_lock = threading.Lock()
_dp_ic_snapshot = 0
@staticmethod @staticmethod
def get_address_for_if(name, bind_port, prefer_ipv6=False): def get_address_for_if(name, bind_port, prefer_ipv6=False):
@ -137,12 +115,7 @@ class BackboneInterface(Interface):
port = int(c["port"]) if "port" in c else None port = int(c["port"]) if "port" in c else None
bindip = c["listen_ip"] if "listen_ip" in c else None bindip = c["listen_ip"] if "listen_ip" in c else None
bindport = int(c["listen_port"]) if "listen_port" in c else None bindport = int(c["listen_port"]) if "listen_port" in c else None
client_limit = int(c["client_limit"]) if "client_limit" in c else None
prefer_ipv6 = c.as_bool("prefer_ipv6") if "prefer_ipv6" in c else False prefer_ipv6 = c.as_bool("prefer_ipv6") if "prefer_ipv6" in c else False
flap_block = c.as_bool("block_fast_flapping") if "block_fast_flapping" in c else BackboneInterface.BLOCK_FAST_FLAPPING
flap_threshold = c.as_float("fast_flapping_threshold") if "fast_flapping_threshold" in c else BackboneInterface.FAST_FLAP_THRESHOLD
flap_grace = c.as_int("fast_flapping_grace") if "fast_flapping_grace" in c else BackboneInterface.FAST_FLAP_GRACE
flap_expiry = c.as_float("fast_flapping_block_time")*60 if "fast_flapping_block_time" in c else BackboneInterface.FAST_FLAP_EXPIRY
if port != None: bindport = port if port != None: bindport = port
@ -155,16 +128,11 @@ class BackboneInterface(Interface):
self.mode = RNS.Interfaces.Interface.Interface.MODE_FULL self.mode = RNS.Interfaces.Interface.Interface.MODE_FULL
self.spawned_interfaces = [] self.spawned_interfaces = []
self.supports_discovery = True self.supports_discovery = True
self.tunnels_allowed = True
self.lossless_medium = True
self.client_limit = client_limit or self.CLIENT_LIMIT
self.block_fast_flapping = flap_block
self.fast_flap_threshold = flap_threshold
self.fast_flap_grace = flap_grace
self.fast_flap_expiry = flap_expiry
if bindport == None: raise SystemError(f"No TCP port configured for interface \"{name}\"") if bindport == None:
else: self.bind_port = bindport raise SystemError(f"No TCP port configured for interface \"{name}\"")
else:
self.bind_port = bindport
bind_address = None bind_address = None
if device != None: if device != None:
@ -188,34 +156,13 @@ class BackboneInterface(Interface):
else: else:
raise SystemError("Insufficient parameters to create listener") raise SystemError("Insufficient parameters to create listener")
__ic_burst_stats_throttle = 0.95
__last_ic_burst_count_check = 0
__last_ic_burst_count_state = 0
@property
def ic_burst_count(self):
if time.time() > self.__last_ic_burst_count_check + self.__ic_burst_stats_throttle:
self.__last_ic_burst_count_state = len([i.ic_burst_active for i in self.spawned_interfaces if i.ic_burst_active])
self.__last_ic_burst_count_check = time.time()
return self.__last_ic_burst_count_state
__last_ic_pr_burst_count_check = 0
__last_ic_pr_burst_count_state = 0
@property
def ic_pr_burst_count(self):
if time.time() > self.__last_ic_pr_burst_count_check + self.__ic_burst_stats_throttle:
self.__last_ic_pr_burst_count_state = len([i.ic_pr_burst_active for i in self.spawned_interfaces if i.ic_pr_burst_active])
self.__last_ic_pr_burst_count_check = time.time()
return self.__last_ic_pr_burst_count_state
__last_ic_burst_check = 0 __last_ic_burst_check = 0
__last_ic_burst_state = False __last_ic_burst_state = False
@property @property
def ic_burst_active(self): def ic_burst_active(self):
if time.time() > self.__last_ic_burst_check + self.__ic_burst_stats_throttle: if time.time() > self.__last_ic_burst_check + 2:
self.__last_ic_burst_state = any(i.ic_burst_active for i in self.spawned_interfaces) self.__last_ic_burst_state = any(i.ic_burst_active for i in self.spawned_interfaces)
self.__last_ic_burst_check = time.time()
return self.__last_ic_burst_state return self.__last_ic_burst_state
@ -226,10 +173,9 @@ class BackboneInterface(Interface):
__ic_burst_activated = 0 __ic_burst_activated = 0
@property @property
def ic_burst_activated(self): def ic_burst_activated(self):
if time.time() > self.__ic_burst_activated_check + self.__ic_burst_stats_throttle: if time.time() > self.__ic_burst_activated_check + 2:
activated = [i.ic_burst_activated for i in self.spawned_interfaces if i.ic_burst_active] activated = [i.ic_burst_activated for i in self.spawned_interfaces if i.ic_burst_active]
if activated: self.__ic_burst_activated = min(activated) if activated: self.__ic_burst_activated = min(activated)
self.__ic_burst_activated_check = time.time()
return self.__ic_burst_activated return self.__ic_burst_activated
@ -241,9 +187,8 @@ class BackboneInterface(Interface):
__last_ic_pr_burst_state = False __last_ic_pr_burst_state = False
@property @property
def ic_pr_burst_active(self): def ic_pr_burst_active(self):
if time.time() > self.__last_ic_pr_burst_check + self.__ic_burst_stats_throttle: if time.time() > self.__last_ic_pr_burst_check + 2:
self.__last_ic_pr_burst_state = any(i.ic_pr_burst_active for i in self.spawned_interfaces) self.__last_ic_pr_burst_state = any(i.ic_pr_burst_active for i in self.spawned_interfaces)
self.__last_ic_pr_burst_check = time.time()
return self.__last_ic_pr_burst_state return self.__last_ic_pr_burst_state
@ -254,10 +199,9 @@ class BackboneInterface(Interface):
__ic_pr_burst_activated = 0 __ic_pr_burst_activated = 0
@property @property
def ic_pr_burst_activated(self): def ic_pr_burst_activated(self):
if time.time() > self.__ic_pr_burst_activated_check + self.__ic_burst_stats_throttle: if time.time() > self.__ic_pr_burst_activated_check + 2:
activated = [i.ic_pr_burst_activated for i in self.spawned_interfaces if i.ic_pr_burst_active] activated = [i.ic_pr_burst_activated for i in self.spawned_interfaces if i.ic_pr_burst_active]
if activated: self.__ic_pr_burst_activated = min(activated) if activated: self.__ic_pr_burst_activated = min(activated)
self.__ic_pr_burst_activated_check = time.time()
return self.__ic_pr_burst_activated return self.__ic_pr_burst_activated
@ -267,8 +211,6 @@ class BackboneInterface(Interface):
@staticmethod @staticmethod
def start(): def start():
if not BackboneInterface._job_active: threading.Thread(target=BackboneInterface.__job, daemon=True).start() if not BackboneInterface._job_active: threading.Thread(target=BackboneInterface.__job, daemon=True).start()
if not BackboneInterface._ic_job_active: threading.Thread(target=BackboneInterface.__dp_ic_job, daemon=True).start()
if not BackboneInterface._ec_job_active: threading.Thread(target=BackboneInterface.__dp_ec_job, daemon=True).start()
@staticmethod @staticmethod
def ensure_epoll(): def ensure_epoll():
@ -290,7 +232,7 @@ class BackboneInterface(Interface):
server_socket.bind(bind_address) server_socket.bind(bind_address)
else: raise TypeError(f"Invalid socket type {socket_type} for {interface}") else: raise TypeError(f"Invalid socket type {socket_type} for {interface}")
server_socket.listen(16) server_socket.listen(1)
server_socket.setblocking(0) server_socket.setblocking(0)
BackboneInterface.listener_filenos[server_socket.fileno()] = (interface, server_socket) BackboneInterface.listener_filenos[server_socket.fileno()] = (interface, server_socket)
BackboneInterface.epoll.register(server_socket.fileno(), select.EPOLLIN) BackboneInterface.epoll.register(server_socket.fileno(), select.EPOLLIN)
@ -309,9 +251,9 @@ class BackboneInterface(Interface):
RNS.log(f"Attempt to register invalid file descriptor {fileno}", RNS.LOG_WARNING) RNS.log(f"Attempt to register invalid file descriptor {fileno}", RNS.LOG_WARNING)
return return
try: BackboneInterface.epoll.register(fileno, select.EPOLLIN | select.EPOLLHUP) try: BackboneInterface.epoll.register(fileno, select.EPOLLIN)
except Exception as e: except Exception as e:
RNS.log(f"An error occurred while registering EPOLLIN for file descriptor {fileno}: {e}", RNS.LOG_WARNING) RNS.log(f"An error occurred while registering EPOLL_IN for file descriptor {fileno}: {e}", RNS.LOG_WARNING)
@staticmethod @staticmethod
def deregister_fileno(fileno): def deregister_fileno(fileno):
@ -321,9 +263,7 @@ class BackboneInterface(Interface):
try: BackboneInterface.epoll.unregister(fileno) try: BackboneInterface.epoll.unregister(fileno)
except Exception as e: except Exception as e:
if str(e).endswith("No such file or directory"): pass RNS.log(f"An error occurred while deregistering file descriptor {fileno}: {e}", RNS.LOG_DEBUG)
elif str(e).endswith("Bad file descriptor"): pass
else: RNS.log(f"An error occurred while deregistering file descriptor {fileno}: {e}", RNS.LOG_DEBUG)
@staticmethod @staticmethod
def deregister_listeners(): def deregister_listeners():
@ -340,232 +280,11 @@ class BackboneInterface(Interface):
if interface.socket: if interface.socket:
fileno = interface.socket.fileno() fileno = interface.socket.fileno()
if fileno in BackboneInterface.spawned_interface_filenos: if fileno in BackboneInterface.spawned_interface_filenos:
try: try: BackboneInterface.epoll.modify(fileno, select.EPOLLOUT)
events = select.EPOLLOUT | select.EPOLLHUP
if not interface.dp_ingress_gated: events |= select.EPOLLIN
BackboneInterface.epoll.modify(fileno, events)
except Exception as e: except Exception as e:
if str(e).endswith("No such file or directory"): pass RNS.log(f"Error occurred on {interface} while modifying socket EPOLL state: {e}", RNS.LOG_WARNING)
elif str(e).endswith("Bad file descriptor"): pass
else: RNS.log(f"Error occurred on {interface} while modifying socket EPOLL state: {e}", RNS.LOG_WARNING)
raise e raise e
@staticmethod
def __throttle_ingress(interface, hold=None):
if interface.dp_ingress_gated: return False
try:
fileno = interface.socket.fileno()
if fileno in BackboneInterface.spawned_interface_filenos:
interface.dp_ingress_gated = True
interface.dp_ingress_tcount += 1
events = select.EPOLLHUP
if interface.transmit_buffer.sendable > 0: events |= select.EPOLLOUT
BackboneInterface.epoll.modify(fileno, events)
if hold: interface.dp_ingress_hold = time.time() + hold
RNS.log(f"Ingress throttled on {interface}", RNS.LOG_NOTICE) if RNS.sl(RNS.LOG_NOTICE) else None
return True
except Exception as e: RNS.log(f"Error throttling {interface}: {e}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return False
@staticmethod
def __release_ingress(interface):
if not interface.dp_ingress_gated: return False
try:
fileno = interface.socket.fileno()
if fileno in BackboneInterface.spawned_interface_filenos:
events = select.EPOLLIN | select.EPOLLHUP
if interface.transmit_buffer.sendable > 0: events |= select.EPOLLOUT
BackboneInterface.epoll.modify(fileno, events)
interface.dp_ingress_gated = False
RNS.log(f"Released ingress throttle on {interface}", RNS.LOG_NOTICE) if RNS.sl(RNS.LOG_NOTICE) else None
return True
except Exception as e: RNS.log(f"Error releasing throttle on {interface}: {e}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return False
@staticmethod
def _throttle_immediate(q_depth):
with BackboneInterface._dp_ic_lock:
st = time.time()
interfaces = list(BackboneInterface.spawned_interface_filenos.values())
producers = [iface for iface in interfaces if iface.dp_ingress_packets > 0]
if producers:
producers.sort(key=lambda p: p.dp_ingress_packets, reverse=True)
selected = producers[0]
now = time.time()
span = now - BackboneInterface._dp_ic_snapshot
total = sum(iface.dp_ingress_packets for iface in producers)
avail = sum(iface.dp_ingress_bytes for iface in interfaces) / span
share = selected.dp_ingress_packets / total
rate = selected.dp_ingress_packets / span
speed = selected.dp_ingress_bytes / span
data = selected.dp_ingress_bytes
pkts = selected.dp_ingress_packets
alloc = 1/((max(len(interfaces), BackboneInterface.DP_IC_IF_HEADROOM))*BackboneInterface.DP_IC_PENALTY)
hold = (avail/(avail*alloc))*span
if BackboneInterface.__throttle_ingress(selected, hold=hold):
taken = time.time()-st
if RNS.sl(RNS.LOG_DEBUG):
RNS.log(f"Throttled producer with {pkts} packets at {RNS.prettysize(rate, suffix='pps')} / {RNS.prettyspeed(speed*8)}, {round(share*100.0, 2)}% ingress share", RNS.LOG_DEBUG)
RNS.log(f"Available ingress budget {RNS.prettyspeed(avail*8)}, hard-allocated {round(alloc*100.0, 2)}% ({RNS.prettyspeed(avail*alloc*8)}), handled in {RNS.prettyshorttime(taken, compact=True, tight=True)}", RNS.LOG_DEBUG)
RNS.log(f"Holding for {RNS.prettyshorttime(hold, compact=True, tight=True)}, throttling handled in {RNS.prettyshorttime(taken, compact=True, tight=True)} at depth {q_depth}", RNS.LOG_DEBUG)
@staticmethod
def _dp_ec_reset(interface):
interface._dp_ec_prev_sent = 0
interface._dp_ec_zero_ticks = 0
interface._dp_ec_last_drain = time.time()
interface.transmit_buffer = TransmitBuffer()
interface.tx_stalled = False
@staticmethod
def _dp_ec_evaluate(interface, now):
tb = interface.transmit_buffer
drained = tb._tx_sent - interface._dp_ec_prev_sent
interface._dp_ec_prev_sent = tb._tx_sent
sendable = tb.sendable
buffered = len(tb)
if buffered == 0 or sendable == 0:
interface._dp_ec_zero_ticks = 0
interface._dp_ec_last_drain = now
interface.tx_stalled = False
return False
if now - interface._dp_ec_last_drain >= BackboneInterface.DP_EC_DEAD_TIME:
RNS.log(f"No egress control drain progress for {RNS.prettyshorttime(BackboneInterface.DP_EC_DEAD_TIME, compact=True)} on {interface}, tearing down", RNS.LOG_NOTICE)
try:
if hasattr(interface, "socket") and interface.socket:
fileno = interface.socket.fileno()
BackboneInterface.deregister_fileno(fileno)
if fileno in BackboneInterface.spawned_interface_filenos: BackboneInterface.spawned_interface_filenos.pop(fileno)
try: interface.socket.close()
except Exception as e: RNS.log(f"Egress control could not close socket for {interface}: {e}", RNS.LOG_ERROR)
except Exception as e: RNS.log(f"Egress control cleanup error for {interface}: {e}", RNS.LOG_ERROR)
BackboneInterface._dp_ec_reset(interface)
interface.receive(b"")
return True
if drained > 0:
interface._dp_ec_last_drain = now
interface._dp_ec_zero_ticks = 0
drain_rate = drained / BackboneInterface.DP_EC_INTERVAL
clear_eta = buffered / drain_rate if drain_rate > 0 else float("inf")
if buffered > BackboneInterface.DP_EC_MID_WM and clear_eta > BackboneInterface.DP_EC_MAX_ETA:
if not interface.tx_stalled: RNS.log(f"Egress control drain ETA of {RNS.prettyshorttime(clear_eta, compact=True)} exceeds maximum on {interface}, gating outbound", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
interface.tx_stalled = True
elif clear_eta < BackboneInterface.DP_EC_RELEASE_ETA or buffered <= BackboneInterface.DP_EC_MID_WM:
if interface.tx_stalled: RNS.log(f"Egress control drain recovered on {interface}, resuming outbound", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
interface.tx_stalled = False
else:
# No drain progress on this tick
if buffered > BackboneInterface.DP_EC_MID_WM:
interface._dp_ec_zero_ticks += 1
if interface._dp_ec_zero_ticks >= BackboneInterface.DP_EC_STALL_TICKS:
if not interface.tx_stalled: RNS.log(f"No egress control drain progress on {interface}, gating outbound", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
interface.tx_stalled = True
else:
interface._dp_ec_zero_ticks = 0
interface.tx_stalled = False
return False
@staticmethod
def __dp_ec_job():
with BackboneInterface._ec_job_lock:
if BackboneInterface._ec_job_active: return
else:
BackboneInterface._ec_job_active = True
RNS.log(f"Started dataplane egress control", RNS.LOG_DEBUG)
try:
while True:
time.sleep(BackboneInterface.DP_EC_INTERVAL)
now = time.time()
try: interfaces = list(BackboneInterface.spawned_interface_filenos.values())
except RuntimeError as e:
RNS.log(f"Deferring egress control evaluation due to error: {e}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
continue
for interface in interfaces:
if interface.detached: continue
if isinstance(interface, RNS.Interfaces.LocalInterface.LocalClientInterface): continue
BackboneInterface._dp_ec_evaluate(interface, now)
except Exception as e:
RNS.log(f"BackboneInterface egress control error: {e}", RNS.LOG_ERROR)
RNS.trace_exception(e)
@staticmethod
def __dp_ic_job():
with BackboneInterface._ic_job_lock:
if BackboneInterface._ic_job_active: return
else:
BackboneInterface._ic_job_active = True
RNS.log(f"Started dataplane ingress control: High/mid/low water marks are {BackboneInterface.DP_IC_HIGH_WM}/{BackboneInterface.DP_IC_MID_WM}/{BackboneInterface.DP_IC_LOW_WM} packets", RNS.LOG_DEBUG)
try:
while True:
time.sleep(BackboneInterface.DP_IC_INTERVAL)
try:
if RNS.Transport.inbound_queues != None: q_depth = RNS.Transport.inbound_queues.qsize(RNS.Transport.TC_DATA)
else: continue
except Exception: continue
with BackboneInterface._dp_ic_lock:
st = time.time()
interfaces = list(BackboneInterface.spawned_interface_filenos.values())
if q_depth > BackboneInterface.DP_IC_MID_WM:
producers = [iface for iface in interfaces if iface.dp_ingress_packets > 0 and not isinstance(iface, RNS.Interfaces.LocalInterface.LocalClientInterface)]
if producers:
producers.sort(key=lambda p: p.dp_ingress_packets, reverse=True)
selected = producers[0]
now = time.time()
span = max(now - BackboneInterface._dp_ic_snapshot, 0.001)
total = sum(iface.dp_ingress_packets for iface in producers)
avail = sum(iface.dp_ingress_bytes for iface in interfaces) / span
mean = total / max(len(producers), 2)
share = selected.dp_ingress_packets / total
rate = selected.dp_ingress_packets / span
speed = selected.dp_ingress_bytes / span
data = selected.dp_ingress_bytes
pkts = selected.dp_ingress_packets
alloc = 1/((max(len(interfaces), BackboneInterface.DP_IC_IF_HEADROOM))*BackboneInterface.DP_IC_PENALTY)
hold = (avail/max(avail*alloc, 1))*span
if BackboneInterface.__throttle_ingress(selected, hold=hold):
taken = time.time()-st
if RNS.sl(RNS.LOG_DEBUG):
RNS.log(f"Throttled producer with {pkts} packets at {RNS.prettysize(rate, suffix='pps')} / {RNS.prettyspeed(speed*8)}, {round(share*100.0, 2)}% ingress share", RNS.LOG_DEBUG)
RNS.log(f"Available ingress budget {RNS.prettyspeed(avail*8)}, hard-allocated {round(alloc*100.0, 2)}% ({RNS.prettyspeed(avail*alloc*8)}), handled in {RNS.prettyshorttime(taken, compact=True, tight=True)}", RNS.LOG_DEBUG)
RNS.log(f"Holding for {RNS.prettyshorttime(hold, compact=True, tight=True)}, throttling handled in {RNS.prettyshorttime(taken, compact=True, tight=True)}", RNS.LOG_DEBUG)
elif q_depth < BackboneInterface.DP_IC_LOW_WM:
now = time.time()
for iface in interfaces:
if iface.dp_ingress_gated:
if iface.dp_ingress_hold and now < iface.dp_ingress_hold: continue
BackboneInterface.__release_ingress(iface)
break
BackboneInterface._dp_ic_snapshot = time.time()
for iface in interfaces:
iface.dp_ingress_bytes = 0
iface.dp_ingress_packets = 0
except Exception as e:
RNS.log(f"BackboneInterface ingress control error: {e}", RNS.LOG_ERROR)
RNS.trace_exception(e)
@staticmethod @staticmethod
def __job(): def __job():
with BackboneInterface._job_lock: with BackboneInterface._job_lock:
@ -575,21 +294,18 @@ class BackboneInterface(Interface):
BackboneInterface.ensure_epoll() BackboneInterface.ensure_epoll()
try: try:
while True: while True:
events = BackboneInterface.epoll.poll(1)
for fileno, event in BackboneInterface.epoll.poll(1): for fileno, event in BackboneInterface.epoll.poll(1):
if fileno in BackboneInterface.spawned_interface_filenos: if fileno in BackboneInterface.spawned_interface_filenos:
spawned_interface = BackboneInterface.spawned_interface_filenos[fileno] spawned_interface = BackboneInterface.spawned_interface_filenos[fileno]
client_socket = spawned_interface.socket client_socket = spawned_interface.socket
socket_valid = client_socket and fileno == client_socket.fileno() if client_socket and fileno == client_socket.fileno() and (event & select.EPOLLIN):
if socket_valid and (event & select.EPOLLIN) and not spawned_interface.dp_ingress_gated:
if spawned_interface.dp_ingress_gated: continue
try: received_bytes = client_socket.recv(spawned_interface.HW_MTU) try: received_bytes = client_socket.recv(spawned_interface.HW_MTU)
except Exception as e: except Exception as e:
RNS.log(f"Error while reading from {spawned_interface}: {e}", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None RNS.log(f"Error while reading from {spawned_interface}: {e}", RNS.LOG_DEBUG)
received_bytes = b"" received_bytes = b""
if len(received_bytes): if len(received_bytes): spawned_interface.receive(received_bytes)
spawned_interface.dp_ingress_bytes += len(received_bytes)
spawned_interface.receive(received_bytes)
else: else:
BackboneInterface.deregister_fileno(fileno); client_socket.close() BackboneInterface.deregister_fileno(fileno); client_socket.close()
try: try:
@ -605,18 +321,11 @@ class BackboneInterface(Interface):
spawned_interface.receive(received_bytes) spawned_interface.receive(received_bytes)
socket_valid_after_read = socket_valid and fileno in BackboneInterface.spawned_interface_filenos elif client_socket and fileno == client_socket.fileno() and (event & select.EPOLLOUT):
if socket_valid_after_read and (event & select.EPOLLOUT): try: written = client_socket.send(spawned_interface.transmit_buffer)
try: written = spawned_interface.transmit_buffer.drain_to(client_socket)
except Exception as e: except Exception as e:
written = 0 written = 0
if not spawned_interface.detached: if not spawned_interface.detached: RNS.log(f"Error while writing to {spawned_interface}: {e}", RNS.LOG_DEBUG)
if RNS.sl(RNS.LOG_DEBUG):
if str(e).endswith("Connection timed out"): pass
elif str(e).endswith("Connection reset by peer"): pass
elif str(e).endswith("No route to host"): pass
elif str(e).endswith("Broken pipe"): pass
else: RNS.log(f"Error while writing to {spawned_interface}: {e}", RNS.LOG_DEBUG)
BackboneInterface.deregister_fileno(fileno) BackboneInterface.deregister_fileno(fileno)
try: try:
@ -634,17 +343,16 @@ class BackboneInterface(Interface):
except Exception as e: RNS.log(f"Error while closing socket for {spawned_interface}: {e}", RNS.LOG_WARNING) except Exception as e: RNS.log(f"Error while closing socket for {spawned_interface}: {e}", RNS.LOG_WARNING)
spawned_interface.receive(b"") spawned_interface.receive(b"")
spawned_interface.transmit_buffer = spawned_interface.transmit_buffer[written:]
try: try:
if spawned_interface.transmit_buffer.sendable == 0: if len(spawned_interface.transmit_buffer) == 0: BackboneInterface.epoll.modify(fileno, select.EPOLLIN)
events = select.EPOLLHUP except Exception as e:
if not spawned_interface.dp_ingress_gated: events |= select.EPOLLIN RNS.log(f"Error while setting EPOLLIN on {spawned_interface}: {e}", RNS.LOG_ERROR)
BackboneInterface.epoll.modify(fileno, events)
except Exception as e: RNS.log(f"Error while setting EPOLLIN on {spawned_interface}: {e}", RNS.LOG_ERROR)
spawned_interface.txb += written spawned_interface.txb += written
if spawned_interface.parent_interface: spawned_interface.parent_interface.txb += written if spawned_interface.parent_interface: spawned_interface.parent_interface.txb += written
elif socket_valid_after_read and (event & select.EPOLLHUP): elif client_socket and fileno == client_socket.fileno() and event & (select.EPOLLHUP):
BackboneInterface.deregister_fileno(fileno) BackboneInterface.deregister_fileno(fileno)
try: try:
if fileno in BackboneInterface.spawned_interface_filenos: BackboneInterface.spawned_interface_filenos.pop(fileno) if fileno in BackboneInterface.spawned_interface_filenos: BackboneInterface.spawned_interface_filenos.pop(fileno)
@ -667,12 +375,11 @@ class BackboneInterface(Interface):
try: try:
client_socket, address = server_socket.accept() client_socket, address = server_socket.accept()
client_socket.setblocking(0) client_socket.setblocking(0)
client_socket.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, BackboneInterface.DP_IC_RCVBUF)
if not owner_interface.incoming_connection(client_socket): if not owner_interface.incoming_connection(client_socket):
try: client_socket.close() try: client_socket.close()
except Exception as e: RNS.log(f"Error while closing socket for failed incoming connection: {e}", RNS.LOG_WARNING) except Exception as e: RNS.log(f"Error while closing socket for failed incoming connection: {e}", RNS.LOG_WARNING)
except Exception as e: except:
RNS.log(f"Accepting socket failed for incoming connection: {e}", RNS.LOG_WARNING) RNS.log(f"Accepting socket failed for incoming connection: {e}", RNS.LOG_WARNING)
try: client_socket.close() try: client_socket.close()
except Exception as e: RNS.log(f"Error while closing socket for failed incoming socket accept: {e}", RNS.LOG_WARNING) except Exception as e: RNS.log(f"Error while closing socket for failed incoming socket accept: {e}", RNS.LOG_WARNING)
@ -692,24 +399,8 @@ class BackboneInterface(Interface):
BackboneInterface.deregister_listeners() BackboneInterface.deregister_listeners()
def incoming_connection(self, socket): def incoming_connection(self, socket):
RNS.log("Accepting incoming connection", RNS.LOG_VERBOSE)
try: try:
if len(self.spawned_interfaces) >= self.client_limit: return False
remote_ip = socket.getpeername()[0]
remote_port = str(socket.getpeername()[1])
if self.blocked_ip_count > 0:
with BackboneInterface.fast_flapping_lock:
if remote_ip in BackboneInterface.fast_flapping:
now = time.time()
ffe = BackboneInterface.fast_flapping[remote_ip]
started_flapping = ffe[0]
last_flap = ffe[1]
flaps = ffe[2]
if flaps > self.fast_flap_grace:
RNS.log(f"Ignoring incoming connection from fast-flapping IP {remote_ip}", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
return False
RNS.log("Accepting incoming connection", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
spawned_configuration = {"name": "Client on "+self.name, "target_host": None, "target_port": None} spawned_configuration = {"name": "Client on "+self.name, "target_host": None, "target_port": None}
spawned_interface = BackboneClientInterface(self.owner, spawned_configuration, connected_socket=socket) spawned_interface = BackboneClientInterface(self.owner, spawned_configuration, connected_socket=socket)
spawned_interface.OUT = self.OUT spawned_interface.OUT = self.OUT
@ -729,12 +420,9 @@ class BackboneInterface(Interface):
spawned_interface.ic_pr_burst_freq_new = self.ic_pr_burst_freq_new spawned_interface.ic_pr_burst_freq_new = self.ic_pr_burst_freq_new
spawned_interface.ic_pr_burst_freq = self.ic_pr_burst_freq spawned_interface.ic_pr_burst_freq = self.ic_pr_burst_freq
spawned_interface._destination_cost = self._destination_cost
spawned_interface._pr_cost = self._pr_cost
spawned_interface.socket = socket spawned_interface.socket = socket
spawned_interface.target_ip = remote_ip spawned_interface.target_ip = socket.getpeername()[0]
spawned_interface.target_port = remote_port spawned_interface.target_port = str(socket.getpeername()[1])
spawned_interface.parent_interface = self spawned_interface.parent_interface = self
spawned_interface.bitrate = self.bitrate spawned_interface.bitrate = self.bitrate
spawned_interface.optimise_mtu() spawned_interface.optimise_mtu()
@ -744,12 +432,18 @@ class BackboneInterface(Interface):
spawned_interface.ifac_netkey = self.ifac_netkey spawned_interface.ifac_netkey = self.ifac_netkey
if spawned_interface.ifac_netname != None or spawned_interface.ifac_netkey != None: if spawned_interface.ifac_netname != None or spawned_interface.ifac_netkey != None:
ifac_origin = b"" ifac_origin = b""
if spawned_interface.ifac_netname != None: ifac_origin += RNS.Identity.full_hash(spawned_interface.ifac_netname.encode("utf-8")) if spawned_interface.ifac_netname != None:
if spawned_interface.ifac_netkey != None: ifac_origin += RNS.Identity.full_hash(spawned_interface.ifac_netkey.encode("utf-8")) ifac_origin += RNS.Identity.full_hash(spawned_interface.ifac_netname.encode("utf-8"))
if spawned_interface.ifac_netkey != None:
ifac_origin += RNS.Identity.full_hash(spawned_interface.ifac_netkey.encode("utf-8"))
ifac_origin_hash = RNS.Identity.full_hash(ifac_origin) ifac_origin_hash = RNS.Identity.full_hash(ifac_origin)
spawned_interface.ifac_key = RNS.Cryptography.hkdf(length=64, derive_from=ifac_origin_hash, spawned_interface.ifac_key = RNS.Cryptography.hkdf(
salt=RNS.Reticulum.IFAC_SALT, context=None) length=64,
derive_from=ifac_origin_hash,
salt=RNS.Reticulum.IFAC_SALT,
context=None
)
spawned_interface.ifac_identity = RNS.Identity.from_bytes(spawned_interface.ifac_key) spawned_interface.ifac_identity = RNS.Identity.from_bytes(spawned_interface.ifac_key)
spawned_interface.ifac_signature = spawned_interface.ifac_identity.sign(RNS.Identity.full_hash(spawned_interface.ifac_key)) spawned_interface.ifac_signature = spawned_interface.ifac_identity.sign(RNS.Identity.full_hash(spawned_interface.ifac_key))
@ -757,20 +451,13 @@ class BackboneInterface(Interface):
spawned_interface.announce_rate_grace = self.announce_rate_grace spawned_interface.announce_rate_grace = self.announce_rate_grace
spawned_interface.announce_rate_penalty = self.announce_rate_penalty spawned_interface.announce_rate_penalty = self.announce_rate_penalty
spawned_interface.mode = self.mode spawned_interface.mode = self.mode
spawned_interface.gravity = self.gravity
spawned_interface.hop_limiter = self.hop_limiter
spawned_interface.recursive_prs = self.recursive_prs
spawned_interface.announces_from_internal = self.announces_from_internal
spawned_interface.announces_to_internal = self.announces_to_internal
spawned_interface.announce_cap = self.announce_cap
spawned_interface.HW_MTU = self.HW_MTU spawned_interface.HW_MTU = self.HW_MTU
RNS.log("Spawned new BackboneClient Interface: "+str(spawned_interface), RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None spawned_interface.online = True
RNS.log("Spawned new BackboneClient Interface: "+str(spawned_interface), RNS.LOG_VERBOSE)
RNS.Transport.add_interface(spawned_interface) RNS.Transport.add_interface(spawned_interface)
while spawned_interface in self.spawned_interfaces: self.spawned_interfaces.remove(spawned_interface) while spawned_interface in self.spawned_interfaces: self.spawned_interfaces.remove(spawned_interface)
self.spawned_interfaces.append(spawned_interface) self.spawned_interfaces.append(spawned_interface)
BackboneInterface.add_client_socket(socket, spawned_interface) BackboneInterface.add_client_socket(socket, spawned_interface)
spawned_interface.spawned_at = time.time()
spawned_interface.online = True
except Exception as e: except Exception as e:
RNS.log(f"An error occurred while accepting incoming connection on {self}: {e}", RNS.LOG_ERROR) RNS.log(f"An error occurred while accepting incoming connection on {self}: {e}", RNS.LOG_ERROR)
@ -778,25 +465,17 @@ class BackboneInterface(Interface):
return True return True
def received_announce(self, size=0, from_spawned=False): def received_announce(self, from_spawned=False):
if from_spawned: if from_spawned: self.ia_freq_deque.append(time.time())
self.ia_freq_deque.append(time.time())
self.arxb += size
def sent_announce(self, size=0, from_spawned=False): def sent_announce(self, from_spawned=False):
if from_spawned: if from_spawned: self.oa_freq_deque.append(time.time())
self.oa_freq_deque.append(time.time())
self.atxb += size
def received_path_request(self, size=0, from_spawned=False): def received_path_request(self, from_spawned=False):
if from_spawned: if from_spawned: self.ip_freq_deque.append(time.time())
self.ip_freq_deque.append(time.time())
self.prxb += size
def sent_path_request(self, size=0, from_spawned=False): def sent_path_request(self, from_spawned=False):
if from_spawned: if from_spawned: self.op_freq_deque.append(time.time())
self.op_freq_deque.append(time.time())
self.ptxb += size
def process_outgoing(self, data): def process_outgoing(self, data):
pass pass
@ -813,42 +492,14 @@ class BackboneInterface(Interface):
try: listener_socket.shutdown(socket.SHUT_RDWR) try: listener_socket.shutdown(socket.SHUT_RDWR)
except Exception as e: except Exception as e:
if str(e).endswith("Transport endpoint is not connected"): pass if str(e).endswith("Transport endpoint is not connected"): pass
elif str(e).endswith("Bad file descriptor"): pass
else: RNS.log("Error while shutting down socket for "+str(self)+": "+str(e), RNS.LOG_ERROR) else: RNS.log("Error while shutting down socket for "+str(self)+": "+str(e), RNS.LOG_ERROR)
@property
def blocked_ip_list(self):
if not self.block_fast_flapping: return []
else: return [ip for ip in self.fast_flapping if self.fast_flapping[ip][2] > self.fast_flap_grace]
@property
def blocked_ip_count(self):
if not self.block_fast_flapping: return 0
else:
count = 0
expired = []
now = time.time()
with BackboneInterface.fast_flapping_lock:
for remote_ip in BackboneInterface.fast_flapping:
ffe = BackboneInterface.fast_flapping[remote_ip]
started_flapping = ffe[0]
last_flap = ffe[1]
flaps = ffe[2]
if now - last_flap > self.fast_flap_expiry: expired.append(remote_ip)
elif flaps > self.fast_flap_grace: count += 1
for remote_ip in expired:
if remote_ip in BackboneInterface.fast_flapping:
try:
BackboneInterface.fast_flapping.pop(remote_ip)
RNS.log(f"Fast-flapping block expired for {remote_ip}", RNS.LOG_DEBUG)
except Exception as e: RNS.log(f"Error while expiring fast-flapping block for {remote_ip}: {e}", RNS.LOG_ERROR)
return count
def __str__(self): def __str__(self):
if ":" in self.bind_ip: ip_str = f"[{self.bind_ip}]" if ":" in self.bind_ip:
else: ip_str = f"{self.bind_ip}" ip_str = f"[{self.bind_ip}]"
else:
ip_str = f"{self.bind_ip}"
return "BackboneInterface["+self.name+"/"+ip_str+":"+str(self.bind_port)+"]" return "BackboneInterface["+self.name+"/"+ip_str+":"+str(self.bind_port)+"]"
@ -886,7 +537,6 @@ class BackboneClientInterface(Interface):
self.OUT = False self.OUT = False
self.socket = None self.socket = None
self.parent_interface = None self.parent_interface = None
self.tunnels_allowed = True
self.name = name self.name = name
self.initiator = False self.initiator = False
self.reconnecting = False self.reconnecting = False
@ -894,15 +544,12 @@ class BackboneClientInterface(Interface):
self.owner = owner self.owner = owner
self.online = False self.online = False
self.detached = False self.detached = False
self.lossless_medium = True
self.prefer_ipv6 = prefer_ipv6 self.prefer_ipv6 = prefer_ipv6
self.i2p_tunneled = i2p_tunneled self.i2p_tunneled = i2p_tunneled
self.mode = RNS.Interfaces.Interface.Interface.MODE_FULL self.mode = RNS.Interfaces.Interface.Interface.MODE_FULL
self.bitrate = BackboneClientInterface.BITRATE_GUESS self.bitrate = BackboneClientInterface.BITRATE_GUESS
self.transmit_buffer = TransmitBuffer() self.frame_buffer = b""
self.receive_buffer = ReceiveBuffer(mtu=lambda: self.HW_MTU, min_frame_len=RNS.Reticulum.HEADER_MINSIZE, self.transmit_buffer = b""
max_frame_len=lambda: self.HW_MTU+(getattr(self, "ifac_size", None) or 0),
on_frame=self.process_incoming, on_invalid=self.invalid_frame)
if max_reconnect_tries == None: if max_reconnect_tries == None:
self.max_reconnect_tries = BackboneClientInterface.RECONNECT_MAX_TRIES self.max_reconnect_tries = BackboneClientInterface.RECONNECT_MAX_TRIES
@ -924,11 +571,13 @@ class BackboneClientInterface(Interface):
self.target_port = target_port self.target_port = target_port
self.initiator = True self.initiator = True
if connect_timeout != None: self.connect_timeout = connect_timeout if connect_timeout != None:
else: self.connect_timeout = BackboneClientInterface.INITIAL_CONNECT_TIMEOUT self.connect_timeout = connect_timeout
else:
self.connect_timeout = BackboneClientInterface.INITIAL_CONNECT_TIMEOUT
def final_init(self): if BackboneClientInterface.SYNCHRONOUS_START:
if BackboneClientInterface.SYNCHRONOUS_START: self.initial_connect() self.initial_connect()
else: else:
thread = threading.Thread(target=self.initial_connect) thread = threading.Thread(target=self.initial_connect)
thread.daemon = True thread.daemon = True
@ -939,6 +588,8 @@ class BackboneClientInterface(Interface):
thread = threading.Thread(target=self.reconnect) thread = threading.Thread(target=self.reconnect)
thread.daemon = True thread.daemon = True
thread.start() thread.start()
else:
self.wants_tunnel = True
def set_timeouts_linux(self): def set_timeouts_linux(self):
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_USER_TIMEOUT, int(BackboneClientInterface.TCP_USER_TIMEOUT * 1000)) self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_USER_TIMEOUT, int(BackboneClientInterface.TCP_USER_TIMEOUT * 1000))
@ -949,15 +600,15 @@ class BackboneClientInterface(Interface):
def detach(self): def detach(self):
self.online = False self.online = False
self.detached = True
if self.socket != None: if self.socket != None:
if hasattr(self.socket, "close"): if hasattr(self.socket, "close"):
if callable(self.socket.close): if callable(self.socket.close):
self.detached = True
try: try:
if self.socket != None: self.socket.shutdown(socket.SHUT_RDWR) if self.socket != None: self.socket.shutdown(socket.SHUT_RDWR)
except Exception as e: except Exception as e:
if str(e).endswith("Transport endpoint is not connected"): pass if str(e).endswith("Transport endpoint is not connected"): pass
elif str(e).endswith("Bad file descriptor"): pass
else: RNS.log("Error while shutting down socket for "+str(self)+": "+str(e), RNS.LOG_ERROR) else: RNS.log("Error while shutting down socket for "+str(self)+": "+str(e), RNS.LOG_ERROR)
try: try:
@ -966,17 +617,6 @@ class BackboneClientInterface(Interface):
self.socket = None self.socket = None
def early_synthesis(self, packet):
if not self.detached:
if not packet.packed: packet.pack()
if self.ifac_identity: raw = RNS.Transport.handle_outgoing_ifac(self, packet.raw);
else: raw = packet.raw
frame = HDLC.frame(raw)
if not self.transmit_buffer.append(frame, self.tx_hwm):
raise IOError("Transmit buffer overflow at early tunnel synthesis")
BackboneInterface.tx_ready(self)
def connect(self, initial=False): def connect(self, initial=False):
try: try:
if initial: if initial:
@ -997,14 +637,9 @@ class BackboneClientInterface(Interface):
self.socket.settimeout(BackboneClientInterface.INITIAL_CONNECT_TIMEOUT) self.socket.settimeout(BackboneClientInterface.INITIAL_CONNECT_TIMEOUT)
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.socket.connect(target_address) self.socket.connect(target_address)
# TODO: Check missing setblocking(0)
self.socket.settimeout(None) self.socket.settimeout(None)
self.wants_tunnel = True
self.remote_transport = None
self.synthesis_received = False
BackboneInterface.add_client_socket(self.socket, self) BackboneInterface.add_client_socket(self.socket, self)
RNS.Transport.synthesize_tunnel(self)
self.online = True self.online = True
if initial: if initial:
@ -1029,7 +664,6 @@ class BackboneClientInterface(Interface):
def reconnect(self): def reconnect(self):
if self.initiator: if self.initiator:
if not self.reconnecting: if not self.reconnecting:
BackboneInterface._dp_ec_reset(self)
self.reconnecting = True self.reconnecting = True
attempts = 0 attempts = 0
while not self.online and not self.detached: while not self.online and not self.detached:
@ -1051,6 +685,7 @@ class BackboneClientInterface(Interface):
RNS.log("Reconnected socket for "+str(self)+".", RNS.LOG_INFO) RNS.log("Reconnected socket for "+str(self)+".", RNS.LOG_INFO)
self.reconnecting = False self.reconnecting = False
RNS.Transport.synthesize_tunnel(self)
else: else:
RNS.log("Attempt to reconnect on a non-initiator TCP interface. This should not happen.", RNS.LOG_ERROR) RNS.log("Attempt to reconnect on a non-initiator TCP interface. This should not happen.", RNS.LOG_ERROR)
@ -1058,30 +693,44 @@ class BackboneClientInterface(Interface):
def process_incoming(self, data): def process_incoming(self, data):
if self.online and not self.detached: if self.online and not self.detached:
self.dp_ingress_packets += 1 self.rxb += len(data)
if hasattr(self, "parent_interface") and self.parent_interface != None:
self.parent_interface.rxb += len(data)
self.owner.inbound(data, self) self.owner.inbound(data, self)
def process_outgoing(self, data): def process_outgoing(self, data):
if self.online and not self.detached: if self.online and not self.detached:
try: try:
frame = HDLC.frame(data) self.transmit_buffer += bytes([HDLC.FLAG])+HDLC.escape(data)+bytes([HDLC.FLAG])
if self.tx_stalled or not self.transmit_buffer.append(frame, self.tx_hwm): BackboneInterface.tx_ready(self)
self.tx_drops += 1
self.tx_dropped_bytes += len(frame)
RNS.log(f"Egress control dropping outbound frame of {RNS.prettysize(len(frame))} on {self}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
else: BackboneInterface.tx_ready(self)
except Exception as e: except Exception as e:
RNS.log("Exception occurred while transmitting via "+str(self)+", tearing down interface", RNS.LOG_ERROR) RNS.log("Exception occurred while transmitting via "+str(self)+", tearing down interface", RNS.LOG_ERROR)
RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
self.teardown() self.teardown()
def invalid_frame(self, frame_len):
RNS.log(f"Invalid HDLC frame of {RNS.prettysize(frame_len)} received on {self}, dropping frame", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
def receive(self, data_in): def receive(self, data_in):
try: try:
if len(data_in) > 0: self.receive_buffer.feed(data_in) if len(data_in) > 0:
self.frame_buffer += data_in
flags_remaining = True
while flags_remaining:
frame_start = self.frame_buffer.find(HDLC.FLAG)
if frame_start != -1:
frame_end = self.frame_buffer.find(HDLC.FLAG, frame_start+1)
if frame_end != -1:
frame = self.frame_buffer[frame_start+1:frame_end]
frame = frame.replace(bytes([HDLC.ESC, HDLC.FLAG ^ HDLC.ESC_MASK]), bytes([HDLC.FLAG]))
frame = frame.replace(bytes([HDLC.ESC, HDLC.ESC ^ HDLC.ESC_MASK]), bytes([HDLC.ESC]))
if len(frame) > RNS.Reticulum.HEADER_MINSIZE:
self.process_incoming(frame)
self.frame_buffer = self.frame_buffer[frame_end:]
else:
flags_remaining = False
else:
flags_remaining = False
else: else:
self.online = False self.online = False
if self.initiator and not self.detached: if self.initiator and not self.detached:
@ -1089,7 +738,7 @@ class BackboneClientInterface(Interface):
def job(): self.reconnect() def job(): self.reconnect()
threading.Thread(target=job, daemon=True).start() threading.Thread(target=job, daemon=True).start()
else: else:
RNS.log("The socket for remote client "+str(self)+" was closed.", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None RNS.log("The socket for remote client "+str(self)+" was closed.", RNS.LOG_DEBUG)
self.teardown() self.teardown()
except Exception as e: except Exception as e:
@ -1100,34 +749,17 @@ class BackboneClientInterface(Interface):
RNS.log("Attempting to reconnect...", RNS.LOG_WARNING) RNS.log("Attempting to reconnect...", RNS.LOG_WARNING)
def job(): self.reconnect() def job(): self.reconnect()
threading.Thread(target=job, daemon=True).start() threading.Thread(target=job, daemon=True).start()
else: self.teardown() else:
self.teardown()
def teardown(self): def teardown(self):
if self.initiator and not self.detached: if self.initiator and not self.detached:
RNS.log("The interface "+str(self)+" experienced an unrecoverable error and is being torn down. Restart Reticulum to attempt to open this interface again.", RNS.LOG_ERROR) RNS.log("The interface "+str(self)+" experienced an unrecoverable error and is being torn down. Restart Reticulum to attempt to open this interface again.", RNS.LOG_ERROR)
if RNS.Reticulum.panic_on_interface_error: RNS.panic() if RNS.Reticulum.panic_on_interface_error:
RNS.panic()
else: else:
RNS.log("The interface "+str(self)+" is being torn down.", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None RNS.log("The interface "+str(self)+" is being torn down.", RNS.LOG_VERBOSE)
if self.parent_interface and self.parent_interface.block_fast_flapping and hasattr(self, "spawned_at"):
connected_time = time.time() - self.spawned_at
if connected_time < self.parent_interface.fast_flap_threshold:
try:
now = time.time()
remote_id = f"{self.target_ip}"
with BackboneInterface.fast_flapping_lock:
if remote_id in BackboneInterface.fast_flapping: ffe = BackboneInterface.fast_flapping[remote_id]
else: ffe = [now, now, 0]
ffe[1] = now
ffe[2] += 1
with BackboneInterface.fast_flapping_lock: BackboneInterface.fast_flapping[remote_id] = ffe
dt = now-ffe[0]; fff = ffe[2]/dt if dt > 0 else None
freq_str = f" at {RNS.prettyfrequency(fff)}" if fff else ""
RNS.log(f"{self} is fast flapping{freq_str}, connection time was {RNS.prettytime(connected_time)}, {ffe[2]} fast flaps", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
if ffe[2] > self.parent_interface.fast_flap_grace: RNS.log(f"Ignoring further connections from {remote_id} due to fast-flapping", RNS.LOG_WARNING)
except Exception as e: RNS.log(f"Error while updating fast-flapping interface statistics: {e}", RNS.LOG_ERROR)
self.online = False self.online = False
self.OUT = False self.OUT = False

View file

@ -88,7 +88,8 @@ class I2PController:
self.ready = False self.ready = False
self.storagepath = rns_storagepath+"/i2p" self.storagepath = rns_storagepath+"/i2p"
if not os.path.isdir(self.storagepath): os.makedirs(self.storagepath) if not os.path.isdir(self.storagepath):
os.makedirs(self.storagepath)
def start(self): def start(self):
@ -109,22 +110,28 @@ class I2PController:
except Exception as e: except Exception as e:
self.ready = False self.ready = False
RNS.log("Exception on event loop for "+str(self)+": "+str(e), RNS.LOG_ERROR) RNS.log("Exception on event loop for "+str(self)+": "+str(e), RNS.LOG_ERROR)
finally: self.loop.close() finally:
self.loop.close()
def stop(self): def stop(self):
for i2ptunnel in self.i2plib_tunnels: for i2ptunnel in self.i2plib_tunnels:
if hasattr(i2ptunnel, "stop") and callable(i2ptunnel.stop): i2ptunnel.stop() if hasattr(i2ptunnel, "stop") and callable(i2ptunnel.stop):
i2ptunnel.stop()
if hasattr(asyncio.Task, "all_tasks") and callable(asyncio.Task.all_tasks): if hasattr(asyncio.Task, "all_tasks") and callable(asyncio.Task.all_tasks):
for task in asyncio.Task.all_tasks(loop=self.loop): task.cancel() for task in asyncio.Task.all_tasks(loop=self.loop):
task.cancel()
time.sleep(0.2) time.sleep(0.2)
self.loop.stop() self.loop.stop()
def get_free_port(self): def get_free_port(self):
return self.i2plib.utils.get_free_port() return self.i2plib.utils.get_free_port()
def stop_tunnel(self, i2ptunnel): def stop_tunnel(self, i2ptunnel):
if hasattr(i2ptunnel, "stop") and callable(i2ptunnel.stop): if hasattr(i2ptunnel, "stop") and callable(i2ptunnel.stop):
i2ptunnel.stop() i2ptunnel.stop()
@ -152,9 +159,10 @@ class I2PController:
tn = self.i2plib_tunnels[i2p_destination] tn = self.i2plib_tunnels[i2p_destination]
if tn != None and hasattr(tn, "status"): if tn != None and hasattr(tn, "status"):
RNS.log("Waiting for status from I2P control process", RNS.LOG_DEBUG) RNS.log("Waiting for status from I2P control process", RNS.LOG_EXTREME)
while not tn.status["setup_ran"]: time.sleep(0.1) while not tn.status["setup_ran"]:
RNS.log("Got status from I2P control process", RNS.LOG_DEBUG) time.sleep(0.1)
RNS.log("Got status from I2P control process", RNS.LOG_EXTREME)
if tn.status["setup_failed"]: if tn.status["setup_failed"]:
self.stop_tunnel(tn) self.stop_tunnel(tn)
@ -164,19 +172,29 @@ class I2PController:
if owner.socket != None: if owner.socket != None:
if hasattr(owner.socket, "close"): if hasattr(owner.socket, "close"):
if callable(owner.socket.close): if callable(owner.socket.close):
try: owner.socket.shutdown(socket.SHUT_RDWR) try:
except Exception as e: RNS.log("Error while shutting down socket for "+str(owner)+": "+str(e)) owner.socket.shutdown(socket.SHUT_RDWR)
try: owner.socket.close() except Exception as e:
except Exception as e: RNS.log("Error while closing socket for "+str(owner)+": "+str(e)) RNS.log("Error while shutting down socket for "+str(owner)+": "+str(e))
try:
owner.socket.close()
except Exception as e:
RNS.log("Error while closing socket for "+str(owner)+": "+str(e))
self.client_tunnels[i2p_destination] = True self.client_tunnels[i2p_destination] = True
owner.awaiting_i2p_tunnel = False owner.awaiting_i2p_tunnel = False
RNS.log(str(owner)+" tunnel setup complete", RNS.LOG_VERBOSE) RNS.log(str(owner)+" tunnel setup complete", RNS.LOG_VERBOSE)
else: raise IOError("Got no status response from SAM API") else:
raise IOError("Got no status response from SAM API")
except ConnectionRefusedError as e:
raise e
except ConnectionAbortedError as e:
raise e
except ConnectionRefusedError as e: raise e
except ConnectionAbortedError as e: raise e
except Exception as e: except Exception as e:
RNS.log("Unexpected error type from I2P SAM: "+str(e), RNS.LOG_ERROR) RNS.log("Unexpected error type from I2P SAM: "+str(e), RNS.LOG_ERROR)
raise e raise e
@ -188,33 +206,54 @@ class I2PController:
if i2ptunnel.status["setup_ran"] == False: if i2ptunnel.status["setup_ran"] == False:
RNS.log(str(self)+" I2P tunnel setup did not complete", RNS.LOG_ERROR) RNS.log(str(self)+" I2P tunnel setup did not complete", RNS.LOG_ERROR)
self.stop_tunnel(i2ptunnel) self.stop_tunnel(i2ptunnel)
return False return False
elif i2p_exception != None: elif i2p_exception != None:
RNS.log("An error ocurred while setting up I2P tunnel to "+str(i2p_destination), RNS.LOG_ERROR) RNS.log("An error ocurred while setting up I2P tunnel to "+str(i2p_destination), RNS.LOG_ERROR)
if isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.CantReachPeer): RNS.log("The I2P daemon can't reach peer "+str(i2p_destination), RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.DuplicatedDest): RNS.log("The I2P daemon reported that the destination is already in use", RNS.LOG_ERROR) if isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.CantReachPeer):
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.DuplicatedId): RNS.log("The I2P daemon reported that the ID is arleady in use", RNS.LOG_ERROR) RNS.log("The I2P daemon can't reach peer "+str(i2p_destination), RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.InvalidId): RNS.log("The I2P daemon reported that the stream session ID doesn't exist", RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.InvalidKey): RNS.log("The I2P daemon reported that the key for "+str(i2p_destination)+" is invalid", RNS.LOG_ERROR) elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.DuplicatedDest):
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.KeyNotFound): RNS.log("The I2P daemon could not find the key for "+str(i2p_destination), RNS.LOG_ERROR) RNS.log("The I2P daemon reported that the destination is already in use", RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.PeerNotFound): RNS.log("The I2P daemon could not find the peer "+str(i2p_destination), RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.I2PError): RNS.log("The I2P daemon experienced an unspecified error", RNS.LOG_ERROR) elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.DuplicatedId):
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.Timeout): RNS.log("I2P daemon timed out while setting up client tunnel to "+str(i2p_destination), RNS.LOG_ERROR) RNS.log("The I2P daemon reported that the ID is arleady in use", RNS.LOG_ERROR)
else: RNS.log(f"Unspecified I2P daemon error: {i2p_exception}", RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.InvalidId):
RNS.log("The I2P daemon reported that the stream session ID doesn't exist", RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.InvalidKey):
RNS.log("The I2P daemon reported that the key for "+str(i2p_destination)+" is invalid", RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.KeyNotFound):
RNS.log("The I2P daemon could not find the key for "+str(i2p_destination), RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.PeerNotFound):
RNS.log("The I2P daemon mould not find the peer "+str(i2p_destination), RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.I2PError):
RNS.log("The I2P daemon experienced an unspecified error", RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.Timeout):
RNS.log("I2P daemon timed out while setting up client tunnel to "+str(i2p_destination), RNS.LOG_ERROR)
RNS.log("Resetting I2P tunnel and retrying later", RNS.LOG_ERROR) RNS.log("Resetting I2P tunnel and retrying later", RNS.LOG_ERROR)
self.stop_tunnel(i2ptunnel) self.stop_tunnel(i2ptunnel)
return False return False
elif i2ptunnel.status["setup_failed"] == True: elif i2ptunnel.status["setup_failed"] == True:
RNS.log(str(self)+" Unspecified I2P tunnel setup error, resetting I2P tunnel", RNS.LOG_ERROR) RNS.log(str(self)+" Unspecified I2P tunnel setup error, resetting I2P tunnel", RNS.LOG_ERROR)
self.stop_tunnel(i2ptunnel) self.stop_tunnel(i2ptunnel)
return False return False
else: else:
RNS.log(str(self)+" Got no status from SAM API, resetting I2P tunnel", RNS.LOG_ERROR) RNS.log(str(self)+" Got no status from SAM API, resetting I2P tunnel", RNS.LOG_ERROR)
self.stop_tunnel(i2ptunnel) self.stop_tunnel(i2ptunnel)
return False return False
@ -235,8 +274,10 @@ class I2PController:
i2p_keyfile_nf = self.storagepath+"/"+RNS.hexrep(i2p_dest_hash_nf, delimit=False)+".i2p" i2p_keyfile_nf = self.storagepath+"/"+RNS.hexrep(i2p_dest_hash_nf, delimit=False)+".i2p"
# Use old format if a key is already present # Use old format if a key is already present
if os.path.isfile(i2p_keyfile_of): i2p_keyfile = i2p_keyfile_of if os.path.isfile(i2p_keyfile_of):
else: i2p_keyfile = i2p_keyfile_nf i2p_keyfile = i2p_keyfile_of
else:
i2p_keyfile = i2p_keyfile_nf
i2p_dest = None i2p_dest = None
if not os.path.isfile(i2p_keyfile): if not os.path.isfile(i2p_keyfile):
@ -271,7 +312,8 @@ class I2PController:
asyncio.run_coroutine_threadsafe(tunnel_up(), self.loop).result() asyncio.run_coroutine_threadsafe(tunnel_up(), self.loop).result()
self.server_tunnels[i2p_b32] = True self.server_tunnels[i2p_b32] = True
except Exception as e: raise e except Exception as e:
raise e
else: else:
i2ptunnel = self.i2plib_tunnels[i2p_b32] i2ptunnel = self.i2plib_tunnels[i2p_b32]
@ -280,45 +322,66 @@ class I2PController:
if i2ptunnel.status["setup_ran"] == False: if i2ptunnel.status["setup_ran"] == False:
RNS.log(str(self)+" I2P tunnel setup did not complete", RNS.LOG_ERROR) RNS.log(str(self)+" I2P tunnel setup did not complete", RNS.LOG_ERROR)
self.stop_tunnel(i2ptunnel) self.stop_tunnel(i2ptunnel)
return False return False
elif i2p_exception != None: elif i2p_exception != None:
RNS.log("An error ocurred while setting up I2P tunnel", RNS.LOG_ERROR) RNS.log("An error ocurred while setting up I2P tunnel", RNS.LOG_ERROR)
if isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.CantReachPeer): RNS.log("The I2P daemon can't reach peer "+str(i2p_destination), RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.DuplicatedDest): RNS.log("The I2P daemon reported that the destination is already in use", RNS.LOG_ERROR) if isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.CantReachPeer):
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.DuplicatedId): RNS.log("The I2P daemon reported that the ID is arleady in use", RNS.LOG_ERROR) RNS.log("The I2P daemon can't reach peer "+str(i2p_destination), RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.InvalidId): RNS.log("The I2P daemon reported that the stream session ID doesn't exist", RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.InvalidKey): RNS.log("The I2P daemon reported that the key for "+str(i2p_destination)+" is invalid", RNS.LOG_ERROR) elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.DuplicatedDest):
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.KeyNotFound): RNS.log("The I2P daemon could not find the key for "+str(i2p_destination), RNS.LOG_ERROR) RNS.log("The I2P daemon reported that the destination is already in use", RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.PeerNotFound): RNS.log("The I2P daemon could not find the peer "+str(i2p_destination), RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.I2PError): RNS.log("The I2P daemon experienced an unspecified error", RNS.LOG_ERROR) elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.DuplicatedId):
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.Timeout): RNS.log("I2P daemon timed out while setting up client tunnel to "+str(i2p_destination), RNS.LOG_ERROR) RNS.log("The I2P daemon reported that the ID is arleady in use", RNS.LOG_ERROR)
else: RNS.log(f"Unspecified I2P daemon error: {i2p_exception}", RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.InvalidId):
RNS.log("The I2P daemon reported that the stream session ID doesn't exist", RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.InvalidKey):
RNS.log("The I2P daemon reported that the key for "+str(i2p_destination)+" is invalid", RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.KeyNotFound):
RNS.log("The I2P daemon could not find the key for "+str(i2p_destination), RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.PeerNotFound):
RNS.log("The I2P daemon mould not find the peer "+str(i2p_destination), RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.I2PError):
RNS.log("The I2P daemon experienced an unspecified error", RNS.LOG_ERROR)
elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.Timeout):
RNS.log("I2P daemon timed out while setting up client tunnel to "+str(i2p_destination), RNS.LOG_ERROR)
RNS.log("Resetting I2P tunnel and retrying later", RNS.LOG_ERROR) RNS.log("Resetting I2P tunnel and retrying later", RNS.LOG_ERROR)
self.stop_tunnel(i2ptunnel) self.stop_tunnel(i2ptunnel)
return False return False
elif i2ptunnel.status["setup_failed"] == True: elif i2ptunnel.status["setup_failed"] == True:
RNS.log(str(self)+" Unspecified I2P tunnel setup error, resetting I2P tunnel", RNS.LOG_ERROR) RNS.log(str(self)+" Unspecified I2P tunnel setup error, resetting I2P tunnel", RNS.LOG_ERROR)
self.stop_tunnel(i2ptunnel) self.stop_tunnel(i2ptunnel)
return False return False
else: else:
RNS.log(str(self)+" Got no status from SAM API, resetting I2P tunnel", RNS.LOG_ERROR) RNS.log(str(self)+" Got no status from SAM API, resetting I2P tunnel", RNS.LOG_ERROR)
self.stop_tunnel(i2ptunnel) self.stop_tunnel(i2ptunnel)
return False return False
time.sleep(5) time.sleep(5)
def get_loop(self): return asyncio.get_event_loop() def get_loop(self):
return asyncio.get_event_loop()
class ThreadingI2PServer(socketserver.ThreadingMixIn, socketserver.TCPServer): class ThreadingI2PServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
pass pass
class I2PInterfacePeer(Interface): class I2PInterfacePeer(Interface):
RECONNECT_WAIT = 15 RECONNECT_WAIT = 15
RECONNECT_MAX_TRIES = None RECONNECT_MAX_TRIES = None
@ -353,11 +416,9 @@ class I2PInterfacePeer(Interface):
self.online = False self.online = False
self.detached = False self.detached = False
self.kiss_framing = False self.kiss_framing = False
self.tunnels_allowed = True
self.i2p_tunneled = True self.i2p_tunneled = True
self.i2p_dest = None self.i2p_dest = None
self.i2p_tunnel_ready = False self.i2p_tunnel_ready = False
self.lossless_medium = True
self.mode = RNS.Interfaces.Interface.Interface.MODE_FULL self.mode = RNS.Interfaces.Interface.Interface.MODE_FULL
self.bitrate = I2PInterface.BITRATE_GUESS self.bitrate = I2PInterface.BITRATE_GUESS
self.last_read = 0 self.last_read = 0
@ -370,11 +431,18 @@ class I2PInterfacePeer(Interface):
self.ifac_netkey = self.parent_interface.ifac_netkey self.ifac_netkey = self.parent_interface.ifac_netkey
if self.ifac_netname != None or self.ifac_netkey != None: if self.ifac_netname != None or self.ifac_netkey != None:
ifac_origin = b"" ifac_origin = b""
if self.ifac_netname != None: ifac_origin += RNS.Identity.full_hash(self.ifac_netname.encode("utf-8")) if self.ifac_netname != None:
if self.ifac_netkey != None: ifac_origin += RNS.Identity.full_hash(self.ifac_netkey.encode("utf-8")) ifac_origin += RNS.Identity.full_hash(self.ifac_netname.encode("utf-8"))
if self.ifac_netkey != None:
ifac_origin += RNS.Identity.full_hash(self.ifac_netkey.encode("utf-8"))
ifac_origin_hash = RNS.Identity.full_hash(ifac_origin) ifac_origin_hash = RNS.Identity.full_hash(ifac_origin)
self.ifac_key = RNS.Cryptography.hkdf(length=64, derive_from=ifac_origin_hash, salt=RNS.Reticulum.IFAC_SALT, context=None) self.ifac_key = RNS.Cryptography.hkdf(
length=64,
derive_from=ifac_origin_hash,
salt=RNS.Reticulum.IFAC_SALT,
context=None
)
self.ifac_identity = RNS.Identity.from_bytes(self.ifac_key) self.ifac_identity = RNS.Identity.from_bytes(self.ifac_key)
self.ifac_signature = self.ifac_identity.sign(RNS.Identity.full_hash(self.ifac_key)) self.ifac_signature = self.ifac_identity.sign(RNS.Identity.full_hash(self.ifac_key))
@ -382,8 +450,10 @@ class I2PInterfacePeer(Interface):
self.announce_rate_grace = None self.announce_rate_grace = None
self.announce_rate_penalty = None self.announce_rate_penalty = None
if max_reconnect_tries == None: self.max_reconnect_tries = I2PInterfacePeer.RECONNECT_MAX_TRIES if max_reconnect_tries == None:
else: self.max_reconnect_tries = max_reconnect_tries self.max_reconnect_tries = I2PInterfacePeer.RECONNECT_MAX_TRIES
else:
self.max_reconnect_tries = max_reconnect_tries
if connected_socket != None: if connected_socket != None:
self.receives = True self.receives = True
@ -391,12 +461,15 @@ class I2PInterfacePeer(Interface):
self.target_port = None self.target_port = None
self.socket = connected_socket self.socket = connected_socket
if platform.system() == "Linux": self.set_timeouts_linux() if platform.system() == "Linux":
elif platform.system() == "Darwin": self.set_timeouts_osx() self.set_timeouts_linux()
elif platform.system() == "Darwin":
self.set_timeouts_osx()
elif target_i2p_dest != None: elif target_i2p_dest != None:
self.receives = True self.receives = True
self.initiator = True self.initiator = True
self.bind_ip = "127.0.0.1" self.bind_ip = "127.0.0.1"
self.awaiting_i2p_tunnel = True self.awaiting_i2p_tunnel = True
@ -424,9 +497,13 @@ class I2PInterfacePeer(Interface):
thread.start() thread.start()
def wait_job(): def wait_job():
while self.awaiting_i2p_tunnel: time.sleep(0.25) while self.awaiting_i2p_tunnel:
time.sleep(0.25)
time.sleep(2) time.sleep(2)
if not self.kiss_framing:
self.wants_tunnel = True
if not self.connect(initial=True): if not self.connect(initial=True):
thread = threading.Thread(target=self.reconnect) thread = threading.Thread(target=self.reconnect)
thread.daemon = True thread.daemon = True
@ -440,6 +517,7 @@ class I2PInterfacePeer(Interface):
thread.daemon = True thread.daemon = True
thread.start() thread.start()
def set_timeouts_linux(self): def set_timeouts_linux(self):
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_USER_TIMEOUT, int(I2PInterfacePeer.I2P_USER_TIMEOUT * 1000)) self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_USER_TIMEOUT, int(I2PInterfacePeer.I2P_USER_TIMEOUT * 1000))
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
@ -448,8 +526,10 @@ class I2PInterfacePeer(Interface):
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, int(I2PInterfacePeer.I2P_PROBES)) self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, int(I2PInterfacePeer.I2P_PROBES))
def set_timeouts_osx(self): def set_timeouts_osx(self):
if hasattr(socket, "TCP_KEEPALIVE"): TCP_KEEPIDLE = socket.TCP_KEEPALIVE if hasattr(socket, "TCP_KEEPALIVE"):
else: TCP_KEEPIDLE = 0x10 TCP_KEEPIDLE = socket.TCP_KEEPALIVE
else:
TCP_KEEPIDLE = 0x10
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
self.socket.setsockopt(socket.IPPROTO_TCP, TCP_KEEPIDLE, int(I2PInterfacePeer.I2P_PROBE_AFTER)) self.socket.setsockopt(socket.IPPROTO_TCP, TCP_KEEPIDLE, int(I2PInterfacePeer.I2P_PROBE_AFTER))
@ -457,12 +537,16 @@ class I2PInterfacePeer(Interface):
def shutdown_socket(self, target_socket): def shutdown_socket(self, target_socket):
if callable(target_socket.close): if callable(target_socket.close):
try: try:
if socket != None: target_socket.shutdown(socket.SHUT_RDWR) if socket != None:
except Exception as e: RNS.log("Error while shutting down socket for "+str(self)+": "+str(e)) target_socket.shutdown(socket.SHUT_RDWR)
except Exception as e:
RNS.log("Error while shutting down socket for "+str(self)+": "+str(e))
try: try:
if socket != None: target_socket.close() if socket != None:
except Exception as e: RNS.log("Error while closing socket for "+str(self)+": "+str(e)) target_socket.close()
except Exception as e:
RNS.log("Error while closing socket for "+str(self)+": "+str(e))
def detach(self): def detach(self):
RNS.log("Detaching "+str(self), RNS.LOG_DEBUG) RNS.log("Detaching "+str(self), RNS.LOG_DEBUG)
@ -471,39 +555,22 @@ class I2PInterfacePeer(Interface):
if callable(self.socket.close): if callable(self.socket.close):
self.detached = True self.detached = True
try: self.socket.shutdown(socket.SHUT_RDWR) try:
except Exception as e: RNS.log("Error while shutting down socket for "+str(self)+": "+str(e)) self.socket.shutdown(socket.SHUT_RDWR)
except Exception as e:
RNS.log("Error while shutting down socket for "+str(self)+": "+str(e))
try: self.socket.close() try:
except Exception as e: RNS.log("Error while closing socket for "+str(self)+": "+str(e)) self.socket.close()
except Exception as e:
RNS.log("Error while closing socket for "+str(self)+": "+str(e))
self.socket = None self.socket = None
def early_synthesis(self, packet):
if not self.detached:
if not packet.packed: packet.pack()
if self.ifac_identity: raw = RNS.Transport.handle_outgoing_ifac(self, packet.raw);
else: raw = packet.raw
if self.kiss_framing: frame = bytes([KISS.FEND])+bytes([KISS.CMD_DATA])+KISS.escape(raw)+bytes([KISS.FEND])
else: frame = bytes([HDLC.FLAG])+HDLC.escape(raw)+bytes([HDLC.FLAG])
self.socket.sendall(frame)
self.txb += len(frame)
self.last_write = time.time()
if hasattr(self, "parent_interface") and self.parent_interface != None and self.parent_count: self.parent_interface.txb += len(frame)
def connect(self, initial=False): def connect(self, initial=False):
try: try:
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.connect((self.target_ip, self.target_port)) self.socket.connect((self.target_ip, self.target_port))
if not self.kiss_framing:
self.wants_tunnel = True
self.remote_transport = None
self.synthesis_received = False
RNS.Transport.synthesize_tunnel(self)
self.online = True self.online = True
except Exception as e: except Exception as e:
@ -511,16 +578,24 @@ class I2PInterfacePeer(Interface):
if not self.awaiting_i2p_tunnel: if not self.awaiting_i2p_tunnel:
RNS.log("Initial connection for "+str(self)+" could not be established: "+str(e), RNS.LOG_ERROR) RNS.log("Initial connection for "+str(self)+" could not be established: "+str(e), RNS.LOG_ERROR)
RNS.log("Leaving unconnected and retrying connection in "+str(I2PInterfacePeer.RECONNECT_WAIT)+" seconds.", RNS.LOG_ERROR) RNS.log("Leaving unconnected and retrying connection in "+str(I2PInterfacePeer.RECONNECT_WAIT)+" seconds.", RNS.LOG_ERROR)
return False
else: raise e
if platform.system() == "Linux": self.set_timeouts_linux() return False
elif platform.system() == "Darwin": self.set_timeouts_osx()
else:
raise e
if platform.system() == "Linux":
self.set_timeouts_linux()
elif platform.system() == "Darwin":
self.set_timeouts_osx()
self.online = True self.online = True
self.writing = False self.writing = False
self.never_connected = False self.never_connected = False
if not self.kiss_framing and self.wants_tunnel:
RNS.Transport.synthesize_tunnel(self)
return True return True
def reconnect(self): def reconnect(self):
@ -537,10 +612,14 @@ class I2PInterfacePeer(Interface):
self.teardown() self.teardown()
break break
try: self.connect() try:
self.connect()
except Exception as e: except Exception as e:
if not self.awaiting_i2p_tunnel: RNS.log("Connection attempt for "+str(self)+" failed: "+str(e), RNS.LOG_DEBUG) if not self.awaiting_i2p_tunnel:
else: RNS.log(str(self)+" still waiting for I2P tunnel to appear", RNS.LOG_VERBOSE) RNS.log("Connection attempt for "+str(self)+" failed: "+str(e), RNS.LOG_DEBUG)
else:
RNS.log(str(self)+" still waiting for I2P tunnel to appear", RNS.LOG_VERBOSE)
if not self.never_connected: if not self.never_connected:
RNS.log(str(self)+" Re-established connection via I2P tunnel", RNS.LOG_INFO) RNS.log(str(self)+" Re-established connection via I2P tunnel", RNS.LOG_INFO)
@ -549,31 +628,40 @@ class I2PInterfacePeer(Interface):
thread = threading.Thread(target=self.read_loop) thread = threading.Thread(target=self.read_loop)
thread.daemon = True thread.daemon = True
thread.start() thread.start()
if not self.kiss_framing:
RNS.Transport.synthesize_tunnel(self)
else: else:
RNS.log("Attempt to reconnect on a non-initiator I2P interface. This should not happen.", RNS.LOG_ERROR) RNS.log("Attempt to reconnect on a non-initiator I2P interface. This should not happen.", RNS.LOG_ERROR)
raise IOError("Attempt to reconnect on a non-initiator I2P interface") raise IOError("Attempt to reconnect on a non-initiator I2P interface")
def process_incoming(self, data): def process_incoming(self, data):
if not data: return self.rxb += len(data)
if hasattr(self, "parent_interface") and self.parent_interface != None and self.parent_count:
self.parent_interface.rxb += len(data)
self.owner.inbound(data, self) self.owner.inbound(data, self)
def process_outgoing(self, data): def process_outgoing(self, data):
if self.online: if self.online:
while self.writing: time.sleep(0.001) while self.writing:
time.sleep(0.001)
try: try:
self.writing = True self.writing = True
if self.kiss_framing: data = bytes([KISS.FEND])+bytes([KISS.CMD_DATA])+KISS.escape(data)+bytes([KISS.FEND]) if self.kiss_framing:
else: data = bytes([HDLC.FLAG])+HDLC.escape(data)+bytes([HDLC.FLAG]) data = bytes([KISS.FEND])+bytes([KISS.CMD_DATA])+KISS.escape(data)+bytes([KISS.FEND])
else:
data = bytes([HDLC.FLAG])+HDLC.escape(data)+bytes([HDLC.FLAG])
self.socket.sendall(data) self.socket.sendall(data)
self.writing = False self.writing = False
self.txb += len(data) self.txb += len(data)
self.last_write = time.time() self.last_write = time.time()
if hasattr(self, "parent_interface") and self.parent_interface != None and self.parent_count: self.parent_interface.txb += len(data) if hasattr(self, "parent_interface") and self.parent_interface != None and self.parent_count:
self.parent_interface.txb += len(data)
except Exception as e: except Exception as e:
RNS.log("Exception occurred while transmitting via "+str(self)+", tearing down interface", RNS.LOG_ERROR) RNS.log("Exception occurred while transmitting via "+str(self)+", tearing down interface", RNS.LOG_ERROR)
@ -582,7 +670,8 @@ class I2PInterfacePeer(Interface):
def read_watchdog(self): def read_watchdog(self):
while self.wd_reset: time.sleep(0.25) while self.wd_reset:
time.sleep(0.25)
should_run = True should_run = True
try: try:
@ -590,13 +679,17 @@ class I2PInterfacePeer(Interface):
time.sleep(1) time.sleep(1)
if (time.time()-self.last_read > I2PInterfacePeer.I2P_PROBE_AFTER*2): if (time.time()-self.last_read > I2PInterfacePeer.I2P_PROBE_AFTER*2):
if self.i2p_tunnel_state != I2PInterfacePeer.TUNNEL_STATE_STALE: RNS.log("I2P tunnel became unresponsive", RNS.LOG_DEBUG) if self.i2p_tunnel_state != I2PInterfacePeer.TUNNEL_STATE_STALE:
RNS.log("I2P tunnel became unresponsive", RNS.LOG_DEBUG)
self.i2p_tunnel_state = I2PInterfacePeer.TUNNEL_STATE_STALE self.i2p_tunnel_state = I2PInterfacePeer.TUNNEL_STATE_STALE
else: self.i2p_tunnel_state = I2PInterfacePeer.TUNNEL_STATE_ACTIVE else:
self.i2p_tunnel_state = I2PInterfacePeer.TUNNEL_STATE_ACTIVE
if (time.time()-self.last_write > I2PInterfacePeer.I2P_PROBE_AFTER*1): if (time.time()-self.last_write > I2PInterfacePeer.I2P_PROBE_AFTER*1):
try: try:
if self.socket != None: self.socket.sendall(bytes([HDLC.FLAG, HDLC.FLAG])) if self.socket != None:
self.socket.sendall(bytes([HDLC.FLAG, HDLC.FLAG]))
except Exception as e: except Exception as e:
RNS.log("An error ocurred while sending I2P keepalive. The contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("An error ocurred while sending I2P keepalive. The contained exception was: "+str(e), RNS.LOG_ERROR)
self.shutdown_socket(self.socket) self.shutdown_socket(self.socket)
@ -605,17 +698,22 @@ class I2PInterfacePeer(Interface):
if (time.time()-self.last_read > I2PInterfacePeer.I2P_READ_TIMEOUT): if (time.time()-self.last_read > I2PInterfacePeer.I2P_READ_TIMEOUT):
RNS.log("I2P socket is unresponsive, restarting...", RNS.LOG_WARNING) RNS.log("I2P socket is unresponsive, restarting...", RNS.LOG_WARNING)
if self.socket != None: if self.socket != None:
try: self.socket.shutdown(socket.SHUT_RDWR) try:
except Exception as e: RNS.log("Error while shutting down socket for "+str(self)+": "+str(e)) self.socket.shutdown(socket.SHUT_RDWR)
except Exception as e:
RNS.log("Error while shutting down socket for "+str(self)+": "+str(e))
try: self.socket.close() try:
except Exception as e: RNS.log("Error while closing socket for "+str(self)+": "+str(e)) self.socket.close()
except Exception as e:
RNS.log("Error while closing socket for "+str(self)+": "+str(e))
should_run = False should_run = False
self.wd_reset = False self.wd_reset = False
finally: self.wd_reset = False finally:
self.wd_reset = False
def read_loop(self): def read_loop(self):
try: try:
@ -686,6 +784,7 @@ class I2PInterfacePeer(Interface):
data_buffer = data_buffer+bytes([byte]) data_buffer = data_buffer+bytes([byte])
else: else:
self.online = False self.online = False
self.wd_reset = True self.wd_reset = True
time.sleep(2) time.sleep(2)
self.wd_reset = False self.wd_reset = False
@ -707,12 +806,15 @@ class I2PInterfacePeer(Interface):
if self.initiator: if self.initiator:
RNS.log("Attempting to reconnect...", RNS.LOG_WARNING) RNS.log("Attempting to reconnect...", RNS.LOG_WARNING)
self.reconnect() self.reconnect()
else: self.teardown() else:
self.teardown()
def teardown(self): def teardown(self):
if self.initiator and not self.detached: if self.initiator and not self.detached:
RNS.log("The interface "+str(self)+" experienced an unrecoverable error and is being torn down. Restart Reticulum to attempt to open this interface again.", RNS.LOG_ERROR) RNS.log("The interface "+str(self)+" experienced an unrecoverable error and is being torn down. Restart Reticulum to attempt to open this interface again.", RNS.LOG_ERROR)
if RNS.Reticulum.panic_on_interface_error: RNS.panic() if RNS.Reticulum.panic_on_interface_error:
RNS.panic()
else: else:
RNS.log("The interface "+str(self)+" is being torn down.", RNS.LOG_VERBOSE) RNS.log("The interface "+str(self)+" is being torn down.", RNS.LOG_VERBOSE)
@ -724,7 +826,9 @@ class I2PInterfacePeer(Interface):
while self in self.parent_interface.spawned_interfaces: while self in self.parent_interface.spawned_interfaces:
self.parent_interface.spawned_interfaces.remove(self) self.parent_interface.spawned_interfaces.remove(self)
if not self.initiator: RNS.Transport.remove_interface(self) if not self.initiator:
RNS.Transport.remove_interface(self)
def __str__(self): def __str__(self):
return "I2PInterfacePeer["+str(self.name)+"]" return "I2PInterfacePeer["+str(self.name)+"]"
@ -735,7 +839,8 @@ class I2PInterface(Interface):
DEFAULT_IFAC_SIZE = 16 DEFAULT_IFAC_SIZE = 16
@property @property
def clients(self): return len(self.spawned_interfaces) def clients(self):
return len(self.spawned_interfaces)
def __init__(self, owner, configuration): def __init__(self, owner, configuration):
super().__init__() super().__init__()
@ -756,8 +861,6 @@ class I2PInterface(Interface):
self.owner = owner self.owner = owner
self.connectable = connectable self.connectable = connectable
self.i2p_tunneled = True self.i2p_tunneled = True
self.tunnels_allowed = True
self.peers = peers
self.mode = RNS.Interfaces.Interface.Interface.MODE_FULL self.mode = RNS.Interfaces.Interface.Interface.MODE_FULL
self.b32 = None self.b32 = None
@ -767,6 +870,7 @@ class I2PInterface(Interface):
self.OUT = False self.OUT = False
self.name = name self.name = name
self.receives = True self.receives = True
self.bind_ip = "127.0.0.1" self.bind_ip = "127.0.0.1"
self.bind_port = self.i2p.get_free_port() self.bind_port = self.i2p.get_free_port()
@ -776,7 +880,6 @@ class I2PInterface(Interface):
self.ifac_netname = ifac_netname self.ifac_netname = ifac_netname
self.ifac_netkey = ifac_netkey self.ifac_netkey = ifac_netkey
self.supports_discovery = True self.supports_discovery = True
self.lossless_medium = True
self.online = False self.online = False
@ -791,9 +894,11 @@ class I2PInterface(Interface):
RNS.log("I2P controller did not become available in time, waiting for controller", RNS.LOG_VERBOSE) RNS.log("I2P controller did not become available in time, waiting for controller", RNS.LOG_VERBOSE)
i2p_notready_warning = True i2p_notready_warning = True
while not self.i2p.ready: time.sleep(0.25) while not self.i2p.ready:
time.sleep(0.25)
if i2p_notready_warning == True: RNS.log("I2P controller ready, continuing setup", RNS.LOG_VERBOSE) if i2p_notready_warning == True:
RNS.log("I2P controller ready, continuing setup", RNS.LOG_VERBOSE)
def handlerFactory(callback): def handlerFactory(callback):
def createHandler(*args, **keys): def createHandler(*args, **keys):
@ -807,7 +912,6 @@ class I2PInterface(Interface):
thread.daemon = True thread.daemon = True
thread.start() thread.start()
def final_init(self):
if self.connectable: if self.connectable:
def tunnel_job(): def tunnel_job():
while True: while True:
@ -827,61 +931,22 @@ class I2PInterface(Interface):
thread.daemon = True thread.daemon = True
thread.start() thread.start()
threading.Thread(target=self._spawn_peers, daemon=True).start() if peers != None:
for peer_addr in peers:
def _spawn_peers(self):
time.sleep(1.5)
if self.peers != None:
for peer_addr in self.peers:
try:
interface_name = self.name+" to "+peer_addr interface_name = self.name+" to "+peer_addr
peer_interface = I2PInterfacePeer(self, self.owner, interface_name, peer_addr) peer_interface = I2PInterfacePeer(self, self.owner, interface_name, peer_addr)
peer_interface.OUT = self.OUT peer_interface.OUT = True
peer_interface.IN = self.IN peer_interface.IN = True
peer_interface.parent_interface = self peer_interface.parent_interface = self
peer_interface.parent_count = False peer_interface.parent_count = False
peer_interface.mode = self.mode
peer_interface.bitrate = self.bitrate
peer_interface.recursive_prs = self.recursive_prs
peer_interface.gravity = self.gravity
peer_interface.HW_MTU = self.HW_MTU
peer_interface.announces_to_internal = self.announces_to_internal
peer_interface.announces_from_internal = self.announces_from_internal
peer_interface.announce_cap = self.announce_cap
peer_interface.ingress_control = self.ingress_control
peer_interface.ic_max_held_announces = self.ic_max_held_announces
peer_interface.ic_burst_hold = self.ic_burst_hold
peer_interface.ic_burst_freq = self.ic_burst_freq
peer_interface.ic_burst_freq_new = self.ic_burst_freq_new
peer_interface.ic_new_time = self.ic_new_time
peer_interface.ic_burst_penalty = self.ic_burst_penalty
peer_interface.ic_held_release_interval = self.ic_held_release_interval
peer_interface.egress_control = self.egress_control
peer_interface.ec_pr_freq = self.ec_pr_freq
peer_interface.ic_pr_burst_freq_new = self.ic_pr_burst_freq_new
peer_interface.ic_pr_burst_freq = self.ic_pr_burst_freq
peer_interface.announce_rate_target = self.announce_rate_target
peer_interface.announce_rate_grace = self.announce_rate_grace
peer_interface.announce_rate_penalty = self.announce_rate_penalty
peer_interface._destination_cost = self._destination_cost
peer_interface._pr_cost = self._pr_cost
RNS.Transport.add_interface(peer_interface) RNS.Transport.add_interface(peer_interface)
except Exception as e: RNS.log(f"Could not create interface for {self} to {peer_addr}: {e}", RNS.LOG_ERROR)
def incoming_connection(self, handler): def incoming_connection(self, handler):
RNS.log("Accepting incoming I2P connection", RNS.LOG_VERBOSE) RNS.log("Accepting incoming I2P connection", RNS.LOG_VERBOSE)
interface_name = "Connected peer on "+self.name interface_name = "Connected peer on "+self.name
spawned_interface = I2PInterfacePeer(self, self.owner, interface_name, connected_socket=handler.request) spawned_interface = I2PInterfacePeer(self, self.owner, interface_name, connected_socket=handler.request)
spawned_interface.OUT = self.OUT spawned_interface.OUT = True
spawned_interface.IN = self.IN spawned_interface.IN = True
spawned_interface.ingress_control = self.ingress_control spawned_interface.ingress_control = self.ingress_control
spawned_interface.ic_max_held_announces = self.ic_max_held_announces spawned_interface.ic_max_held_announces = self.ic_max_held_announces
@ -892,18 +957,11 @@ class I2PInterface(Interface):
spawned_interface.ic_burst_penalty = self.ic_burst_penalty spawned_interface.ic_burst_penalty = self.ic_burst_penalty
spawned_interface.ic_held_release_interval = self.ic_held_release_interval spawned_interface.ic_held_release_interval = self.ic_held_release_interval
spawned_interface.announces_to_internal = self.announces_to_internal
spawned_interface.announces_from_internal = self.announces_from_internal
spawned_interface.announce_cap = self.announce_cap
spawned_interface.egress_control = self.egress_control spawned_interface.egress_control = self.egress_control
spawned_interface.ec_pr_freq = self.ec_pr_freq spawned_interface.ec_pr_freq = self.ec_pr_freq
spawned_interface.ic_pr_burst_freq_new = self.ic_pr_burst_freq_new spawned_interface.ic_pr_burst_freq_new = self.ic_pr_burst_freq_new
spawned_interface.ic_pr_burst_freq = self.ic_pr_burst_freq spawned_interface.ic_pr_burst_freq = self.ic_pr_burst_freq
spawned_interface._destination_cost = self._destination_cost
spawned_interface._pr_cost = self._pr_cost
spawned_interface.parent_interface = self spawned_interface.parent_interface = self
spawned_interface.online = True spawned_interface.online = True
spawned_interface.bitrate = self.bitrate spawned_interface.bitrate = self.bitrate
@ -913,11 +971,18 @@ class I2PInterface(Interface):
spawned_interface.ifac_netkey = self.ifac_netkey spawned_interface.ifac_netkey = self.ifac_netkey
if spawned_interface.ifac_netname != None or spawned_interface.ifac_netkey != None: if spawned_interface.ifac_netname != None or spawned_interface.ifac_netkey != None:
ifac_origin = b"" ifac_origin = b""
if spawned_interface.ifac_netname != None: ifac_origin += RNS.Identity.full_hash(spawned_interface.ifac_netname.encode("utf-8")) if spawned_interface.ifac_netname != None:
if spawned_interface.ifac_netkey != None: ifac_origin += RNS.Identity.full_hash(spawned_interface.ifac_netkey.encode("utf-8")) ifac_origin += RNS.Identity.full_hash(spawned_interface.ifac_netname.encode("utf-8"))
if spawned_interface.ifac_netkey != None:
ifac_origin += RNS.Identity.full_hash(spawned_interface.ifac_netkey.encode("utf-8"))
ifac_origin_hash = RNS.Identity.full_hash(ifac_origin) ifac_origin_hash = RNS.Identity.full_hash(ifac_origin)
spawned_interface.ifac_key = RNS.Cryptography.hkdf(length=64, derive_from=ifac_origin_hash, salt=RNS.Reticulum.IFAC_SALT, context=None) spawned_interface.ifac_key = RNS.Cryptography.hkdf(
length=64,
derive_from=ifac_origin_hash,
salt=RNS.Reticulum.IFAC_SALT,
context=None
)
spawned_interface.ifac_identity = RNS.Identity.from_bytes(spawned_interface.ifac_key) spawned_interface.ifac_identity = RNS.Identity.from_bytes(spawned_interface.ifac_key)
spawned_interface.ifac_signature = spawned_interface.ifac_identity.sign(RNS.Identity.full_hash(spawned_interface.ifac_key)) spawned_interface.ifac_signature = spawned_interface.ifac_identity.sign(RNS.Identity.full_hash(spawned_interface.ifac_key))
@ -925,48 +990,40 @@ class I2PInterface(Interface):
spawned_interface.announce_rate_grace = self.announce_rate_grace spawned_interface.announce_rate_grace = self.announce_rate_grace
spawned_interface.announce_rate_penalty = self.announce_rate_penalty spawned_interface.announce_rate_penalty = self.announce_rate_penalty
spawned_interface.mode = self.mode spawned_interface.mode = self.mode
spawned_interface.recursive_prs = self.recursive_prs
spawned_interface.gravity = self.gravity
spawned_interface.hop_limiter = self.hop_limiter
spawned_interface.HW_MTU = self.HW_MTU spawned_interface.HW_MTU = self.HW_MTU
RNS.log("Spawned new I2PInterface Peer: "+str(spawned_interface), RNS.LOG_VERBOSE) RNS.log("Spawned new I2PInterface Peer: "+str(spawned_interface), RNS.LOG_VERBOSE)
RNS.Transport.add_interface(spawned_interface) RNS.Transport.add_interface(spawned_interface)
while spawned_interface in self.spawned_interfaces: self.spawned_interfaces.remove(spawned_interface) while spawned_interface in self.spawned_interfaces:
self.spawned_interfaces.remove(spawned_interface)
self.spawned_interfaces.append(spawned_interface) self.spawned_interfaces.append(spawned_interface)
spawned_interface.read_loop() spawned_interface.read_loop()
def process_outgoing(self, data): def process_outgoing(self, data):
pass pass
def received_announce(self, size=0, from_spawned=False): def received_announce(self, from_spawned=False):
if from_spawned: if from_spawned: self.ia_freq_deque.append(time.time())
self.ia_freq_deque.append(time.time())
self.arxb += size
def sent_announce(self, size=0, from_spawned=False): def sent_announce(self, from_spawned=False):
if from_spawned: if from_spawned: self.oa_freq_deque.append(time.time())
self.oa_freq_deque.append(time.time())
self.atxb += size
def received_path_request(self, size=0, from_spawned=False): def received_path_request(self, from_spawned=False):
if from_spawned: if from_spawned: self.ip_freq_deque.append(time.time())
self.ip_freq_deque.append(time.time())
self.prxb += size
def sent_path_request(self, size=0, from_spawned=False): def sent_path_request(self, from_spawned=False):
if from_spawned: if from_spawned: self.op_freq_deque.append(time.time())
self.op_freq_deque.append(time.time())
self.ptxb += size
def detach(self): def detach(self):
RNS.log("Detaching "+str(self), RNS.LOG_DEBUG) RNS.log("Detaching "+str(self), RNS.LOG_DEBUG)
self.i2p.stop() self.i2p.stop()
def __str__(self): return "I2PInterface["+self.name+"]" def __str__(self):
return "I2PInterface["+self.name+"]"
class I2PInterfaceHandler(socketserver.BaseRequestHandler): class I2PInterfaceHandler(socketserver.BaseRequestHandler):
def __init__(self, callback, *args, **keys): def __init__(self, callback, *args, **keys):
self.callback = callback self.callback = callback
socketserver.BaseRequestHandler.__init__(self, *args, **keys) socketserver.BaseRequestHandler.__init__(self, *args, **keys)
def handle(self): self.callback(handler=self) def handle(self):
self.callback(handler=self)

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -39,6 +39,7 @@ class Interface:
OUT = False OUT = False
FWD = False FWD = False
RPT = False RPT = False
name = None
# Interface mode definitions # Interface mode definitions
MODE_FULL = 0x01 MODE_FULL = 0x01
@ -47,12 +48,10 @@ class Interface:
MODE_ROAMING = 0x04 MODE_ROAMING = 0x04
MODE_BOUNDARY = 0x05 MODE_BOUNDARY = 0x05
MODE_GATEWAY = 0x06 MODE_GATEWAY = 0x06
MODE_INTERNAL = 0x07
# Which interface modes a Transport Node should # Which interface modes a Transport Node should
# actively discover paths for. # actively discover paths for.
DISCOVER_PATHS_FOR = [MODE_ACCESS_POINT, MODE_GATEWAY, MODE_ROAMING, MODE_INTERNAL] DISCOVER_PATHS_FOR = [MODE_ACCESS_POINT, MODE_GATEWAY, MODE_ROAMING]
BOUNDARY_SEARCH_MODES = [MODE_BOUNDARY, MODE_GATEWAY]
# How many samples to use for announce # How many samples to use for announce
# frequency calculations # frequency calculations
@ -68,10 +67,11 @@ class Interface:
# Maximum amount of ingress limited announces # Maximum amount of ingress limited announces
# to hold at any given time. # to hold at any given time.
MAX_HELD_ANNOUNCES = 64 MAX_HELD_ANNOUNCES = 256
# Control parameters # How long a spawned interface will be
DEFAULT_GRAVITY = 0 # considered to be newly created. Two
# hours by default.
IC_NEW_TIME = 2*60*60 IC_NEW_TIME = 2*60*60
IC_BURST_FREQ_NEW = 3 IC_BURST_FREQ_NEW = 3
IC_BURST_FREQ = 10 IC_BURST_FREQ = 10
@ -81,7 +81,7 @@ class Interface:
IC_BURST_PENALTY = 15 IC_BURST_PENALTY = 15
IC_HELD_RELEASE_INTERVAL = 5 IC_HELD_RELEASE_INTERVAL = 5
IC_DEQUE_MIN_SAMPLE = 2 IC_DEQUE_MIN_SAMPLE = 2
EC_BURST_MIN_SAMPLES = 2 IC_BURST_MIN_SAMPLES = 6
EC_PR_FREQ = 5 EC_PR_FREQ = 5
EGRESS_CONTROL = False EGRESS_CONTROL = False
@ -92,69 +92,30 @@ class Interface:
AUTOCONFIGURE_MTU = False AUTOCONFIGURE_MTU = False
FIXED_MTU = False FIXED_MTU = False
DEFAULT_IFAC_SIZE = 16
def __init__(self): def __init__(self):
self.name = None
self.rxb = 0 self.rxb = 0
self.txb = 0 self.txb = 0
self.frxb = 0
self.ftxb = 0
self.arxb = 0
self.atxb = 0
self.arxc = 0
self.atxc = 0
self.afrxc = 0
self.prxb = 0
self.ptxb = 0
self.prxc = 0
self.ptxc = 0
self.gravity = 0
self.created = time.time() self.created = time.time()
self.detached = False self.detached = False
self.online = False self.online = False
self.bitrate = 62500 self.bitrate = 62500
self.HW_MTU = None self.HW_MTU = None
self.__hash = None
self.announce_cap = RNS.Reticulum.ANNOUNCE_CAP/100.0
self.announce_allowed_at = 0
self.supports_discovery = False self.supports_discovery = False
self.discoverable = False self.discoverable = False
self.last_discovery_announce = 0 self.last_discovery_announce = 0
self.bootstrap_only = False self.bootstrap_only = False
self.recursive_prs = False
self.announces_from_internal = True
self.announces_to_internal = None
self.is_local_shared_instance = False
self.parent_interface = None self.parent_interface = None
self.spawned_interfaces = None self.spawned_interfaces = None
self.tunnel_id = None self.tunnel_id = None
self.local_tunnel_id = None
self.tunnels_allowed = False
self.synthesis_received = False
self.synthesis_emitted = False
self.wants_tunnel = False
self.remote_transport = None
self.ingress_control = True self.ingress_control = True
self.hop_limiter = None
self.phy_keepalive = False self.phy_keepalive = False
self.shared_medium = False
self.lossless_medium = False
self.remote_impl = None
self.remote_vers = None
self.remote_dst_cost = None
self.remote_pr_cost = None
self.remote_is_tn = None
self.ic_burst_active = False self.ic_burst_active = False
self.ic_burst_activated = 0 self.ic_burst_activated = 0
self.ic_burst_sustained = 0
self.ic_pr_burst_active = False self.ic_pr_burst_active = False
self.ic_pr_burst_activated = 0 self.ic_pr_burst_activated = 0
self.ic_pr_burst_sustained = 0
self.ic_pr_burst_cooldown = 0
self.ic_held_release = 0 self.ic_held_release = 0
self.ic_max_held_announces = RNS.Reticulum.get_instance()._default_ic_max_held_announces() self.ic_max_held_announces = RNS.Reticulum.get_instance()._default_ic_max_held_announces()
self.ic_burst_hold = RNS.Reticulum.get_instance()._default_ic_burst_hold() self.ic_burst_hold = RNS.Reticulum.get_instance()._default_ic_burst_hold()
@ -167,47 +128,15 @@ class Interface:
self.ic_held_release_interval = RNS.Reticulum.get_instance()._default_ic_held_release_interval() self.ic_held_release_interval = RNS.Reticulum.get_instance()._default_ic_held_release_interval()
self.ec_pr_freq = RNS.Reticulum.get_instance()._default_ec_pr_freq() self.ec_pr_freq = RNS.Reticulum.get_instance()._default_ec_pr_freq()
self.egress_control = RNS.Reticulum.get_instance()._default_egress_control() self.egress_control = RNS.Reticulum.get_instance()._default_egress_control()
self.announce_queue = []
self.held_announces = {} self.held_announces = {}
self.ifac_identity = None
self.transmit_buffer = None
self.ia_freq_deque = deque(maxlen=Interface.IA_FREQ_SAMPLES) self.ia_freq_deque = deque(maxlen=Interface.IA_FREQ_SAMPLES)
self.oa_freq_deque = deque(maxlen=Interface.OA_FREQ_SAMPLES) self.oa_freq_deque = deque(maxlen=Interface.OA_FREQ_SAMPLES)
self.ip_freq_deque = deque(maxlen=Interface.IA_FREQ_SAMPLES) self.ip_freq_deque = deque(maxlen=Interface.IA_FREQ_SAMPLES)
self.op_freq_deque = deque(maxlen=Interface.OA_FREQ_SAMPLES) self.op_freq_deque = deque(maxlen=Interface.OA_FREQ_SAMPLES)
self.protocol_violations = 0
self.ifac_violations = 0
self.packet_filter_hits = 0
self.hop_limiter_hits = 0
self.dp_ingress_tcount = 0
self.dp_ingress_bytes = 0
self.dp_ingress_packets = 0
self.dp_ingress_hold = None
self.dp_ingress_gated = False
self.tx_hwm = 4*1024*1024
self.tx_stalled = False
self.tx_drops = 0
self.tx_dropped_bytes = 0
self._dp_ec_prev_sent = 0
self._dp_ec_zero_ticks = 0
self._dp_ec_last_drain = time.time()
self._destination_cost = None
self._pr_cost = None
self.reports_phy_stats = False
self.r_stat_rssi = None
self.r_stat_snr = None
self.r_stat_q = None
self.is_connected_to_shared_instance = False
def get_hash(self): def get_hash(self):
if not self.__hash: self.__hash = RNS.Identity.full_hash(str(self).encode("utf-8")) return RNS.Identity.full_hash(str(self).encode("utf-8"))
return self.__hash
# This is a generic function for determining when an interface # This is a generic function for determining when an interface
# should activate ingress limiting. Since this can vary for # should activate ingress limiting. Since this can vary for
@ -219,10 +148,8 @@ class Interface:
ia_freq = self.incoming_announce_frequency() ia_freq = self.incoming_announce_frequency()
if self.ic_burst_active: if self.ic_burst_active:
if ia_freq < freq_threshold and time.time() > self.ic_burst_activated+self.ic_burst_hold and time.time() > self.ic_burst_sustained+self.ic_burst_hold: if ia_freq < freq_threshold and time.time() > self.ic_burst_activated+self.ic_burst_hold:
if len(self.ia_freq_deque) >= self.IC_DEQUE_MIN_SAMPLE: self.ic_burst_active = False if len(self.ia_freq_deque) >= self.IC_BURST_MIN_SAMPLES: self.ic_burst_active = False
else:
if ia_freq >= freq_threshold: self.ic_burst_sustained = time.time()
return True return True
@ -230,7 +157,6 @@ class Interface:
if ia_freq > freq_threshold: if ia_freq > freq_threshold:
self.ic_burst_active = True self.ic_burst_active = True
self.ic_burst_activated = time.time() self.ic_burst_activated = time.time()
self.ic_burst_sustained = time.time()
self.ic_held_release = time.time() + self.ic_burst_penalty self.ic_held_release = time.time() + self.ic_burst_penalty
return True return True
@ -244,12 +170,8 @@ class Interface:
ip_freq = self.incoming_pr_frequency() ip_freq = self.incoming_pr_frequency()
if self.ic_pr_burst_active: if self.ic_pr_burst_active:
if ip_freq < freq_threshold and time.time() > self.ic_pr_burst_activated+self.ic_burst_hold and time.time() > self.ic_pr_burst_sustained+self.ic_burst_hold: if ip_freq < freq_threshold and time.time() > self.ic_pr_burst_activated+self.ic_burst_hold:
if self.ic_pr_burst_cooldown <= 0: self.ic_pr_burst_active = False self.ic_pr_burst_active = False
else: self.ic_pr_burst_cooldown -= 1
else:
self.ic_pr_burst_cooldown = 3
if ip_freq >= freq_threshold: self.ic_pr_burst_sustained = time.time()
return True return True
@ -257,8 +179,6 @@ class Interface:
if ip_freq > freq_threshold: if ip_freq > freq_threshold:
self.ic_pr_burst_active = True self.ic_pr_burst_active = True
self.ic_pr_burst_activated = time.time() self.ic_pr_burst_activated = time.time()
self.ic_pr_burst_sustained = time.time()
self.ic_pr_burst_cooldown = 3
return True return True
else: return False else: return False
@ -268,42 +188,48 @@ class Interface:
def should_egress_limit_pr(self): def should_egress_limit_pr(self):
if self.egress_control: if self.egress_control:
freq_threshold = self.ec_pr_freq freq_threshold = self.ec_pr_freq
op_freq = self.outgoing_pr_frequency(preemptive=True) op_freq = self.outgoing_pr_frequency()
if op_freq > freq_threshold: if op_freq > freq_threshold:
if len(self.op_freq_deque) >= self.EC_BURST_MIN_SAMPLES: return True if len(self.op_freq_deque) >= self.IC_BURST_MIN_SAMPLES: return True
return False return False
def optimise_mtu(self): def optimise_mtu(self):
if self.AUTOCONFIGURE_MTU: if self.AUTOCONFIGURE_MTU:
if self.bitrate >= 1_000_000_000: self.HW_MTU = 524288 if self.bitrate >= 1_000_000_000:
elif self.bitrate >= 750_000_000: self.HW_MTU = 262144 self.HW_MTU = 524288
elif self.bitrate >= 400_000_000: self.HW_MTU = 131072 elif self.bitrate > 750_000_000:
elif self.bitrate >= 200_000_000: self.HW_MTU = 65536 self.HW_MTU = 262144
elif self.bitrate >= 100_000_000: self.HW_MTU = 32768 elif self.bitrate > 400_000_000:
elif self.bitrate >= 10_000_000: self.HW_MTU = 16384 self.HW_MTU = 131072
elif self.bitrate >= 5_000_000: self.HW_MTU = 8192 elif self.bitrate > 200_000_000:
elif self.bitrate >= 2_000_000: self.HW_MTU = 4096 self.HW_MTU = 65536
elif self.bitrate >= 1_000_000: self.HW_MTU = 2048 elif self.bitrate > 100_000_000:
elif self.bitrate >= 62_500: self.HW_MTU = 1024 self.HW_MTU = 32768
else: self.HW_MTU = None elif self.bitrate > 10_000_000:
self.HW_MTU = 16384
elif self.bitrate > 5_000_000:
self.HW_MTU = 8192
elif self.bitrate > 2_000_000:
self.HW_MTU = 4096
elif self.bitrate > 1_000_000:
self.HW_MTU = 2048
elif self.bitrate > 62_500:
self.HW_MTU = 1024
else:
self.HW_MTU = None
RNS.log(f"{self} hardware MTU set to {self.HW_MTU}", RNS.LOG_EXTREME) RNS.log(f"{self} hardware MTU set to {self.HW_MTU}", RNS.LOG_DEBUG)
def age(self): def age(self):
return time.time()-self.created return time.time()-self.created
def hold_announce(self, announce_packet): def hold_announce(self, announce_packet):
if announce_packet.hops >= RNS.Transport.PATHFINDER_M-1: if announce_packet.destination_hash in self.held_announces:
return self.filtered_announce()
elif announce_packet.destination_hash in self.held_announces:
self.held_announces[announce_packet.destination_hash] = announce_packet self.held_announces[announce_packet.destination_hash] = announce_packet
return self.filtered_announce()
elif not len(self.held_announces) >= self.ic_max_held_announces: elif not len(self.held_announces) >= self.ic_max_held_announces:
self.held_announces[announce_packet.destination_hash] = announce_packet self.held_announces[announce_packet.destination_hash] = announce_packet
return True
else: return self.filtered_announce()
def process_held_announces(self): def process_held_announces(self):
try: try:
@ -313,101 +239,42 @@ class Interface:
if ia_freq < freq_threshold: if ia_freq < freq_threshold:
selected_announce_packet = None selected_announce_packet = None
min_hops = RNS.Transport.PATHFINDER_M min_hops = RNS.Transport.PATHFINDER_M
cull_entries = [] for destination_hash in self.held_announces:
for destination_hash in self.held_announces.copy(): announce_packet = self.held_announces[destination_hash]
announce_packet = self.held_announces.get(destination_hash)
if not announce_packet: continue
# If we can determine that a held announce has become
# useless for any potential path table updates, we'll
# remove it immediately instead of waiting for it to
# eventually get released, freeing up other held queue
# slots for potentially useful new announces.
path_entry = RNS.Transport.path_table.get(destination_hash)
if path_entry and not RNS.Transport.announce_prefilter(announce_packet, path_entry):
cull_entries.append(destination_hash)
continue
if announce_packet.hops < min_hops: if announce_packet.hops < min_hops:
min_hops = announce_packet.hops min_hops = announce_packet.hops
selected_announce_packet = announce_packet selected_announce_packet = announce_packet
for destination_hash in cull_entries: self.held_announces.pop(destination_hash, None)
if selected_announce_packet != None: if selected_announce_packet != None:
RNS.log("Releasing held announce packet "+str(selected_announce_packet)+" from "+str(self), RNS.LOG_EXTREME) RNS.log("Releasing held announce packet "+str(selected_announce_packet)+" from "+str(self), RNS.LOG_EXTREME)
self.ic_held_release = time.time() + self.ic_held_release_interval self.ic_held_release = time.time() + self.ic_held_release_interval
self.held_announces.pop(selected_announce_packet.destination_hash, None) self.held_announces.pop(selected_announce_packet.destination_hash)
RNS.Transport.inbound(selected_announce_packet.raw, selected_announce_packet.receiving_interface, tc=RNS.Transport.TC_INGRESS_LIMITED, ifac_handled=True) def release(): RNS.Transport.inbound(selected_announce_packet.raw, selected_announce_packet.receiving_interface)
threading.Thread(target=release, daemon=True).start()
except Exception as e: except Exception as e:
RNS.log("An error occurred while processing held announces for "+str(self), RNS.LOG_ERROR) RNS.log("An error occurred while processing held announces for "+str(self), RNS.LOG_ERROR)
RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
def received_announce(self, size=0, from_spawned=False): def received_announce(self, from_spawned=False):
self.arxc += 1; self.arxb += size
self.ia_freq_deque.append(time.time()) self.ia_freq_deque.append(time.time())
if self.parent_interface != None: if hasattr(self, "parent_interface") and self.parent_interface != None:
self.parent_interface.received_announce(size=size, from_spawned=True) self.parent_interface.received_announce(from_spawned=True)
def sent_announce(self, size=0, from_spawned=False): def sent_announce(self, from_spawned=False):
self.atxc += 1; self.atxb += size
self.oa_freq_deque.append(time.time()) self.oa_freq_deque.append(time.time())
if self.parent_interface != None: if hasattr(self, "parent_interface") and self.parent_interface != None:
self.parent_interface.sent_announce(size=size, from_spawned=True) self.parent_interface.sent_announce(from_spawned=True)
def filtered_announce(self, size=0, from_spawned=False): def received_path_request(self, from_spawned=False):
self.afrxc += 1
RNS.Transport.filtered_arxc += 1
if self.parent_interface != None: self.parent_interface.afrxc += 1
return None
def received_path_request(self, size=0, from_spawned=False):
self.prxc += 1; self.prxb += size
self.ip_freq_deque.append(time.time()) self.ip_freq_deque.append(time.time())
if self.parent_interface != None: if hasattr(self, "parent_interface") and self.parent_interface != None:
self.parent_interface.received_path_request(size=size, from_spawned=True) self.parent_interface.received_path_request(from_spawned=True)
def sent_path_request(self, size=0, from_spawned=False): def sent_path_request(self, from_spawned=False):
self.ptxc += 1; self.ptxb += size
self.op_freq_deque.append(time.time()) self.op_freq_deque.append(time.time())
if self.parent_interface != None: if hasattr(self, "parent_interface") and self.parent_interface != None:
self.parent_interface.sent_path_request(size=size, from_spawned=True) self.parent_interface.sent_path_request(from_spawned=True)
def protocol_violation(self, description=None, level=None):
if level == None: level = RNS.LOG_DEBUG
self.protocol_violations += 1
RNS.log(f"Protocol violation on {self}: {description}", level) if RNS.sl(level) else None
return None
def ifac_violation(self, description=None):
self.ifac_violations += 1
RNS.log(f"IFAC violation on {self}: {description}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return None
def packet_filter_hit(self):
self.packet_filter_hits += 1
return None
def hop_limiter_hit(self):
self.hop_limiter_hits += 1
return None
@property
def destination_cost(self):
if not self._destination_cost == None: return self._destination_cost
else: return RNS.Transport.DESTINATION_COST
@property
def pr_cost(self):
if not self._pr_cost == None: return self._pr_cost
else: return RNS.Transport.PR_COST
@property
def ic_burst_count(self): return None
@property
def ic_pr_burst_count(self): return None
def incoming_announce_frequency(self): def incoming_announce_frequency(self):
n = len(self.ia_freq_deque) n = len(self.ia_freq_deque)
@ -442,8 +309,8 @@ class Interface:
hz = n / span hz = n / span
return hz return hz
def outgoing_pr_frequency(self, preemptive=False): def outgoing_pr_frequency(self):
n = len(self.op_freq_deque)+(1 if preemptive else 0) n = len(self.op_freq_deque)
if not len(self.op_freq_deque) > 1: return 0 if not len(self.op_freq_deque) > 1: return 0
else: else:
oldest = self.op_freq_deque[0] oldest = self.op_freq_deque[0]
@ -454,12 +321,10 @@ class Interface:
return hz return hz
def process_announce_queue(self): def process_announce_queue(self):
if not self.announce_queue: return
if not hasattr(self, "announce_cap"): if not hasattr(self, "announce_cap"):
self.announce_cap = RNS.Reticulum.ANNOUNCE_CAP/100.0 self.announce_cap = RNS.Reticulum.ANNOUNCE_CAP
if self.announce_queue: if hasattr(self, "announce_queue"):
try: try:
now = time.time() now = time.time()
stale = [] stale = []
@ -478,13 +343,12 @@ class Interface:
selected = entries[0] selected = entries[0]
now = time.time() now = time.time()
tx_size = len(selected["raw"]) tx_time = (len(selected["raw"])*8) / self.bitrate
tx_time = (tx_size*8) / self.bitrate
wait_time = (tx_time / self.announce_cap) wait_time = (tx_time / self.announce_cap)
self.announce_allowed_at = now + wait_time self.announce_allowed_at = now + wait_time
RNS.Transport.transmit(self, selected["raw"]) self.process_outgoing(selected["raw"])
self.sent_announce(tx_size) self.sent_announce()
if selected in self.announce_queue: if selected in self.announce_queue:
self.announce_queue.remove(selected) self.announce_queue.remove(selected)
@ -504,14 +368,13 @@ class Interface:
def detach(self): def detach(self):
pass pass
def teardown(self):
pass
@staticmethod @staticmethod
def get_config_obj(config_in): def get_config_obj(config_in):
if type(config_in) == ConfigObj: return config_in if type(config_in) == ConfigObj:
return config_in
else: else:
try: return ConfigObj(config_in) try:
return ConfigObj(config_in)
except Exception as e: except Exception as e:
RNS.log(f"Could not parse supplied configuration data. The contained exception was: {e}", RNS.LOG_ERROR) RNS.log(f"Could not parse supplied configuration data. The contained exception was: {e}", RNS.LOG_ERROR)
raise SystemError("Invalid configuration data supplied") raise SystemError("Invalid configuration data supplied")

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -122,9 +122,7 @@ class KISSInterface(Interface):
self.packet_queue = [] self.packet_queue = []
self.flow_control = flow_control self.flow_control = flow_control
self.shared_medium = True
self.interface_ready = False self.interface_ready = False
self.reconnecting = False
self.flow_control_timeout = 5 self.flow_control_timeout = 5
self.flow_control_locked = time.time() self.flow_control_locked = time.time()
@ -141,17 +139,15 @@ class KISSInterface(Interface):
try: try:
self.open_port() self.open_port()
if self.serial.is_open: self.configure_device()
else: raise IOError("Could not open serial port")
except Exception as e: except Exception as e:
RNS.log(f"Could not open serial port for {self}", RNS.LOG_ERROR) RNS.log("Could not open serial port "+self.port, RNS.LOG_ERROR)
RNS.log(f"The contained exception was: {e}", RNS.LOG_ERROR) raise e
RNS.log(f"Reticulum will attempt to bring up this interface periodically", RNS.LOG_ERROR)
if not self.detached and not self.reconnecting: if self.serial.is_open:
thread = threading.Thread(target=self.reconnect_port) self.configure_device()
thread.daemon = True else:
thread.start() raise IOError("Could not open serial port")
def open_port(self): def open_port(self):
RNS.log("Opening serial port "+self.port+"...", RNS.LOG_VERBOSE) RNS.log("Opening serial port "+self.port+"...", RNS.LOG_VERBOSE)
@ -169,11 +165,6 @@ class KISSInterface(Interface):
dsrdtr = False, dsrdtr = False,
) )
def detach(self):
self.detached = True
self.online = False
try: self.serial.close()
except Exception as e: RNS.log(f"Could not close port for {self} at detach: {e}", RNS.LOG_ERROR)
def configure_device(self): def configure_device(self):
# Allow time for interface to initialise before config # Allow time for interface to initialise before config
@ -254,7 +245,7 @@ class KISSInterface(Interface):
def process_incoming(self, data): def process_incoming(self, data):
if not data: return self.rxb += len(data)
self.owner.inbound(data, self) self.owner.inbound(data, self)
@ -368,28 +359,30 @@ class KISSInterface(Interface):
RNS.log("A serial port error occurred, the contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("A serial port error occurred, the contained exception was: "+str(e), RNS.LOG_ERROR)
RNS.log("The interface "+str(self)+" experienced an unrecoverable error and is now offline.", RNS.LOG_ERROR) RNS.log("The interface "+str(self)+" experienced an unrecoverable error and is now offline.", RNS.LOG_ERROR)
if RNS.Reticulum.panic_on_interface_error: RNS.panic() if RNS.Reticulum.panic_on_interface_error:
RNS.panic()
RNS.log("Reticulum will attempt to reconnect the interface periodically.", RNS.LOG_ERROR) RNS.log("Reticulum will attempt to reconnect the interface periodically.", RNS.LOG_ERROR)
self.online = False self.online = False
try: self.serial.close() self.serial.close()
except Exception as e: RNS.log(f"Could not close port for {self}", RNS.LOG_ERROR) self.reconnect_port()
if not self.detached and not self.reconnecting: self.reconnect_port()
def reconnect_port(self): def reconnect_port(self):
self.reconnecting = True while not self.online:
while not self.online and not self.detached:
try: try:
time.sleep(5) time.sleep(5)
RNS.log("Attempting to reconnect serial port "+str(self.port)+" for "+str(self)+"...", RNS.LOG_VERBOSE) RNS.log("Attempting to reconnect serial port "+str(self.port)+" for "+str(self)+"...", RNS.LOG_VERBOSE)
self.open_port() self.open_port()
if self.serial.is_open: self.configure_device() if self.serial.is_open:
except Exception as e: RNS.log("Error while reconnecting port, the contained exception was: "+str(e), RNS.LOG_ERROR) self.configure_device()
except Exception as e:
RNS.log("Error while reconnecting port, the contained exception was: "+str(e), RNS.LOG_ERROR)
self.reconnecting = False RNS.log("Reconnected serial port for "+str(self))
if self.online: RNS.log("Reconnected serial port for "+str(self))
def should_ingress_limit(self): return False def should_ingress_limit(self):
return False
def __str__(self): return "KISSInterface["+self.name+"]" def __str__(self):
return "KISSInterface["+self.name+"]"

View file

@ -30,8 +30,6 @@
from RNS.Interfaces.Interface import Interface from RNS.Interfaces.Interface import Interface
from RNS.Interfaces.BackboneInterface import BackboneInterface from RNS.Interfaces.BackboneInterface import BackboneInterface
from RNS.Interfaces.util.TransmitBuffer import TransmitBuffer
from RNS.Interfaces.util.HDLC import HDLC, ReceiveBuffer
import socketserver import socketserver
import threading import threading
import socket import socket
@ -41,6 +39,17 @@ import os
import RNS import RNS
from threading import Lock from threading import Lock
class HDLC():
FLAG = 0x7E
ESC = 0x7D
ESC_MASK = 0x20
@staticmethod
def escape(data):
data = data.replace(bytes([HDLC.ESC]), bytes([HDLC.ESC, HDLC.ESC^HDLC.ESC_MASK]))
data = data.replace(bytes([HDLC.FLAG]), bytes([HDLC.ESC, HDLC.FLAG^HDLC.ESC_MASK]))
return data
class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
def server_bind(self): def server_bind(self):
if RNS.vendor.platformutils.is_windows(): if RNS.vendor.platformutils.is_windows():
@ -58,12 +67,9 @@ class LocalClientInterface(Interface):
def __init__(self, owner, name, target_port = None, connected_socket=None, socket_path=None): def __init__(self, owner, name, target_port = None, connected_socket=None, socket_path=None):
super().__init__() super().__init__()
self.owner = owner
self.bitrate = 1_000_000_000
self.epoll_backend = False self.epoll_backend = False
self.HW_MTU = 262144 self.HW_MTU = 262144
self.online = False self.online = False
self.ifac_size = self.DEFAULT_IFAC_SIZE
if socket_path != None and RNS.Reticulum.get_instance().use_af_unix: self.socket_path = f"\0rns/{socket_path}" if socket_path != None and RNS.Reticulum.get_instance().use_af_unix: self.socket_path = f"\0rns/{socket_path}"
else: self.socket_path = None else: self.socket_path = None
@ -77,9 +83,8 @@ class LocalClientInterface(Interface):
self.detached = False self.detached = False
self.name = name self.name = name
self.mode = RNS.Interfaces.Interface.Interface.MODE_FULL self.mode = RNS.Interfaces.Interface.Interface.MODE_FULL
self.transmit_buffer = TransmitBuffer() self.frame_buffer = b""
self.receive_buffer = ReceiveBuffer(mtu=lambda: self.HW_MTU, min_frame_len=RNS.Reticulum.HEADER_MINSIZE, self.transmit_buffer = b""
max_frame_len=lambda: self.HW_MTU, on_frame=self.process_incoming)
if RNS.vendor.platformutils.use_epoll(): self.epoll_backend = True if RNS.vendor.platformutils.use_epoll(): self.epoll_backend = True
@ -112,6 +117,8 @@ class LocalClientInterface(Interface):
self.target_port = target_port self.target_port = target_port
self.connect() self.connect()
self.owner = owner
self.bitrate = 1_000_000_000
self.online = True self.online = True
self.writing = False self.writing = False
@ -187,21 +194,24 @@ class LocalClientInterface(Interface):
def send_keepalive(self): def send_keepalive(self):
if self.online: if self.online:
RNS.log(f"Sending keepalive on {self}", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None RNS.log(f"Sending keepalive on {self}", RNS.LOG_DEBUG) # TODO: Remove
try: try:
if self.epoll_backend: if self.epoll_backend:
self.transmit_buffer.append(bytes([HDLC.FLAG])+bytes([HDLC.FLAG])) self.transmit_buffer += bytes([HDLC.FLAG])+bytes([HDLC.FLAG])
BackboneInterface.tx_ready(self) BackboneInterface.tx_ready(self)
else: else:
self.writing = True self.writing = True
data = bytes([HDLC.FLAG])+bytes([HDLC.FLAG]) data = bytes([HDLC.FLAG])+HDLC.escape(data)+bytes([HDLC.FLAG])
self.socket.sendall(data) self.socket.sendall(data)
self.writing = False self.writing = False
except Exception as e: RNS.log(f"Exception occurred while sending keepalive on {self}: {e}", RNS.LOG_ERROR) except Exception as e: RNS.log(f"Exception occurred while sending keepalive on {self}: {e}", RNS.LOG_ERROR)
def process_incoming(self, data): def process_incoming(self, data):
self.rxb += len(data)
if self.parent_interface != None: self.parent_interface.rxb += len(data)
try: self.owner.inbound(data, self) try: self.owner.inbound(data, self)
except Exception as e: except Exception as e:
RNS.log(f"An error occurred in the processing of an incoming frame for {self}: {e}", RNS.LOG_ERROR) RNS.log(f"An error occurred in the processing of an incoming frame for {self}: {e}", RNS.LOG_ERROR)
@ -209,30 +219,28 @@ class LocalClientInterface(Interface):
def process_outgoing(self, data): def process_outgoing(self, data):
if self.pause_on_client_sleep and time.time() > self.pause_timeout: if self.pause_on_client_sleep and time.time() > self.pause_timeout:
RNS.log(f"TX paused for LocalInterface client, dropping outbound packet", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None # TODO: Remove RNS.log(f"TX paused for LocalInterface client, dropping outbound packet", RNS.LOG_DEBUG) # TODO: Remove
return return
if self.online: if self.online:
try: try:
if self.epoll_backend: if self.epoll_backend:
frame = HDLC.frame(data) self.transmit_buffer += bytes([HDLC.FLAG])+HDLC.escape(data)+bytes([HDLC.FLAG])
if self.tx_stalled or not self.transmit_buffer.append(frame, self.tx_hwm): BackboneInterface.tx_ready(self)
self.tx_drops += 1
self.tx_dropped_bytes += len(frame)
RNS.log(f"Egress control dropping outbound frame of {RNS.prettysize(len(frame))} on {self}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
else: BackboneInterface.tx_ready(self)
else: else:
self.writing = True self.writing = True
if self._force_bitrate: if self._force_bitrate:
if not hasattr(self, "send_lock"): self.send_lock = Lock() if not hasattr(self, "send_lock"):
self.send_lock = Lock()
with self.send_lock: with self.send_lock:
# RNS.log(f"Simulating latency of {RNS.prettytime(s)} for {len(data)} bytes", RNS.LOG_EXTREME) # RNS.log(f"Simulating latency of {RNS.prettytime(s)} for {len(data)} bytes", RNS.LOG_EXTREME)
s = len(data) / self.bitrate * 8 s = len(data) / self.bitrate * 8
time.sleep(s) time.sleep(s)
data = HDLC.frame(data) data = bytes([HDLC.FLAG])+HDLC.escape(data)+bytes([HDLC.FLAG])
self.socket.sendall(data) self.socket.sendall(data)
self.writing = False self.writing = False
self.txb += len(data) self.txb += len(data)
@ -246,7 +254,22 @@ class LocalClientInterface(Interface):
self.teardown() self.teardown()
def handle_hdlc(self, data_in): def handle_hdlc(self, data_in):
self.receive_buffer.feed(data_in) self.frame_buffer += data_in
flags_remaining = True
while flags_remaining:
frame_start = self.frame_buffer.find(HDLC.FLAG)
if frame_start != -1:
frame_end = self.frame_buffer.find(HDLC.FLAG, frame_start+1)
if frame_end != -1:
frame = self.frame_buffer[frame_start+1:frame_end]
frame = frame.replace(bytes([HDLC.ESC, HDLC.FLAG ^ HDLC.ESC_MASK]), bytes([HDLC.FLAG]))
frame = frame.replace(bytes([HDLC.ESC, HDLC.ESC ^ HDLC.ESC_MASK]), bytes([HDLC.ESC]))
if len(frame) > RNS.Reticulum.HEADER_MINSIZE: self.process_incoming(frame)
self.frame_buffer = self.frame_buffer[frame_end:]
else: flags_remaining = False
else: flags_remaining = False
def receive(self, data_in): def receive(self, data_in):
try: try:
@ -260,7 +283,8 @@ class LocalClientInterface(Interface):
# there's no other connectivity left to block anyway, it might be # there's no other connectivity left to block anyway, it might be
# unnecessary. # unnecessary.
self.reconnect() self.reconnect()
else: self.teardown(nowarning=True) else:
self.teardown(nowarning=True)
except Exception as e: except Exception as e:
self.online = False self.online = False
@ -272,7 +296,7 @@ class LocalClientInterface(Interface):
def read_loop(self): def read_loop(self):
try: try:
self.receive_buffer.reset() self.frame_buffer = b""
data_in = b"" data_in = b""
while True: while True:
data_in = self.socket.recv(4096) data_in = self.socket.recv(4096)
@ -358,7 +382,6 @@ class LocalServerInterface(Interface):
self.epoll_backend = False self.epoll_backend = False
self.online = False self.online = False
self.clients = 0 self.clients = 0
self.ifac_size = self.DEFAULT_IFAC_SIZE
if socket_path != None and RNS.Reticulum.get_instance().use_af_unix: self.socket_path = f"\0rns/{socket_path}" if socket_path != None and RNS.Reticulum.get_instance().use_af_unix: self.socket_path = f"\0rns/{socket_path}"
else: self.socket_path = None else: self.socket_path = None
@ -406,7 +429,6 @@ class LocalServerInterface(Interface):
self.announce_rate_grace = None self.announce_rate_grace = None
self.announce_rate_penalty = None self.announce_rate_penalty = None
self.HW_MTU = 262144
self.bitrate = 1000*1000*1000 self.bitrate = 1000*1000*1000
self.online = True self.online = True
@ -459,25 +481,17 @@ class LocalServerInterface(Interface):
def process_outgoing(self, data): def process_outgoing(self, data):
pass pass
def received_announce(self, size=0, from_spawned=False): def received_announce(self, from_spawned=False):
if from_spawned: if from_spawned: self.ia_freq_deque.append(time.time())
self.ia_freq_deque.append(time.time())
self.arxb += size
def sent_announce(self, size=0, from_spawned=False): def sent_announce(self, from_spawned=False):
if from_spawned: if from_spawned: self.oa_freq_deque.append(time.time())
self.oa_freq_deque.append(time.time())
self.atxb += size
def received_path_request(self, size=0, from_spawned=False): def received_path_request(self, from_spawned=False):
if from_spawned: if from_spawned: self.ip_freq_deque.append(time.time())
self.ip_freq_deque.append(time.time())
self.prxb += size
def sent_path_request(self, size=0, from_spawned=False): def sent_path_request(self, from_spawned=False):
if from_spawned: if from_spawned: self.op_freq_deque.append(time.time())
self.op_freq_deque.append(time.time())
self.ptxb += size
def __str__(self): def __str__(self):
if self.socket_path: return "Shared Instance["+str(self.socket_path.replace("\0", ""))+"]" if self.socket_path: return "Shared Instance["+str(self.socket_path.replace("\0", ""))+"]"
@ -488,4 +502,5 @@ class LocalInterfaceHandler(socketserver.BaseRequestHandler):
self.callback = callback self.callback = callback
socketserver.BaseRequestHandler.__init__(self, *args, **keys) socketserver.BaseRequestHandler.__init__(self, *args, **keys)
def handle(self): self.callback(handler=self) def handle(self):
self.callback(handler=self)

View file

@ -84,7 +84,6 @@ class PipeInterface(Interface):
self.pipe_is_open = False self.pipe_is_open = False
self.bitrate = PipeInterface.BITRATE_GUESS self.bitrate = PipeInterface.BITRATE_GUESS
self.respawn_delay = respawn_delay self.respawn_delay = respawn_delay
self.shared_medium = True
try: try:
self.open_pipe() self.open_pipe()
@ -120,7 +119,7 @@ class PipeInterface(Interface):
def process_incoming(self, data): def process_incoming(self, data):
if not data: return self.rxb += len(data)
self.owner.inbound(data, self) self.owner.inbound(data, self)

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -258,11 +258,9 @@ class RNodeInterface(Interface):
self.r_stat_tx = None self.r_stat_tx = None
self.r_stat_rssi = None self.r_stat_rssi = None
self.r_stat_snr = None self.r_stat_snr = None
self.r_stat_q = None
self.r_st_alock = None self.r_st_alock = None
self.r_lt_alock = None self.r_lt_alock = None
self.r_random = None self.r_random = None
self.r_modulation = "LoRa"
self.r_airtime_short = 0.0 self.r_airtime_short = 0.0
self.r_airtime_long = 0.0 self.r_airtime_long = 0.0
self.r_channel_load_short = 0.0 self.r_channel_load_short = 0.0
@ -280,7 +278,6 @@ class RNodeInterface(Interface):
self.r_noise_floor = None self.r_noise_floor = None
self.r_interference = None self.r_interference = None
self.r_interference_l = None self.r_interference_l = None
self.reports_phy_stats = True
self.r_battery_state = RNodeInterface.BATTERY_STATE_UNKNOWN self.r_battery_state = RNodeInterface.BATTERY_STATE_UNKNOWN
self.r_battery_percent = 0 self.r_battery_percent = 0
@ -297,7 +294,6 @@ class RNodeInterface(Interface):
self.packet_queue = [] self.packet_queue = []
self.flow_control = flow_control self.flow_control = flow_control
self.shared_medium = True
self.interface_ready = False self.interface_ready = False
self.announce_rate_target = None self.announce_rate_target = None
self.supports_discovery = True self.supports_discovery = True
@ -387,7 +383,6 @@ class RNodeInterface(Interface):
RNS.log(f"Opening BLE connection for {self}...") RNS.log(f"Opening BLE connection for {self}...")
self.timeout = 1250 self.timeout = 1250
if self.ble != None and self.ble.running == False: if self.ble != None and self.ble.running == False:
RNS.log(f"Cleaning up previous BLE connection for {self}...")
self.ble.close() self.ble.close()
self.ble.cleanup() self.ble.cleanup()
self.ble = None self.ble = None
@ -444,11 +439,7 @@ class RNodeInterface(Interface):
ble_detect_timeout = 5.0 ble_detect_timeout = 5.0
detect_time = time.time() detect_time = time.time()
while not self.detected and time.time() < detect_time + ble_detect_timeout: time.sleep(0.1) while not self.detected and time.time() < detect_time + ble_detect_timeout: time.sleep(0.1)
if not self.detected: if not self.detected: RNS.log(f"RNode detect timed out over BLE", RNS.LOG_ERROR)
RNS.log(f"RNode detect timed out over BLE", RNS.LOG_ERROR)
if self.ble:
self.ble.should_run = False
self.ble.must_disconnect = True
else: else:
sleep(0.2) sleep(0.2)
@ -486,7 +477,8 @@ class RNodeInterface(Interface):
self.setLTALock() self.setLTALock()
self.setRadioState(KISS.RADIO_STATE_ON) self.setRadioState(KISS.RADIO_STATE_ON)
if self.use_ble: time.sleep(2) if self.use_ble:
time.sleep(2)
def detect(self): def detect(self):
kiss_command = bytes([KISS.FEND, KISS.CMD_DETECT, KISS.DETECT_REQ, KISS.FEND, KISS.CMD_FW_VERSION, 0x00, KISS.FEND, KISS.CMD_PLATFORM, 0x00, KISS.FEND, KISS.CMD_MCU, 0x00, KISS.FEND]) kiss_command = bytes([KISS.FEND, KISS.CMD_DETECT, KISS.DETECT_REQ, KISS.FEND, KISS.CMD_FW_VERSION, 0x00, KISS.FEND, KISS.CMD_PLATFORM, 0x00, KISS.FEND, KISS.CMD_MCU, 0x00, KISS.FEND])
@ -692,8 +684,10 @@ class RNodeInterface(Interface):
RNS.log("Radio state mismatch", RNS.LOG_ERROR) RNS.log("Radio state mismatch", RNS.LOG_ERROR)
self.validcfg = False self.validcfg = False
if (self.validcfg): return True if (self.validcfg):
else: return False return True
else:
return False
def updateBitrate(self): def updateBitrate(self):
@ -705,8 +699,11 @@ class RNodeInterface(Interface):
self.bitrate = 0 self.bitrate = 0
def process_incoming(self, data): def process_incoming(self, data):
if not data: return self.rxb += len(data)
self.owner.inbound(data, self) self.owner.inbound(data, self)
self.r_stat_rssi = None
self.r_stat_snr = None
def process_outgoing(self,data): def process_outgoing(self,data):
datalen = len(data) datalen = len(data)
@ -1157,17 +1154,19 @@ class RNodeInterface(Interface):
except Exception as e: except Exception as e:
self.online = False self.online = False
if not self.detached:
RNS.log("A serial port error occurred, the contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("A serial port error occurred, the contained exception was: "+str(e), RNS.LOG_ERROR)
RNS.log("The interface "+str(self)+" experienced an unrecoverable error and is now offline.", RNS.LOG_ERROR) RNS.log("The interface "+str(self)+" experienced an unrecoverable error and is now offline.", RNS.LOG_ERROR)
if RNS.Reticulum.panic_on_interface_error: RNS.panic() if RNS.Reticulum.panic_on_interface_error:
RNS.panic()
RNS.log("Reticulum will attempt to reconnect the interface periodically.", RNS.LOG_ERROR) RNS.log("Reticulum will attempt to reconnect the interface periodically.", RNS.LOG_ERROR)
self.online = False self.online = False
try: self.serial.close() try:
except Exception as e: pass self.serial.close()
except Exception as e:
pass
if not self.detached and not self.reconnecting: if not self.detached and not self.reconnecting:
self.reconnect_port() self.reconnect_port()
@ -1193,20 +1192,14 @@ class RNodeInterface(Interface):
self.disable_external_framebuffer() self.disable_external_framebuffer()
self.setRadioState(KISS.RADIO_STATE_OFF) self.setRadioState(KISS.RADIO_STATE_OFF)
self.leave() self.leave()
self.serial.close()
except Exception as e: except Exception as e:
RNS.log(f"An error occurred while detaching {self}: {e}", RNS.LOG_ERROR) RNS.log(f"An error occurred while detaching {self}: {e}", RNS.LOG_ERROR)
if self.use_ble: if self.use_ble: self.ble.close()
self.ble.close() if self.use_tcp:
self.ble.cleanup()
self.ble = None
elif self.use_tcp:
time.sleep(0.5) time.sleep(0.5)
self.tcp.close() self.tcp.close()
else:
self.serial.close()
def should_ingress_limit(self): def should_ingress_limit(self):
return False return False
@ -1304,7 +1297,6 @@ class BLEConnection():
self.must_disconnect = False self.must_disconnect = False
self.connect_job_running = False self.connect_job_running = False
self.device_disappeared = False self.device_disappeared = False
self._windows_paired_addrs = None
import importlib.util import importlib.util
if BLEConnection.bleak == None: if BLEConnection.bleak == None:
@ -1315,8 +1307,8 @@ class BLEConnection():
import asyncio import asyncio
BLEConnection.asyncio = asyncio BLEConnection.asyncio = asyncio
else: else:
RNS.log("Using the RNode interface over BLE requires the \"bleak\" module to be installed.", RNS.LOG_CRITICAL) RNS.log("Using the RNode interface over BLE requires a the \"bleak\" module to be installed.", RNS.LOG_CRITICAL)
RNS.log("You can install it with the command: python3 -m pip install bleak", RNS.LOG_CRITICAL) RNS.log("You can install one with the command: python3 -m pip install bleak", RNS.LOG_CRITICAL)
RNS.panic() RNS.panic()
self.should_run = True self.should_run = True
@ -1332,7 +1324,6 @@ class BLEConnection():
self.should_run = False self.should_run = False
def connection_job(self): def connection_job(self):
self.running = True
while self.should_run: while self.should_run:
if self.ble_device == None: if self.ble_device == None:
self.ble_device = self.find_target_device() self.ble_device = self.find_target_device()
@ -1359,7 +1350,6 @@ class BLEConnection():
self.owner.ble_receive(data) self.owner.ble_receive(data)
self.connected = True self.connected = True
self.device_disappeared = False
self.ble_device = ble_client self.ble_device = ble_client
self.last_client = ble_client self.last_client = ble_client
self.owner.port = str(f"ble://{ble_client.address}") self.owner.port = str(f"ble://{ble_client.address}")
@ -1394,8 +1384,6 @@ class BLEConnection():
def find_target_device(self): def find_target_device(self):
RNS.log(f"Searching for attachable BLE device for {self.owner}...", RNS.LOG_EXTREME) RNS.log(f"Searching for attachable BLE device for {self.owner}...", RNS.LOG_EXTREME)
if RNS.vendor.platformutils.is_windows():
self._windows_paired_addrs = self._get_windows_paired_ble_addresses()
def device_filter(device: self.bleak.backends.device.BLEDevice, adv: self.bleak.backends.scanner.AdvertisementData): def device_filter(device: self.bleak.backends.device.BLEDevice, adv: self.bleak.backends.scanner.AdvertisementData):
if BLEConnection.UART_SERVICE_UUID.lower() in adv.service_uuids: if BLEConnection.UART_SERVICE_UUID.lower() in adv.service_uuids:
if self.device_bonded(device): if self.device_bonded(device):
@ -1427,9 +1415,6 @@ class BLEConnection():
def device_bonded(self, device): def device_bonded(self, device):
try: try:
if self._windows_paired_addrs is not None:
return device.address is not None and device.address.lower() in self._windows_paired_addrs
if hasattr(device, "details"): if hasattr(device, "details"):
if "props" in device.details and "Bonded" in device.details["props"]: if "props" in device.details and "Bonded" in device.details["props"]:
if device.details["props"]["Bonded"] == True: if device.details["props"]["Bonded"] == True:
@ -1440,15 +1425,6 @@ class BLEConnection():
return False return False
def _get_windows_paired_ble_addresses(self):
from winrt.windows.devices.bluetooth import BluetoothLEDevice
from winrt.windows.devices.enumeration import DeviceInformation
async def _query():
selector = BluetoothLEDevice.get_device_selector_from_pairing_state(True)
infos = await DeviceInformation.find_all_async_aqs_filter(selector)
return set(info.id.split("-")[-1].lower() for info in infos)
return self.asyncio.run(_query())
class TCPConnection(): class TCPConnection():
TARGET_PORT = 7633 TARGET_PORT = 7633
CONNECT_TIMEOUT = 5.0 CONNECT_TIMEOUT = 5.0

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -269,7 +269,6 @@ class RNodeMultiInterface(Interface):
self.r_stat_tx = None self.r_stat_tx = None
self.r_stat_rssi = None self.r_stat_rssi = None
self.r_stat_snr = None self.r_stat_snr = None
self.r_stat_q = None
self.r_st_alock = None self.r_st_alock = None
self.r_lt_alock = None self.r_lt_alock = None
self.r_random = None self.r_random = None
@ -277,8 +276,6 @@ class RNodeMultiInterface(Interface):
self.packet_queue = [] self.packet_queue = []
self.interface_ready = False self.interface_ready = False
self.announce_rate_target = None self.announce_rate_target = None
self.reports_phy_stats = True
self.r_modulation = "LoRa"
self.validcfg = True self.validcfg = True
if id_interval != None and id_callsign != None: if id_interval != None and id_callsign != None:
@ -546,25 +543,17 @@ class RNodeMultiInterface(Interface):
if written != len(frame): if written != len(frame):
raise IOError("Serial interface only wrote "+str(written)+" bytes of "+str(len(data))) raise IOError("Serial interface only wrote "+str(written)+" bytes of "+str(len(data)))
def received_announce(self, size=0, from_spawned=False): def received_announce(self, from_spawned=False):
if from_spawned: if from_spawned: self.ia_freq_deque.append(time.time())
self.ia_freq_deque.append(time.time())
self.arxb += size
def sent_announce(self, size=0, from_spawned=False): def sent_announce(self, from_spawned=False):
if from_spawned: if from_spawned: self.oa_freq_deque.append(time.time())
self.oa_freq_deque.append(time.time())
self.atxb += size
def received_path_request(self, size=0, from_spawned=False): def received_path_request(self, from_spawned=False):
if from_spawned: if from_spawned: self.ip_freq_deque.append(time.time())
self.ip_freq_deque.append(time.time())
self.prxb += size
def sent_path_request(self, size=0, from_spawned=False): def sent_path_request(self, from_spawned=False):
if from_spawned: if from_spawned: self.op_freq_deque.append(time.time())
self.op_freq_deque.append(time.time())
self.ptxb += size
def readLoop(self): def readLoop(self):
try: try:
@ -916,7 +905,6 @@ class RNodeMultiInterface(Interface):
if interface != 0: if interface != 0:
self.setRadioState(KISS.RADIO_STATE_OFF, interface) self.setRadioState(KISS.RADIO_STATE_OFF, interface)
self.leave() self.leave()
self.serial.close()
def teardown_subinterfaces(self): def teardown_subinterfaces(self):
for interface in self.subinterfaces: for interface in self.subinterfaces:
@ -1005,7 +993,6 @@ class RNodeSubInterface(Interface):
self.r_premable_time_ms = None self.r_premable_time_ms = None
self.packet_queue = [] self.packet_queue = []
self.shared_medium = True
self.interface_ready = False self.interface_ready = False
self.parent_interface = parent_interface self.parent_interface = parent_interface
self.announce_rate_target = None self.announce_rate_target = None
@ -1129,8 +1116,10 @@ class RNodeSubInterface(Interface):
self.bitrate = 0 self.bitrate = 0
def process_incoming(self, data): def process_incoming(self, data):
if not data: return self.rxb += len(data)
self.owner.inbound(data, self) self.owner.inbound(data, self)
self.r_stat_rssi = None
self.r_stat_snr = None
def process_outgoing(self,data): def process_outgoing(self,data):
if self.online: if self.online:

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -83,7 +83,6 @@ class SerialInterface(Interface):
raise ValueError("No port specified for serial interface") raise ValueError("No port specified for serial interface")
self.HW_MTU = 564 self.HW_MTU = 564
self.shared_medium = True
self.pyserial = serial self.pyserial = serial
self.serial = None self.serial = None
@ -97,7 +96,6 @@ class SerialInterface(Interface):
self.timeout = 100 self.timeout = 100
self.online = False self.online = False
self.bitrate = self.speed self.bitrate = self.speed
self.reconnecting = False
if parity.lower() == "e" or parity.lower() == "even": if parity.lower() == "e" or parity.lower() == "even":
self.parity = serial.PARITY_EVEN self.parity = serial.PARITY_EVEN
@ -107,17 +105,15 @@ class SerialInterface(Interface):
try: try:
self.open_port() self.open_port()
if self.serial.is_open: self.configure_device()
else: raise IOError("Could not open serial port")
except Exception as e: except Exception as e:
RNS.log(f"Could not open serial port for {self}", RNS.LOG_ERROR) RNS.log("Could not open serial port for interface "+str(self), RNS.LOG_ERROR)
RNS.log(f"The contained exception was: {e}", RNS.LOG_ERROR) raise e
RNS.log(f"Reticulum will attempt to bring up this interface periodically", RNS.LOG_ERROR)
if not self.detached and not self.reconnecting: if self.serial.is_open:
thread = threading.Thread(target=self.reconnect_port) self.configure_device()
thread.daemon = True else:
thread.start() raise IOError("Could not open serial port")
def open_port(self): def open_port(self):
RNS.log("Opening serial port "+self.port+"...", RNS.LOG_VERBOSE) RNS.log("Opening serial port "+self.port+"...", RNS.LOG_VERBOSE)
@ -135,11 +131,6 @@ class SerialInterface(Interface):
dsrdtr = False, dsrdtr = False,
) )
def detach(self):
self.detached = True
self.online = False
try: self.serial.close()
except Exception as e: RNS.log(f"Could not close port for {self} at detach: {e}", RNS.LOG_ERROR)
def configure_device(self): def configure_device(self):
sleep(0.5) sleep(0.5)
@ -149,10 +140,12 @@ class SerialInterface(Interface):
self.online = True self.online = True
RNS.log("Serial port "+self.port+" is now open", RNS.LOG_VERBOSE) RNS.log("Serial port "+self.port+" is now open", RNS.LOG_VERBOSE)
def process_incoming(self, data): def process_incoming(self, data):
if not data: return self.rxb += len(data)
self.owner.inbound(data, self) self.owner.inbound(data, self)
def process_outgoing(self,data): def process_outgoing(self,data):
if self.online: if self.online:
data = bytes([HDLC.FLAG])+HDLC.escape(data)+bytes([HDLC.FLAG]) data = bytes([HDLC.FLAG])+HDLC.escape(data)+bytes([HDLC.FLAG])
@ -161,6 +154,7 @@ class SerialInterface(Interface):
if written != len(data): if written != len(data):
raise IOError("Serial interface only wrote "+str(written)+" bytes of "+str(len(data))) raise IOError("Serial interface only wrote "+str(written)+" bytes of "+str(len(data)))
def readLoop(self): def readLoop(self):
try: try:
in_frame = False in_frame = False
@ -204,28 +198,30 @@ class SerialInterface(Interface):
RNS.log("A serial port error occurred, the contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("A serial port error occurred, the contained exception was: "+str(e), RNS.LOG_ERROR)
RNS.log("The interface "+str(self)+" experienced an unrecoverable error and is now offline.", RNS.LOG_ERROR) RNS.log("The interface "+str(self)+" experienced an unrecoverable error and is now offline.", RNS.LOG_ERROR)
if RNS.Reticulum.panic_on_interface_error: RNS.panic() if RNS.Reticulum.panic_on_interface_error:
RNS.panic()
RNS.log("Reticulum will attempt to reconnect the interface periodically.", RNS.LOG_ERROR) RNS.log("Reticulum will attempt to reconnect the interface periodically.", RNS.LOG_ERROR)
self.online = False self.online = False
try: self.serial.close() self.serial.close()
except Exception as e: RNS.log(f"Could not close port for {self}", RNS.LOG_ERROR) self.reconnect_port()
if not self.detached and not self.reconnecting: self.reconnect_port()
def reconnect_port(self): def reconnect_port(self):
self.reconnecting = True while not self.online:
while not self.online and not self.detached:
try: try:
time.sleep(5) time.sleep(5)
RNS.log("Attempting to reconnect serial port "+str(self.port)+" for "+str(self)+"...", RNS.LOG_VERBOSE) RNS.log("Attempting to reconnect serial port "+str(self.port)+" for "+str(self)+"...", RNS.LOG_VERBOSE)
self.open_port() self.open_port()
if self.serial.is_open: self.configure_device() if self.serial.is_open:
except Exception as e: RNS.log("Error while reconnecting port, the contained exception was: "+str(e), RNS.LOG_ERROR) self.configure_device()
except Exception as e:
RNS.log("Error while reconnecting port, the contained exception was: "+str(e), RNS.LOG_ERROR)
self.reconnecting = False RNS.log("Reconnected serial port for "+str(self))
if self.online: RNS.log("Reconnected serial port for "+str(self))
def should_ingress_limit(self): return False def should_ingress_limit(self):
return False
def __str__(self): return "SerialInterface["+self.name+"]" def __str__(self):
return "SerialInterface["+self.name+"]"

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -126,8 +126,6 @@ class TCPClientInterface(Interface):
self.writing = False self.writing = False
self.online = False self.online = False
self.detached = False self.detached = False
self.tunnels_allowed = True
self.lossless_medium = True
self.kiss_framing = kiss_framing self.kiss_framing = kiss_framing
self.i2p_tunneled = i2p_tunneled self.i2p_tunneled = i2p_tunneled
self.mode = RNS.Interfaces.Interface.Interface.MODE_FULL self.mode = RNS.Interfaces.Interface.Interface.MODE_FULL
@ -161,8 +159,8 @@ class TCPClientInterface(Interface):
else: else:
self.connect_timeout = TCPClientInterface.INITIAL_CONNECT_TIMEOUT self.connect_timeout = TCPClientInterface.INITIAL_CONNECT_TIMEOUT
def final_init(self): if TCPClientInterface.SYNCHRONOUS_START:
if TCPClientInterface.SYNCHRONOUS_START: self.initial_connect() self.initial_connect()
else: else:
thread = threading.Thread(target=self.initial_connect) thread = threading.Thread(target=self.initial_connect)
thread.daemon = True thread.daemon = True
@ -177,6 +175,8 @@ class TCPClientInterface(Interface):
thread = threading.Thread(target=self.read_loop) thread = threading.Thread(target=self.read_loop)
thread.daemon = True thread.daemon = True
thread.start() thread.start()
if not self.kiss_framing:
self.wants_tunnel = True
def set_timeouts_linux(self): def set_timeouts_linux(self):
if not self.i2p_tunneled: if not self.i2p_tunneled:
@ -208,7 +208,6 @@ class TCPClientInterface(Interface):
def detach(self): def detach(self):
self.online = False self.online = False
self.detached = True
if self.socket != None: if self.socket != None:
if hasattr(self.socket, "close"): if hasattr(self.socket, "close"):
if callable(self.socket.close): if callable(self.socket.close):
@ -228,20 +227,6 @@ class TCPClientInterface(Interface):
self.socket = None self.socket = None
def early_synthesis(self, packet):
if not self.detached:
if not packet.packed: packet.pack()
if self.ifac_identity: raw = RNS.Transport.handle_outgoing_ifac(self, packet.raw);
else: raw = packet.raw
if self.kiss_framing: frame = bytes([KISS.FEND])+bytes([KISS.CMD_DATA])+KISS.escape(raw)+bytes([KISS.FEND])
else: frame = bytes([HDLC.FLAG])+HDLC.escape(raw)+bytes([HDLC.FLAG])
self.socket.sendall(frame)
self.txb += len(frame)
if hasattr(self, "parent_interface") and self.parent_interface != None:
self.parent_interface.txb += len(frame)
def connect(self, initial=False): def connect(self, initial=False):
try: try:
if initial: if initial:
@ -256,13 +241,6 @@ class TCPClientInterface(Interface):
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.socket.connect(target_address) self.socket.connect(target_address)
self.socket.settimeout(None) self.socket.settimeout(None)
if not self.kiss_framing:
self.wants_tunnel = True
self.remote_transport = None
self.synthesis_received = False
RNS.Transport.synthesize_tunnel(self)
self.online = True self.online = True
if initial: if initial:
@ -294,7 +272,7 @@ class TCPClientInterface(Interface):
if not self.reconnecting: if not self.reconnecting:
self.reconnecting = True self.reconnecting = True
attempts = 0 attempts = 0
while not self.online and not self.detached: while not self.online:
time.sleep(TCPClientInterface.RECONNECT_WAIT) time.sleep(TCPClientInterface.RECONNECT_WAIT)
attempts += 1 attempts += 1
@ -303,7 +281,9 @@ class TCPClientInterface(Interface):
self.teardown() self.teardown()
break break
try: self.connect() try:
self.connect()
except Exception as e: except Exception as e:
RNS.log("Connection attempt for "+str(self)+" failed: "+str(e), RNS.LOG_DEBUG) RNS.log("Connection attempt for "+str(self)+" failed: "+str(e), RNS.LOG_DEBUG)
@ -314,14 +294,19 @@ class TCPClientInterface(Interface):
thread = threading.Thread(target=self.read_loop) thread = threading.Thread(target=self.read_loop)
thread.daemon = True thread.daemon = True
thread.start() thread.start()
if not self.kiss_framing:
RNS.Transport.synthesize_tunnel(self)
else: else:
RNS.log("Attempt to reconnect on a non-initiator TCP interface. This should not happen.", RNS.LOG_ERROR) RNS.log("Attempt to reconnect on a non-initiator TCP interface. This should not happen.", RNS.LOG_ERROR)
raise IOError("Attempt to reconnect on a non-initiator TCP interface") raise IOError("Attempt to reconnect on a non-initiator TCP interface")
def process_incoming(self, data): def process_incoming(self, data):
if not data: return
if self.online and not self.detached: if self.online and not self.detached:
self.rxb += len(data)
if hasattr(self, "parent_interface") and self.parent_interface != None:
self.parent_interface.rxb += len(data)
self.owner.inbound(data, self) self.owner.inbound(data, self)
def process_outgoing(self, data): def process_outgoing(self, data):
@ -348,13 +333,6 @@ class TCPClientInterface(Interface):
RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
self.teardown() self.teardown()
def check_frame_len(self, frame_len):
if frame_len <= RNS.Reticulum.HEADER_MINSIZE: return False
elif frame_len > self.HW_MTU + (self.ifac_size or 0): return False
else: return True
def invalid_frame(self, frame_len):
RNS.log(f"Invalid HDLC frame of {RNS.prettysize(frame_len)} received on {self}, dropping frame", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
def read_loop(self): def read_loop(self):
try: try:
@ -411,18 +389,12 @@ class TCPClientInterface(Interface):
frame = frame_buffer[frame_start+1:frame_end] frame = frame_buffer[frame_start+1:frame_end]
frame = frame.replace(bytes([HDLC.ESC, HDLC.FLAG ^ HDLC.ESC_MASK]), bytes([HDLC.FLAG])) frame = frame.replace(bytes([HDLC.ESC, HDLC.FLAG ^ HDLC.ESC_MASK]), bytes([HDLC.FLAG]))
frame = frame.replace(bytes([HDLC.ESC, HDLC.ESC ^ HDLC.ESC_MASK]), bytes([HDLC.ESC])) frame = frame.replace(bytes([HDLC.ESC, HDLC.ESC ^ HDLC.ESC_MASK]), bytes([HDLC.ESC]))
frame_len = len(frame) if len(frame) > RNS.Reticulum.HEADER_MINSIZE:
if frame_len != 0: self.process_incoming(frame)
if self.check_frame_len(frame_len): self.process_incoming(frame)
else: self.invalid_frame(len(frame))
frame_buffer = frame_buffer[frame_end:] frame_buffer = frame_buffer[frame_end:]
else: else:
if len(frame_buffer) > self.HW_MTU*2: frame_buffer = b""
flags_remaining = False flags_remaining = False
else: else:
frame_buffer = b""
flags_remaining = False flags_remaining = False
else: else:
@ -441,7 +413,7 @@ class TCPClientInterface(Interface):
self.online = False self.online = False
RNS.log("An interface error occurred for "+str(self)+", the contained exception was: "+str(e), RNS.LOG_WARNING) RNS.log("An interface error occurred for "+str(self)+", the contained exception was: "+str(e), RNS.LOG_WARNING)
if self.initiator and not self.detached: if self.initiator:
RNS.log("Attempting to reconnect...", RNS.LOG_WARNING) RNS.log("Attempting to reconnect...", RNS.LOG_WARNING)
self.reconnect() self.reconnect()
else: else:
@ -467,9 +439,12 @@ class TCPClientInterface(Interface):
if not self.initiator: if not self.initiator:
RNS.Transport.remove_interface(self) RNS.Transport.remove_interface(self)
def __str__(self): def __str__(self):
if ":" in self.target_ip: ip_str = f"[{self.target_ip}]" if ":" in self.target_ip:
else: ip_str = f"{self.target_ip}" ip_str = f"[{self.target_ip}]"
else:
ip_str = f"{self.target_ip}"
return "TCPInterface["+str(self.name)+"/"+ip_str+":"+str(self.target_port)+"]" return "TCPInterface["+str(self.name)+"/"+ip_str+":"+str(self.target_port)+"]"
@ -546,8 +521,6 @@ class TCPServerInterface(Interface):
self.OUT = False self.OUT = False
self.name = name self.name = name
self.detached = False self.detached = False
self.tunnels_allowed = True
self.lossless_medium = True
self.i2p_tunneled = i2p_tunneled self.i2p_tunneled = i2p_tunneled
self.mode = RNS.Interfaces.Interface.Interface.MODE_FULL self.mode = RNS.Interfaces.Interface.Interface.MODE_FULL
@ -620,9 +593,6 @@ class TCPServerInterface(Interface):
spawned_interface.ic_pr_burst_freq_new = self.ic_pr_burst_freq_new spawned_interface.ic_pr_burst_freq_new = self.ic_pr_burst_freq_new
spawned_interface.ic_pr_burst_freq = self.ic_pr_burst_freq spawned_interface.ic_pr_burst_freq = self.ic_pr_burst_freq
spawned_interface._destination_cost = self._destination_cost
spawned_interface._pr_cost = self._pr_cost
spawned_interface.target_ip = handler.client_address[0] spawned_interface.target_ip = handler.client_address[0]
spawned_interface.target_port = str(handler.client_address[1]) spawned_interface.target_port = str(handler.client_address[1])
spawned_interface.parent_interface = self spawned_interface.parent_interface = self
@ -653,12 +623,6 @@ class TCPServerInterface(Interface):
spawned_interface.announce_rate_grace = self.announce_rate_grace spawned_interface.announce_rate_grace = self.announce_rate_grace
spawned_interface.announce_rate_penalty = self.announce_rate_penalty spawned_interface.announce_rate_penalty = self.announce_rate_penalty
spawned_interface.mode = self.mode spawned_interface.mode = self.mode
spawned_interface.gravity = self.gravity
spawned_interface.hop_limiter = self.hop_limiter
spawned_interface.recursive_prs = self.recursive_prs
spawned_interface.announces_from_internal = self.announces_from_internal
spawned_interface.announces_to_internal = self.announces_to_internal
spawned_interface.announce_cap = self.announce_cap
spawned_interface.HW_MTU = self.HW_MTU spawned_interface.HW_MTU = self.HW_MTU
spawned_interface.online = True spawned_interface.online = True
RNS.log("Spawned new TCPClient Interface: "+str(spawned_interface), RNS.LOG_VERBOSE) RNS.log("Spawned new TCPClient Interface: "+str(spawned_interface), RNS.LOG_VERBOSE)
@ -668,25 +632,17 @@ class TCPServerInterface(Interface):
self.spawned_interfaces.append(spawned_interface) self.spawned_interfaces.append(spawned_interface)
spawned_interface.read_loop() spawned_interface.read_loop()
def received_announce(self, size=0, from_spawned=False): def received_announce(self, from_spawned=False):
if from_spawned: if from_spawned: self.ia_freq_deque.append(time.time())
self.ia_freq_deque.append(time.time())
self.arxb += size
def sent_announce(self, size=0, from_spawned=False): def sent_announce(self, from_spawned=False):
if from_spawned: if from_spawned: self.oa_freq_deque.append(time.time())
self.oa_freq_deque.append(time.time())
self.atxb += size
def received_path_request(self, size=0, from_spawned=False): def received_path_request(self, from_spawned=False):
if from_spawned: if from_spawned: self.ip_freq_deque.append(time.time())
self.ip_freq_deque.append(time.time())
self.prxb += size
def sent_path_request(self, size=0, from_spawned=False): def sent_path_request(self, from_spawned=False):
if from_spawned: if from_spawned: self.op_freq_deque.append(time.time())
self.op_freq_deque.append(time.time())
self.ptxb += size
def process_outgoing(self, data): def process_outgoing(self, data):
pass pass
@ -708,8 +664,10 @@ class TCPServerInterface(Interface):
def __str__(self): def __str__(self):
if ":" in self.bind_ip: ip_str = f"[{self.bind_ip}]" if ":" in self.bind_ip:
else: ip_str = f"{self.bind_ip}" ip_str = f"[{self.bind_ip}]"
else:
ip_str = f"{self.bind_ip}"
return "TCPServerInterface["+self.name+"/"+ip_str+":"+str(self.bind_port)+"]" return "TCPServerInterface["+self.name+"/"+ip_str+":"+str(self.bind_port)+"]"

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -78,7 +78,6 @@ class UDPInterface(Interface):
self.name = name self.name = name
self.online = False self.online = False
self.bitrate = UDPInterface.BITRATE_GUESS self.bitrate = UDPInterface.BITRATE_GUESS
self.shared_medium = True
if device != None: if device != None:
if bindip == None: if bindip == None:
@ -113,17 +112,9 @@ class UDPInterface(Interface):
self.forward_ip = forwardip self.forward_ip = forwardip
self.forward_port = forwardport self.forward_port = forwardport
def detach(self):
self.detached = True
self.online = False
if self.server:
try:
self.server.server_close()
self.server.shutdown()
except Exception as e: RNS.log(f"Could not shut down UDP listener for {self}: {e}", RNS.LOG_ERROR)
def process_incoming(self, data): def process_incoming(self, data):
if not data: return self.rxb += len(data)
self.owner.inbound(data, self) self.owner.inbound(data, self)
def process_outgoing(self,data): def process_outgoing(self,data):

View file

@ -131,8 +131,8 @@ class WDCL():
self.wdcl_connected = False self.wdcl_connected = False
self.reconnecting = False self.reconnecting = False
self.frame_queue = deque() self.frame_queue = deque()
if not self.as_interface:
if not self.as_interface: self.id = RNS.Identity.full_hash(port.hwid.encode("utf-8")) self.id = RNS.Identity.full_hash(port.hwid.encode("utf-8"))
if self.as_interface: if self.as_interface:
try: try:
@ -168,10 +168,19 @@ class WDCL():
self.owner.wlog(f"Opening serial port {self.port.device}...") self.owner.wlog(f"Opening serial port {self.port.device}...")
target_port = self.port.device target_port = self.port.device
self.serial = self.pyserial.Serial(port = target_port, baudrate = self.speed, self.serial = self.pyserial.Serial(
bytesize = self.databits, parity = self.parity, stopbits = self.stopbits, port = target_port,
xonxoff = False, rtscts = False, timeout = 0.250, inter_byte_timeout = None, baudrate = self.speed,
write_timeout = None, dsrdtr = False) bytesize = self.databits,
parity = self.parity,
stopbits = self.stopbits,
xonxoff = False,
rtscts = False,
timeout = 0.250,
inter_byte_timeout = None,
write_timeout = None,
dsrdtr = False)
else: else:
if self.port != None: if self.port != None:
# Get device parameters # Get device parameters
@ -189,9 +198,19 @@ class WDCL():
from usbserial4a.cdcacmserial4a import CdcAcmSerial from usbserial4a.cdcacmserial4a import CdcAcmSerial
proxy = CdcAcmSerial proxy = CdcAcmSerial
self.serial = proxy(self.port, baudrate = self.speed, bytesize = self.databits, self.serial = proxy(
parity = self.parity, stopbits = self.stopbits, xonxoff = False, self.port,
rtscts = False, timeout = None, inter_byte_timeout = None, dsrdtr = False) baudrate = self.speed,
bytesize = self.databits,
parity = self.parity,
stopbits = self.stopbits,
xonxoff = False,
rtscts = False,
timeout = None,
inter_byte_timeout = None,
# write_timeout = wtimeout,
dsrdtr = False,
)
if vid == 0x0403: if vid == 0x0403:
# Hardware parameters for FTDI devices @ 115200 baud # Hardware parameters for FTDI devices @ 115200 baud
@ -249,6 +268,7 @@ class WDCL():
self.online = True self.online = True
def process_incoming(self, data): def process_incoming(self, data):
self.rxb += len(data)
if self.device: if self.device:
while len(self.frame_queue): self.device.incoming_frame(self.frame_queue.pop()) while len(self.frame_queue): self.device.incoming_frame(self.frame_queue.pop())
self.device.incoming_frame(data) self.device.incoming_frame(data)
@ -279,9 +299,10 @@ class WDCL():
frame = frame.replace(bytes([HDLC.ESC, HDLC.ESC ^ HDLC.ESC_MASK]), bytes([HDLC.ESC])) frame = frame.replace(bytes([HDLC.ESC, HDLC.ESC ^ HDLC.ESC_MASK]), bytes([HDLC.ESC]))
if len(frame) > WDCL.HEADER_MINSIZE: self.process_incoming(frame) if len(frame) > WDCL.HEADER_MINSIZE: self.process_incoming(frame)
self.frame_buffer = self.frame_buffer[frame_end:] self.frame_buffer = self.frame_buffer[frame_end:]
else:
else: flags_remaining = False flags_remaining = False
else: flags_remaining = False else:
flags_remaining = False
except Exception as e: except Exception as e:
self.online = False self.online = False
@ -338,7 +359,6 @@ class Evt():
ET_MSG = 0x0000 ET_MSG = 0x0000
ET_SYSTEM_BOOT = 0x0001 ET_SYSTEM_BOOT = 0x0001
ET_CORE_INIT = 0x0002 ET_CORE_INIT = 0x0002
ET_BOARD_INIT = 0x0003
ET_DRV_UART_INIT = 0x1000 ET_DRV_UART_INIT = 0x1000
ET_DRV_USB_CDC_INIT = 0x1010 ET_DRV_USB_CDC_INIT = 0x1010
ET_DRV_USB_CDC_HOST_AVAIL = 0x1011 ET_DRV_USB_CDC_HOST_AVAIL = 0x1011
@ -408,7 +428,6 @@ class Evt():
event_descriptions = { event_descriptions = {
ET_SYSTEM_BOOT: "System boot", ET_SYSTEM_BOOT: "System boot",
ET_CORE_INIT: "Core initialization", ET_CORE_INIT: "Core initialization",
ET_BOARD_INIT: "Board hardware initialization",
ET_DRV_UART_INIT: "UART driver initialization", ET_DRV_UART_INIT: "UART driver initialization",
ET_DRV_USB_CDC_INIT: "USB CDC driver initialization", ET_DRV_USB_CDC_INIT: "USB CDC driver initialization",
ET_DRV_USB_CDC_HOST_AVAIL: "USB CDC host became available", ET_DRV_USB_CDC_HOST_AVAIL: "USB CDC host became available",
@ -756,8 +775,8 @@ class WeaveDevice():
self.cpu_load = frame.data[0] self.cpu_load = frame.data[0]
self.capture_stats_cpu() self.capture_stats_cpu()
elif frame.event == Evt.ET_STAT_MEMORY: elif frame.event == Evt.ET_STAT_MEMORY:
self.memory_free = int.from_bytes(frame.data[:4], "big") self.memory_free = int.from_bytes(frame.data[:4])
self.memory_total = int.from_bytes(frame.data[4:], "big") self.memory_total = int.from_bytes(frame.data[4:])
self.memory_used = self.memory_total-self.memory_free self.memory_used = self.memory_total-self.memory_free
self.memory_used_pct = round((self.memory_used/self.memory_total)*100, 2) self.memory_used_pct = round((self.memory_used/self.memory_total)*100, 2)
self.capture_stats_memory() self.capture_stats_memory()
@ -933,16 +952,8 @@ class WeaveInterface(Interface):
spawned_interface.ic_burst_penalty = self.ic_burst_penalty spawned_interface.ic_burst_penalty = self.ic_burst_penalty
spawned_interface.ic_held_release_interval = self.ic_held_release_interval spawned_interface.ic_held_release_interval = self.ic_held_release_interval
spawned_interface.announces_to_internal = self.announces_to_internal
spawned_interface.announces_from_internal = self.announces_from_internal
spawned_interface.announce_cap = self.announce_cap
spawned_interface._destination_cost = self._destination_cost
spawned_interface._pr_cost = self._pr_cost
spawned_interface.parent_interface = self spawned_interface.parent_interface = self
spawned_interface.bitrate = self.bitrate spawned_interface.bitrate = self.bitrate
spawned_interface.recursive_prs = self.recursive_prs
spawned_interface.ifac_size = self.ifac_size spawned_interface.ifac_size = self.ifac_size
spawned_interface.ifac_netname = self.ifac_netname spawned_interface.ifac_netname = self.ifac_netname
@ -967,18 +978,9 @@ class WeaveInterface(Interface):
spawned_interface.announce_rate_target = self.announce_rate_target spawned_interface.announce_rate_target = self.announce_rate_target
spawned_interface.announce_rate_grace = self.announce_rate_grace spawned_interface.announce_rate_grace = self.announce_rate_grace
spawned_interface.announce_rate_penalty = self.announce_rate_penalty spawned_interface.announce_rate_penalty = self.announce_rate_penalty
spawned_interface.egress_control = self.egress_control
spawned_interface.ec_pr_freq = self.ec_pr_freq
spawned_interface.ic_pr_burst_freq_new = self.ic_pr_burst_freq_new
spawned_interface.ic_pr_burst_freq = self.ic_pr_burst_freq
spawned_interface.mode = self.mode spawned_interface.mode = self.mode
spawned_interface.gravity = self.gravity
spawned_interface.hop_limiter = self.hop_limiter
spawned_interface.HW_MTU = self.HW_MTU spawned_interface.HW_MTU = self.HW_MTU
spawned_interface._online = True spawned_interface._online = True
RNS.Transport.add_interface(spawned_interface) RNS.Transport.add_interface(spawned_interface)
if endpoint_addr in self.spawned_interfaces: if endpoint_addr in self.spawned_interfaces:
self.spawned_interfaces[endpoint_addr].detach() self.spawned_interfaces[endpoint_addr].detach()
@ -1060,6 +1062,8 @@ class WeaveInterfacePeer(Interface):
self.owner.refresh_peer(self.endpoint_addr) self.owner.refresh_peer(self.endpoint_addr)
self.owner.mif_deque.append(data_hash) self.owner.mif_deque.append(data_hash)
self.owner.mif_deque_times.append([data_hash, time.time()]) self.owner.mif_deque_times.append([data_hash, time.time()])
self.rxb += len(data)
self.owner.rxb += len(data)
self.owner.owner.inbound(data, self) self.owner.owner.inbound(data, self)
def process_outgoing(self, data): def process_outgoing(self, data):

View file

@ -36,7 +36,5 @@ import RNS.Interfaces.util.netinfo as netinfo
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py") py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc") pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
so_modules = glob.glob(os.path.dirname(__file__)+"/*.so") modules = py_modules+pyc_modules
pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd") __all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
modules = py_modules+pyc_modules+so_modules+pyd_modules
__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))

View file

@ -1,115 +0,0 @@
# Reticulum License
#
# Copyright (c) 2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# - The Software shall not be used in any kind of system which includes amongst
# its functions the ability to purposefully do harm to human beings.
#
# - The Software shall not be used, directly or indirectly, in the creation of
# an artificial intelligence, machine learning or language model training
# dataset, including but not limited to any use that contributes to the
# training or development of such a model or algorithm.
#
# - The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
class HDLC():
FLAG = 0x7E
ESC = 0x7D
ESC_MASK = 0x20
FLAG_B = bytes([FLAG])
ESC_B = bytes([ESC])
ESCPD_FLAG = bytes([ESC, FLAG ^ ESC_MASK])
ESCPD_ESC = bytes([ESC, ESC ^ ESC_MASK])
@staticmethod
def escape(data):
data = data.replace(HDLC.ESC_B, HDLC.ESCPD_ESC)
data = data.replace(HDLC.FLAG_B, HDLC.ESCPD_FLAG)
return data
@staticmethod
def unescape(frame):
frame = frame.replace(HDLC.ESCPD_FLAG, HDLC.FLAG_B)
frame = frame.replace(HDLC.ESCPD_ESC, HDLC.ESC_B)
return frame
@staticmethod
def frame(data):
esc = HDLC.escape(data)
frame = bytearray(len(esc)+2)
frame[0] = HDLC.FLAG
frame[1:-1] = esc
frame[-1] = HDLC.FLAG
return frame
class ReceiveBuffer():
def __init__(self, mtu, min_frame_len, max_frame_len=None, on_frame=None, on_invalid=None):
self._mtu = mtu
self._min_frame_len = min_frame_len
self._max_frame_len = max_frame_len
self._on_frame = on_frame
self._on_invalid = on_invalid
self._buf = bytearray()
self._off = 0
def mtu(self):
return self._mtu() if callable(self._mtu) else self._mtu
def max(self):
m = self._max_frame_len
return m() if callable(m) else m
def feed(self, data):
if not data: return
self._buf.extend(data)
buf = self._buf
while True:
frame_start = buf.find(HDLC.FLAG, self._off)
if frame_start == -1:
self.reset()
return
frame_end = buf.find(HDLC.FLAG, frame_start+1)
if frame_end == -1:
if len(buf)-self._off > self.mtu()*2: self.reset()
return
frame = HDLC.unescape(bytes(buf[frame_start+1:frame_end]))
frame_len = len(frame)
if frame_len > 0:
max_len = self.max()
if frame_len > self._min_frame_len and (max_len is None or frame_len <= max_len):
if self._on_frame is not None: self._on_frame(frame)
elif self._on_invalid is not None: self._on_invalid(frame_len)
self._off = frame_end
if self._off > 0 and self._off >= len(buf) // 2:
del buf[:self._off]
self._off = 0
# Drops any partial frames from the buffer and
# resets assembly offset
def reset(self):
self._buf.clear()
self._off = 0
# Buffered bytes currently unconsumed
def __len__(self): return len(self._buf)-self._off

View file

@ -1,171 +0,0 @@
# Reticulum License
#
# Copyright (c) 2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# - The Software shall not be used in any kind of system which includes amongst
# its functions the ability to purposefully do harm to human beings.
#
# - The Software shall not be used, directly or indirectly, in the creation of
# an artificial intelligence, machine learning or language model training
# dataset, including but not limited to any use that contributes to the
# training or development of such a model or algorithm.
#
# - The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from collections import deque
import threading
# Zero-copy coalesced transmit buffer for arbitrary
# frame sizes. It's even almost 100% lock-free.
class TransmitBuffer:
COALESCE_TARGET = 65536
def __init__(self):
self._tail_lock = threading.Lock()
# Producer-owned state
self._cur = None
self._cur_frames = 0
# Consumer-owned state
self._head_offset = 0
# Shared chunk queue. Producer appends right, consumer
# pops from left. Appended chunks are immutable.
self._chunks = deque()
# Accounting counters
# Producer-owned
self._tx_total = 0 # Total bytes appended
self._tx_visible = 0 # Total bytes pushed to the chunk queue
self._tx_frames = 0 # Total frames appended
# Consumer-owned
self._tx_sent = 0 # Total bytes sent to the kernel
self._tx_frames_sent = 0 # Total frames fully sent
# Queue a finalized and complete on-wire frame.
# Executed by producer side only.
def append(self, frame, limit=None):
with self._tail_lock:
if limit is not None and (self._tx_total - self._tx_sent) + len(frame) > limit: return False
# Allocate separate chunks to frames
# larger than the coalescing target.
if len(frame) >= self.COALESCE_TARGET:
self._flush()
self._chunks.append((frame, 1))
self._tx_visible += len(frame)
self._tx_total += len(frame)
self._tx_frames += 1
else:
if self._cur is not None and len(self._cur) + len(frame) > self.COALESCE_TARGET: self._flush()
if self._cur is None:
self._cur = bytearray()
self._cur_frames = 0
self._cur.extend(frame)
self._cur_frames += 1
self._tx_total += len(frame)
self._tx_frames += 1
if not self._chunks:
# Queue is idle. Make the frame visible immediately so
# sparse traffic does not incur coalescing latency.
self._flush()
return True
def _flush(self):
if self._cur is not None and len(self._cur) > 0:
self._chunks.append((self._cur, self._cur_frames))
self._tx_visible += len(self._cur)
self._cur = None
self._cur_frames = 0
# Make producer-side coalescing tail visible to the consumer.
# Called by producer when it has finished producing a burst,
# and wants to make the remaining bytes transmittable. Safe to
# call after every append.
def flush(self):
with self._tail_lock: self._flush()
# Drain buffered frames to a socket. Called by consumer. If the
# chunk queue is drained completely, any potential producer-side
# coalescing tail is released under the tail lock, and drained in
# the same cycle. If you possess wizardry for doing this without
# the lock, I am all ears.
def drain_to(self, socket):
# TODO: At *some* point (no, not now), replace with a
# mechanism for send_msg to immediately drain all
# chunks directly to the kernel.
total = 0
whole = True
while whole:
if not self._chunks:
with self._tail_lock: self._flush()
if not self._chunks: break
chunk, _ = self._chunks[0]
remaining = len(chunk) - self._head_offset
if remaining <= 0:
self._pop_head()
continue
try: written = socket.send(memoryview(chunk)[self._head_offset:])
except BlockingIOError: break
except InterruptedError: continue
if written <= 0: break
total += written
self._head_offset += written
self._tx_sent += written
if self._head_offset >= len(chunk): self._pop_head()
else: whole = False
return total
def _pop_head(self):
chunk, frames = self._chunks.popleft()
self._tx_frames_sent += frames
self._head_offset = 0
# Total number of bytes currently buffered, including the
# producer's in-progress coalescing chunk.
def __len__(self): return self._tx_total-self._tx_sent
# The number of bytes actually ready to send, and will be
# visible to the consumer, that is, what can be drained.
@property
def sendable(self): return self._tx_visible-self._tx_sent
# The number of complete frames currently buffere, including
# ones contained in a potential in-progress coalescing chunk.
@property
def frames_buffered(self): return self._tx_frames-self._tx_frames_sent
# Number of chunks currently queued and visible to the consumer
@property
def chunks_buffered(self): return len(self._chunks)
def __repr__(self): return f"<TransmitBuffer buffered={len(self)} sendable={self.sendable} frames={self.frames_buffered} chunks={self.chunks_buffered}>"

View file

@ -3,7 +3,5 @@ import glob
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py") py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc") pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
so_modules = glob.glob(os.path.dirname(__file__)+"/*.so") modules = py_modules+pyc_modules
pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd") __all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
modules = py_modules+pyc_modules+so_modules+pyd_modules
__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -28,8 +28,8 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE. # SOFTWARE.
from RNS.Cryptography import X25519PrivateKey, X25519PublicKey, Ed25519PrivateKey, Ed25519PublicKey, Token from RNS.Cryptography import X25519PrivateKey, X25519PublicKey, Ed25519PrivateKey, Ed25519PublicKey
from RNS.Stamps import stamp_valid, stamp_value, generate_stamped_key, cancel_work from RNS.Cryptography import Token
from RNS.Channel import Channel, LinkChannelOutlet from RNS.Channel import Channel, LinkChannelOutlet
from time import sleep from time import sleep
@ -70,7 +70,7 @@ class Link:
ECPUBSIZE = 32+32 ECPUBSIZE = 32+32
KEYSIZE = 32 KEYSIZE = 32
MDU = math.floor((RNS.Reticulum.MTU-RNS.Reticulum.IFAC_MIN_SIZE-RNS.Reticulum.HEADER_MINSIZE-RNS.Identity.TOKEN_OVERHEAD)/RNS.Identity.AES256_BLOCKSIZE)*RNS.Identity.AES256_BLOCKSIZE - 1 MDU = math.floor((RNS.Reticulum.MTU-RNS.Reticulum.IFAC_MIN_SIZE-RNS.Reticulum.HEADER_MINSIZE-RNS.Identity.TOKEN_OVERHEAD)/RNS.Identity.AES128_BLOCKSIZE)*RNS.Identity.AES128_BLOCKSIZE - 1
ESTABLISHMENT_TIMEOUT_PER_HOP = RNS.Reticulum.DEFAULT_PER_HOP_TIMEOUT ESTABLISHMENT_TIMEOUT_PER_HOP = RNS.Reticulum.DEFAULT_PER_HOP_TIMEOUT
""" """
@ -116,7 +116,6 @@ class Link:
TIMEOUT = 0x01 TIMEOUT = 0x01
INITIATOR_CLOSED = 0x02 INITIATOR_CLOSED = 0x02
DESTINATION_CLOSED = 0x03 DESTINATION_CLOSED = 0x03
STAMP_FAILURE = 0x04
ACCEPT_NONE = 0x00 ACCEPT_NONE = 0x00
ACCEPT_APP = 0x01 ACCEPT_APP = 0x01
@ -187,35 +186,29 @@ class Link:
def validate_request(owner, data, packet): def validate_request(owner, data, packet):
if len(data) == Link.ECPUBSIZE or len(data) == Link.ECPUBSIZE+Link.LINK_MTU_SIZE: if len(data) == Link.ECPUBSIZE or len(data) == Link.ECPUBSIZE+Link.LINK_MTU_SIZE:
try: try:
peer_pub_bytes = data[:Link.ECPUBSIZE//2] link = Link(owner = owner, peer_pub_bytes=data[:Link.ECPUBSIZE//2], peer_sig_pub_bytes=data[Link.ECPUBSIZE//2:Link.ECPUBSIZE])
peer_sig_pub_bytes = data[Link.ECPUBSIZE//2:Link.ECPUBSIZE]
stamp_material = owner.hash + peer_sig_pub_bytes
value = stamp_value(stamp_material, peer_pub_bytes)
if RNS.Transport.LR_STAMPS_ENFORCED and not stamp_valid(peer_pub_bytes, owner.get_link_cost(), stamp_material): raise ValueError(f"Insufficient link request stamp value {value}")
link = Link(owner=owner, peer_pub_bytes=peer_pub_bytes, peer_sig_pub_bytes=peer_sig_pub_bytes)
link.set_link_id(packet) link.set_link_id(packet)
link.stamp_cost = value
link.stamp_status = True
if len(data) == Link.ECPUBSIZE+Link.LINK_MTU_SIZE: if len(data) == Link.ECPUBSIZE+Link.LINK_MTU_SIZE:
RNS.log("Link request includes MTU signalling", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None RNS.log("Link request includes MTU signalling", RNS.LOG_DEBUG) # TODO: Remove debug
try: link.mtu = Link.mtu_from_lr_packet(packet) or RNS.Reticulum.MTU try:
link.mtu = Link.mtu_from_lr_packet(packet) or Reticulum.MTU
except Exception as e: except Exception as e:
RNS.trace_exception(e) RNS.trace_exception(e)
link.mtu = RNS.Reticulum.MTU link.mtu = RNS.Reticulum.MTU
link.mode = Link.mode_from_lr_packet(packet) link.mode = Link.mode_from_lr_packet(packet)
RNS.log(f"Incoming link request with mode {Link.MODE_DESCRIPTIONS[link.mode]}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
# TODO: Remove debug
RNS.log(f"Incoming link request with mode {Link.MODE_DESCRIPTIONS[link.mode]}", RNS.LOG_DEBUG)
link.update_mdu() link.update_mdu()
link.destination = packet.destination link.destination = packet.destination
link.establishment_timeout = Link.ESTABLISHMENT_TIMEOUT_PER_HOP * max(1, packet.hops) + Link.KEEPALIVE link.establishment_timeout = Link.ESTABLISHMENT_TIMEOUT_PER_HOP * max(1, packet.hops) + Link.KEEPALIVE
link.establishment_cost += len(packet.raw) link.establishment_cost += len(packet.raw)
RNS.log(f"Validating link request {RNS.prettyhexrep(link.link_id)} with value {value}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log(f"Validating link request {RNS.prettyhexrep(link.link_id)}", RNS.LOG_DEBUG)
RNS.log(f"Link MTU configured to {RNS.prettysize(link.mtu)}", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None RNS.log(f"Link MTU configured to {RNS.prettysize(link.mtu)}", RNS.LOG_EXTREME)
RNS.log(f"Establishment timeout is {RNS.prettytime(link.establishment_timeout)} for incoming link request {RNS.prettyhexrep(link.link_id)}", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None RNS.log(f"Establishment timeout is {RNS.prettytime(link.establishment_timeout)} for incoming link request "+RNS.prettyhexrep(link.link_id), RNS.LOG_EXTREME)
link.handshake() link.handshake()
link.attached_interface = packet.receiving_interface link.attached_interface = packet.receiving_interface
link.prove() link.prove()
@ -225,23 +218,21 @@ class Link:
link.__update_phy_stats(packet, force_update=True) link.__update_phy_stats(packet, force_update=True)
link.start_watchdog() link.start_watchdog()
RNS.log(f"Incoming link request {link} accepted on {link.attached_interface}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log("Incoming link request "+str(link)+" accepted on "+str(link.attached_interface), RNS.LOG_DEBUG)
return link return link
except Exception as e: except Exception as e:
RNS.log(f"Validating incoming link request failed: {e}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log(f"Validating link request failed: {e}", RNS.LOG_VERBOSE)
return None return None
else: else:
RNS.log(f"Invalid link request payload size of {len(data)} bytes, dropping request", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log(f"Invalid link request payload size of {len(data)} bytes, dropping request", RNS.LOG_DEBUG)
return None return None
def __init__(self, destination=None, established_callback=None, closed_callback=None, owner=None, peer_pub_bytes=None, peer_sig_pub_bytes=None, mode=MODE_DEFAULT): def __init__(self, destination=None, established_callback=None, closed_callback=None, owner=None, peer_pub_bytes=None, peer_sig_pub_bytes=None, mode=MODE_DEFAULT):
if destination != None and destination.type != RNS.Destination.SINGLE: raise TypeError("Links can only be established to the \"single\" destination type") if destination != None and destination.type != RNS.Destination.SINGLE: raise TypeError("Links can only be established to the \"single\" destination type")
if not mode in Link.ENABLED_MODES: raise TypeError(f"Requested link mode {Link.MODE_DESCRIPTIONS[mode]} not enabled")
self.mode = mode self.mode = mode
self.link_id = None
self.rtt = None self.rtt = None
self.mtu = RNS.Reticulum.MTU self.mtu = RNS.Reticulum.MTU
self.establishment_cost = 0 self.establishment_cost = 0
@ -272,28 +263,12 @@ class Link:
self.stale_time = Link.STALE_TIME self.stale_time = Link.STALE_TIME
self.watchdog_lock = False self.watchdog_lock = False
self.status = Link.PENDING self.status = Link.PENDING
self.stamp_status = None
self.stamp_job_id = None
self.activated_at = None self.activated_at = None
self.type = RNS.Destination.LINK self.type = RNS.Destination.LINK
self.owner = owner self.owner = owner
self.destination = destination self.destination = destination
self.expected_hops = None self.expected_hops = None
self.rebalanced = None
self.attached_interface = None self.attached_interface = None
self.stamp_cost = None
self.stamp_material = None
self.packet = None
self.token = None
self.hash = None
self.prv = None
self.prv_bytes = None
self.sig_prv = None
self.pub = None
self.pub_bytes = None
self.sig_pub = None
self.sig_pub_bytes = None
self.__defer_keys = False
self.__remote_identity = None self.__remote_identity = None
self.__track_phy_stats = False self.__track_phy_stats = False
self._channel = None self._channel = None
@ -302,102 +277,44 @@ class Link:
self.initiator = False self.initiator = False
self.prv = X25519PrivateKey.generate() self.prv = X25519PrivateKey.generate()
self.sig_prv = self.owner.identity.sig_prv self.sig_prv = self.owner.identity.sig_prv
else: else:
self.initiator = True self.initiator = True
self.expected_hops = RNS.Transport.hops_to(self.destination.hash) self.expected_hops = RNS.Transport.hops_to(self.destination.hash)
self.establishment_timeout = RNS.Reticulum.get_instance().get_first_hop_timeout(self.destination.hash) self.establishment_timeout = RNS.Reticulum.get_instance().get_first_hop_timeout(destination.hash)
self.establishment_timeout += Link.ESTABLISHMENT_TIMEOUT_PER_HOP * max(1, RNS.Transport.hops_to(self.destination.hash)) self.establishment_timeout += Link.ESTABLISHMENT_TIMEOUT_PER_HOP * max(1, RNS.Transport.hops_to(destination.hash))
self.stamp_cost = RNS.Identity.recall_link_cost(self.destination.hash)
if self.stamp_cost == None and RNS.Transport.ALWAYS_STAMP_LRS: self.stamp_cost = RNS.Transport.LR_COST
if self.stamp_cost and RNS.Transport.STAMP_LRS: self.__defer_keys = True
else:
self.prv = X25519PrivateKey.generate() self.prv = X25519PrivateKey.generate()
self.sig_prv = Ed25519PrivateKey.generate() self.sig_prv = Ed25519PrivateKey.generate()
if not self.__defer_keys: self.token = None
self.pub = self.prv.public_key() self.pub = self.prv.public_key()
self.pub_bytes = self.pub.public_bytes() self.pub_bytes = self.pub.public_bytes()
self.sig_pub = self.sig_prv.public_key() self.sig_pub = self.sig_prv.public_key()
self.sig_pub_bytes = self.sig_pub.public_bytes() self.sig_pub_bytes = self.sig_pub.public_bytes()
if peer_pub_bytes == None: if peer_pub_bytes == None:
self.peer_pub = None self.peer_pub = None
self.peer_pub_bytes = None self.peer_pub_bytes = None
else:
self.load_peer(peer_pub_bytes, peer_sig_pub_bytes)
else: self.load_peer(peer_pub_bytes, peer_sig_pub_bytes) if established_callback != None:
self.set_link_established_callback(established_callback)
if established_callback != None: self.set_link_established_callback(established_callback) if closed_callback != None:
if closed_callback != None: self.set_link_closed_callback(closed_callback) self.set_link_closed_callback(closed_callback)
if self.initiator: if self.initiator:
if not self.__defer_keys: self.__initiate()
else: threading.Thread(target=self.__initiate, daemon=True).start()
def __stamp_failed(self):
if self.status != Link.CLOSED:
self.teardown_reason = Link.STAMP_FAILURE
self.stamp_status = False
self.status = Link.CLOSED
self.link_closed()
def __initiate(self):
if self.status != Link.PENDING: return
if self.__defer_keys:
try:
def stamp_watchdog():
deadline = time.time() + RNS.Transport.LINK_STAMP_TIMEOUT
while self.stamp_status == None and time.time() <= deadline: time.sleep(1)
if time.time() > deadline and self.stamp_job_id and not self.stamp_status:
cancel_work(self.stamp_job_id)
RNS.log(f"Stamp generation timed out for link request to {RNS.prettyhexrep(self.destination.hash)}", RNS.LOG_NOTICE)
self.sig_prv = Ed25519PrivateKey.generate()
self.sig_pub = self.sig_prv.public_key()
self.sig_pub_bytes = self.sig_pub.public_bytes()
self.stamp_material = self.destination.hash + self.sig_pub_bytes
RNS.log(f"Starting stamp generation for link request to {self.destination}", RNS.LOG_DEBUG)
self.stamp_job_id = self.stamp_material
threading.Thread(target=stamp_watchdog, daemon=True).start()
stamped_key, v = generate_stamped_key(material=self.stamp_material, stamp_cost=self.stamp_cost)
self.stamp_job_id = None
if stamped_key:
stamped_prv = stamped_key[0]
self.prv = X25519PrivateKey.from_private_bytes(stamped_prv)
self.prv_bytes = self.prv.private_bytes()
self.pub = self.prv.public_key()
self.pub_bytes = self.pub.public_bytes()
if self.prv_bytes != stamped_prv or self.prv.public_key().public_bytes() != stamped_key[1]:
RNS.log(f"Stamped link request key generation mismatch, aborting link establishment", RNS.LOG_WARNING)
self.__stamp_failed()
return
self.stamp_status = True
else:
self.__stamp_failed()
return
except Exception as e:
RNS.log(f"Error while generating link request stamp: {e}", RNS.LOG_ERROR)
RNS.trace_exception(e)
self.__stamp_failed()
return
if self.status != Link.PENDING: return
signalling_bytes = b"" signalling_bytes = b""
nh_hw_mtu = RNS.Transport.next_hop_interface_hw_mtu(self.destination.hash) nh_hw_mtu = RNS.Transport.next_hop_interface_hw_mtu(destination.hash)
if RNS.Reticulum.link_mtu_discovery() and nh_hw_mtu: if RNS.Reticulum.link_mtu_discovery() and nh_hw_mtu:
signalling_bytes = Link.signalling_bytes(nh_hw_mtu, self.mode) signalling_bytes = Link.signalling_bytes(nh_hw_mtu, self.mode)
RNS.log(f"Signalling link MTU of {RNS.prettysize(nh_hw_mtu)} for link", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None RNS.log(f"Signalling link MTU of {RNS.prettysize(nh_hw_mtu)} for link", RNS.LOG_DEBUG) # TODO: Remove debug
else: signalling_bytes = Link.signalling_bytes(RNS.Reticulum.MTU, self.mode) else: signalling_bytes = Link.signalling_bytes(RNS.Reticulum.MTU, self.mode)
request_data = self.pub_bytes+self.sig_pub_bytes+signalling_bytes RNS.log(f"Establishing link with mode {Link.MODE_DESCRIPTIONS[self.mode]}", RNS.LOG_DEBUG) # TODO: Remove debug
self.packet = RNS.Packet(self.destination, request_data, packet_type=RNS.Packet.LINKREQUEST) self.request_data = self.pub_bytes+self.sig_pub_bytes+signalling_bytes
self.packet = RNS.Packet(destination, self.request_data, packet_type=RNS.Packet.LINKREQUEST)
self.packet.pack() self.packet.pack()
self.establishment_cost += len(self.packet.raw) self.establishment_cost += len(self.packet.raw)
self.set_link_id(self.packet) self.set_link_id(self.packet)
@ -406,8 +323,9 @@ class Link:
self.start_watchdog() self.start_watchdog()
self.packet.send() self.packet.send()
self.had_outbound() self.had_outbound()
RNS.log(f"Link request {RNS.prettyhexrep(self.link_id)} sent to {self.destination}", RNS.LOG_DEBUG) RNS.log("Link request "+RNS.prettyhexrep(self.link_id)+" sent to "+str(self.destination), RNS.LOG_DEBUG)
RNS.log(f"Establishment timeout is {RNS.prettytime(self.establishment_timeout)} for link request {RNS.prettyhexrep(self.link_id)}", RNS.LOG_EXTREME) RNS.log(f"Establishment timeout is {RNS.prettytime(self.establishment_timeout)} for link request "+RNS.prettyhexrep(self.link_id), RNS.LOG_EXTREME)
def load_peer(self, peer_pub_bytes, peer_sig_pub_bytes): def load_peer(self, peer_pub_bytes, peer_sig_pub_bytes):
self.peer_pub_bytes = peer_pub_bytes self.peer_pub_bytes = peer_pub_bytes
@ -416,7 +334,8 @@ class Link:
self.peer_sig_pub_bytes = peer_sig_pub_bytes self.peer_sig_pub_bytes = peer_sig_pub_bytes
self.peer_sig_pub = Ed25519PublicKey.from_public_bytes(self.peer_sig_pub_bytes) self.peer_sig_pub = Ed25519PublicKey.from_public_bytes(self.peer_sig_pub_bytes)
if not hasattr(self.peer_pub, "curve"): self.peer_pub.curve = Link.CURVE if not hasattr(self.peer_pub, "curve"):
self.peer_pub.curve = Link.CURVE
@staticmethod @staticmethod
def link_id_from_lr_packet(packet): def link_id_from_lr_packet(packet):
@ -480,13 +399,13 @@ class Link:
signalling_bytes = b"" signalling_bytes = b""
confirmed_mtu = None confirmed_mtu = None
mode = Link.mode_from_lp_packet(packet) mode = Link.mode_from_lp_packet(packet)
RNS.log(f"Validating link request proof with mode {Link.MODE_DESCRIPTIONS[mode]}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log(f"Validating link request proof with mode {Link.MODE_DESCRIPTIONS[mode]}", RNS.LOG_DEBUG) # TODO: Remove debug
if mode != self.mode: raise TypeError(f"Invalid link mode {mode} in link request proof") if mode != self.mode: raise TypeError(f"Invalid link mode {mode} in link request proof")
if len(packet.data) == RNS.Identity.SIGLENGTH//8+Link.ECPUBSIZE//2+Link.LINK_MTU_SIZE: if len(packet.data) == RNS.Identity.SIGLENGTH//8+Link.ECPUBSIZE//2+Link.LINK_MTU_SIZE:
confirmed_mtu = Link.mtu_from_lp_packet(packet) confirmed_mtu = Link.mtu_from_lp_packet(packet)
signalling_bytes = Link.signalling_bytes(confirmed_mtu, mode) signalling_bytes = Link.signalling_bytes(confirmed_mtu, mode)
packet.data = packet.data[:RNS.Identity.SIGLENGTH//8+Link.ECPUBSIZE//2] packet.data = packet.data[:RNS.Identity.SIGLENGTH//8+Link.ECPUBSIZE//2]
RNS.log(f"Destination confirmed link MTU of {RNS.prettysize(confirmed_mtu)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log(f"Destination confirmed link MTU of {RNS.prettysize(confirmed_mtu)}", RNS.LOG_DEBUG) # TODO: Remove debug
if self.initiator and len(packet.data) == RNS.Identity.SIGLENGTH//8+Link.ECPUBSIZE//2: if self.initiator and len(packet.data) == RNS.Identity.SIGLENGTH//8+Link.ECPUBSIZE//2:
peer_pub_bytes = packet.data[RNS.Identity.SIGLENGTH//8:RNS.Identity.SIGLENGTH//8+Link.ECPUBSIZE//2] peer_pub_bytes = packet.data[RNS.Identity.SIGLENGTH//8:RNS.Identity.SIGLENGTH//8+Link.ECPUBSIZE//2]
@ -511,8 +430,7 @@ class Link:
self.activated_at = time.time() self.activated_at = time.time()
self.last_proof = self.activated_at self.last_proof = self.activated_at
RNS.Transport.activate_link(self) RNS.Transport.activate_link(self)
RNS.Transport.mark_path_responsive(self.destination.hash) RNS.log("Link "+str(self)+" established with "+str(self.destination)+", RTT is "+RNS.prettyshorttime(self.rtt), RNS.LOG_DEBUG)
RNS.log(f"Link {self} established with {self.destination}, RTT is {RNS.prettyshorttime(self.rtt)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
if self.rtt != None and self.establishment_cost != None and self.rtt > 0 and self.establishment_cost > 0: if self.rtt != None and self.establishment_cost != None and self.rtt > 0 and self.establishment_cost > 0:
self.establishment_rate = self.establishment_cost/self.rtt self.establishment_rate = self.establishment_cost/self.rtt
@ -529,8 +447,8 @@ class Link:
thread = threading.Thread(target=self.callbacks.link_established, args=(self,)) thread = threading.Thread(target=self.callbacks.link_established, args=(self,))
thread.daemon = True thread.daemon = True
thread.start() thread.start()
else:
else: RNS.log(f"Invalid link proof signature received by {self}. Ignoring.", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log("Invalid link proof signature received by "+str(self)+". Ignoring.", RNS.LOG_DEBUG)
except Exception as e: except Exception as e:
self.status = Link.CLOSED self.status = Link.CLOSED
@ -557,8 +475,7 @@ class Link:
self.had_outbound() self.had_outbound()
def request(self, path, data=None, response_callback=None, failed_callback=None, progress_callback=None, def request(self, path, data = None, response_callback = None, failed_callback = None, progress_callback = None, timeout = None):
timeout=None, max_response_size=None):
""" """
Sends a request to the remote peer. Sends a request to the remote peer.
@ -566,7 +483,6 @@ class Link:
:param response_callback: An optional function or method with the signature *response_callback(request_receipt)* to be called when a response is received. See the :ref:`Request Example<example-request>` for more info. :param response_callback: An optional function or method with the signature *response_callback(request_receipt)* to be called when a response is received. See the :ref:`Request Example<example-request>` for more info.
:param failed_callback: An optional function or method with the signature *failed_callback(request_receipt)* to be called when a request fails. See the :ref:`Request Example<example-request>` for more info. :param failed_callback: An optional function or method with the signature *failed_callback(request_receipt)* to be called when a request fails. See the :ref:`Request Example<example-request>` for more info.
:param progress_callback: An optional function or method with the signature *progress_callback(request_receipt)* to be called when progress is made receiving the response. Progress can be accessed as a float between 0.0 and 1.0 by the *request_receipt.progress* property. :param progress_callback: An optional function or method with the signature *progress_callback(request_receipt)* to be called when progress is made receiving the response. Progress can be accessed as a float between 0.0 and 1.0 by the *request_receipt.progress* property.
:param max_response_size: An optional maximum accepted response size, in bytes as an integer.
:param timeout: An optional timeout in seconds for the request. If *None* is supplied it will be calculated based on link RTT. :param timeout: An optional timeout in seconds for the request. If *None* is supplied it will be calculated based on link RTT.
:returns: A :ref:`RNS.RequestReceipt<api-requestreceipt>` instance if the request was sent, or *False* if it was not. :returns: A :ref:`RNS.RequestReceipt<api-requestreceipt>` instance if the request was sent, or *False* if it was not.
""" """
@ -581,24 +497,39 @@ class Link:
request_packet = RNS.Packet(self, packed_request, RNS.Packet.DATA, context = RNS.Packet.REQUEST) request_packet = RNS.Packet(self, packed_request, RNS.Packet.DATA, context = RNS.Packet.REQUEST)
packet_receipt = request_packet.send() packet_receipt = request_packet.send()
if packet_receipt == False: return False if packet_receipt == False:
return False
else: else:
packet_receipt.set_timeout(timeout) packet_receipt.set_timeout(timeout)
return RequestReceipt(self, packet_receipt=packet_receipt, response_callback=response_callback, return RequestReceipt(
failed_callback=failed_callback, progress_callback=progress_callback, timeout=timeout, self,
request_size=len(packed_request), max_response_size=max_response_size) packet_receipt = packet_receipt,
response_callback = response_callback,
failed_callback = failed_callback,
progress_callback = progress_callback,
timeout = timeout,
request_size = len(packed_request),
)
else: else:
request_id = RNS.Identity.truncated_hash(packed_request) request_id = RNS.Identity.truncated_hash(packed_request)
RNS.log("Sending request "+RNS.prettyhexrep(request_id)+" as resource.", RNS.LOG_DEBUG) RNS.log("Sending request "+RNS.prettyhexrep(request_id)+" as resource.", RNS.LOG_DEBUG)
request_resource = RNS.Resource(packed_request, self, request_id = request_id, is_response = False, timeout = timeout) request_resource = RNS.Resource(packed_request, self, request_id = request_id, is_response = False, timeout = timeout)
return RequestReceipt(self, resource=request_resource, response_callback=response_callback, return RequestReceipt(
failed_callback=failed_callback, progress_callback=progress_callback, timeout=timeout, self,
request_size=len(packed_request), max_response_size=max_response_size) resource = request_resource,
response_callback = response_callback,
failed_callback = failed_callback,
progress_callback = progress_callback,
timeout = timeout,
request_size = len(packed_request),
)
def update_mdu(self): def update_mdu(self):
self.mdu = self.mtu - RNS.Reticulum.HEADER_MAXSIZE - RNS.Reticulum.IFAC_MIN_SIZE self.mdu = self.mtu - RNS.Reticulum.HEADER_MAXSIZE - RNS.Reticulum.IFAC_MIN_SIZE
self.mdu = math.floor((self.mtu-RNS.Reticulum.IFAC_MIN_SIZE-RNS.Reticulum.HEADER_MINSIZE-RNS.Identity.TOKEN_OVERHEAD)/RNS.Identity.AES256_BLOCKSIZE)*RNS.Identity.AES256_BLOCKSIZE - 1 self.mdu = math.floor((self.mtu-RNS.Reticulum.IFAC_MIN_SIZE-RNS.Reticulum.HEADER_MINSIZE-RNS.Identity.TOKEN_OVERHEAD)/RNS.Identity.AES128_BLOCKSIZE)*RNS.Identity.AES128_BLOCKSIZE - 1
def rtt_packet(self, packet): def rtt_packet(self, packet):
try: try:
@ -609,7 +540,6 @@ class Link:
self.rtt = max(measured_rtt, rtt) self.rtt = max(measured_rtt, rtt)
self.status = Link.ACTIVE self.status = Link.ACTIVE
self.activated_at = time.time() self.activated_at = time.time()
self.expected_hops = packet.hops
if self.rtt != None and self.establishment_cost != None and self.rtt > 0 and self.establishment_cost > 0: if self.rtt != None and self.establishment_cost != None and self.rtt > 0 and self.establishment_cost > 0:
self.establishment_rate = self.establishment_cost/self.rtt self.establishment_rate = self.establishment_cost/self.rtt
@ -617,8 +547,10 @@ class Link:
self.__update_keepalive() self.__update_keepalive()
try: try:
if self.owner.callbacks.link_established != None: self.owner.callbacks.link_established(self) if self.owner.callbacks.link_established != None:
except Exception as e: RNS.log("Error occurred in external link establishment callback. The contained exception was: "+str(e), RNS.LOG_ERROR) self.owner.callbacks.link_established(self)
except Exception as e:
RNS.log("Error occurred in external link establishment callback. The contained exception was: "+str(e), RNS.LOG_ERROR)
except Exception as e: except Exception as e:
RNS.log("Error occurred while processing RTT packet, tearing down link. The contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("Error occurred while processing RTT packet, tearing down link. The contained exception was: "+str(e), RNS.LOG_ERROR)
@ -633,57 +565,73 @@ class Link:
:param track: Whether or not to keep track of physical layer statistics. Value must be ``True`` or ``False``. :param track: Whether or not to keep track of physical layer statistics. Value must be ``True`` or ``False``.
""" """
if track: self.__track_phy_stats = True if track:
else: self.__track_phy_stats = False self.__track_phy_stats = True
else:
self.__track_phy_stats = False
def get_rssi(self): def get_rssi(self):
""" """
:returns: The physical layer *Received Signal Strength Indication* if available, otherwise ``None``. Physical layer statistics must be enabled on the link for this method to return a value. :returns: The physical layer *Received Signal Strength Indication* if available, otherwise ``None``. Physical layer statistics must be enabled on the link for this method to return a value.
""" """
if self.__track_phy_stats: return self.rssi if self.__track_phy_stats:
else: return None return self.rssi
else:
return None
def get_snr(self): def get_snr(self):
""" """
:returns: The physical layer *Signal-to-Noise Ratio* if available, otherwise ``None``. Physical layer statistics must be enabled on the link for this method to return a value. :returns: The physical layer *Signal-to-Noise Ratio* if available, otherwise ``None``. Physical layer statistics must be enabled on the link for this method to return a value.
""" """
if self.__track_phy_stats: return self.snr if self.__track_phy_stats:
else: return None return self.snr
else:
return None
def get_q(self): def get_q(self):
""" """
:returns: The physical layer *Link Quality* if available, otherwise ``None``. Physical layer statistics must be enabled on the link for this method to return a value. :returns: The physical layer *Link Quality* if available, otherwise ``None``. Physical layer statistics must be enabled on the link for this method to return a value.
""" """
if self.__track_phy_stats: return self.q if self.__track_phy_stats:
else: return None return self.q
else:
return None
def get_establishment_rate(self): def get_establishment_rate(self):
""" """
:returns: The data transfer rate at which the link establishment procedure ocurred, in bits per second. :returns: The data transfer rate at which the link establishment procedure ocurred, in bits per second.
""" """
if self.establishment_rate != None: return self.establishment_rate*8 if self.establishment_rate != None:
else: return None return self.establishment_rate*8
else:
return None
def get_mtu(self): def get_mtu(self):
""" """
:returns: The MTU of an established link. :returns: The MTU of an established link.
""" """
if self.status == Link.ACTIVE: return self.mtu if self.status == Link.ACTIVE:
else: return None return self.mtu
else:
return None
def get_mdu(self): def get_mdu(self):
""" """
:returns: The packet MDU of an established link. :returns: The packet MDU of an established link.
""" """
if self.status == Link.ACTIVE: return self.mdu if self.status == Link.ACTIVE:
else: return None return self.mdu
else:
return None
def get_expected_rate(self): def get_expected_rate(self):
""" """
:returns: The expected in-flight data rate of an established link. :returns: The packet expected in-flight data rate of an established link.
""" """
if self.status == Link.ACTIVE: return self.expected_rate if self.status == Link.ACTIVE:
else: return None return self.expected_rate
else:
return None
def get_mode(self): def get_mode(self):
""" """
@ -701,8 +649,10 @@ class Link:
""" """
:returns: The time in seconds since this link was established. :returns: The time in seconds since this link was established.
""" """
if self.activated_at: return time.time() - self.activated_at if self.activated_at:
else: return None return time.time() - self.activated_at
else:
return None
def no_inbound_for(self): def no_inbound_for(self):
""" """
@ -751,12 +701,10 @@ class Link:
Closes the link and purges encryption keys. New keys will Closes the link and purges encryption keys. New keys will
be used if a new link to the same destination is established. be used if a new link to the same destination is established.
""" """
if self.status == Link.CLOSED: return
if self.status != Link.PENDING and self.status != Link.CLOSED: self.__teardown_packet() if self.status != Link.PENDING and self.status != Link.CLOSED: self.__teardown_packet()
self.status = Link.CLOSED self.status = Link.CLOSED
if self.initiator: self.teardown_reason = Link.INITIATOR_CLOSED if self.initiator: self.teardown_reason = Link.INITIATOR_CLOSED
else: self.teardown_reason = Link.DESTINATION_CLOSED else: self.teardown_reason = Link.DESTINATION_CLOSED
if self.stamp_job_id: cancel_work(self.stamp_job_id); self.stamp_status = False
self.link_closed() self.link_closed()
def teardown_packet(self, packet): def teardown_packet(self, packet):
@ -764,15 +712,18 @@ class Link:
plaintext = self.decrypt(packet.data) plaintext = self.decrypt(packet.data)
if plaintext == self.link_id: if plaintext == self.link_id:
self.status = Link.CLOSED self.status = Link.CLOSED
if self.initiator: self.teardown_reason = Link.DESTINATION_CLOSED if self.initiator:
else: self.teardown_reason = Link.INITIATOR_CLOSED self.teardown_reason = Link.DESTINATION_CLOSED
else:
self.teardown_reason = Link.INITIATOR_CLOSED
self.__update_phy_stats(packet) self.__update_phy_stats(packet)
self.link_closed() self.link_closed()
except Exception as e: pass except Exception as e:
pass
def link_closed(self): def link_closed(self):
for resource in self.incoming_resources.copy(): resource.cancel() for resource in self.incoming_resources: resource.cancel()
for resource in self.outgoing_resources.copy(): resource.cancel() for resource in self.outgoing_resources: resource.cancel()
if self._channel: self._channel._shutdown() if self._channel: self._channel._shutdown()
self.prv = None self.prv = None
@ -801,7 +752,9 @@ class Link:
while not self.status == Link.CLOSED: while not self.status == Link.CLOSED:
while (self.watchdog_lock): while (self.watchdog_lock):
rtt_wait = 0.025 rtt_wait = 0.025
if hasattr(self, "rtt") and self.rtt: rtt_wait = self.rtt if hasattr(self, "rtt") and self.rtt:
rtt_wait = self.rtt
sleep(max(rtt_wait, 0.025)) sleep(max(rtt_wait, 0.025))
if not self.status == Link.CLOSED: if not self.status == Link.CLOSED:
@ -826,25 +779,28 @@ class Link:
self.link_closed() self.link_closed()
sleep_time = 0.001 sleep_time = 0.001
if self.initiator: RNS.log("Timeout waiting for link request proof", RNS.LOG_DEBUG) if self.initiator:
else: RNS.log("Timeout waiting for RTT packet from link initiator", RNS.LOG_DEBUG) RNS.log("Timeout waiting for link request proof", RNS.LOG_DEBUG)
else:
RNS.log("Timeout waiting for RTT packet from link initiator", RNS.LOG_DEBUG)
elif self.status == Link.ACTIVE: elif self.status == Link.ACTIVE:
activated_at = self.activated_at if self.activated_at != None else 0 activated_at = self.activated_at if self.activated_at != None else 0
last_inbound = max(max(self.last_inbound, self.last_proof), activated_at) last_inbound = max(max(self.last_inbound, self.last_proof), activated_at)
now = time.time() now = time.time()
if now >= last_inbound + self.keepalive or now >= self.last_outbound + self.keepalive: if now >= last_inbound + self.keepalive:
if self.initiator and now >= self.last_keepalive + self.keepalive: if self.initiator and now >= self.last_keepalive + self.keepalive:
self.send_keepalive() self.send_keepalive()
if now >= last_inbound + self.stale_time: if time.time() >= last_inbound + self.stale_time:
sleep_time = self.rtt * self.keepalive_timeout_factor + Link.STALE_GRACE sleep_time = self.rtt * self.keepalive_timeout_factor + Link.STALE_GRACE
self.status = Link.STALE self.status = Link.STALE
else:
sleep_time = self.keepalive
else: sleep_time = self.keepalive else:
sleep_time = (last_inbound + self.keepalive) - time.time()
else: sleep_time = (last_inbound + self.keepalive) - now
elif self.status == Link.STALE: elif self.status == Link.STALE:
sleep_time = 0.001 sleep_time = 0.001
@ -854,9 +810,10 @@ class Link:
self.link_closed() self.link_closed()
if sleep_time == 0: RNS.log(f"Link watchdog sleep time of 0 on {self}", RNS.LOG_ERROR) if sleep_time == 0:
RNS.log("Warning! Link watchdog sleep time of 0!", RNS.LOG_ERROR)
if sleep_time == None or sleep_time < 0: if sleep_time == None or sleep_time < 0:
RNS.log(f"Timing error, tearing down link {self} now", RNS.LOG_ERROR) RNS.log("Timing error! Tearing down link "+str(self)+" now.", RNS.LOG_ERROR)
self.teardown() self.teardown()
sleep_time = 0.1 sleep_time = 0.1
@ -868,6 +825,7 @@ class Link:
self.snr = None self.snr = None
self.q = None self.q = None
def __update_phy_stats(self, packet, query_shared = True, force_update = False): def __update_phy_stats(self, packet, query_shared = True, force_update = False):
if self.__track_phy_stats or force_update: if self.__track_phy_stats or force_update:
if query_shared: if query_shared:
@ -876,9 +834,12 @@ class Link:
if packet.snr == None: packet.snr = reticulum.get_packet_snr(packet.packet_hash) if packet.snr == None: packet.snr = reticulum.get_packet_snr(packet.packet_hash)
if packet.q == None: packet.q = reticulum.get_packet_q(packet.packet_hash) if packet.q == None: packet.q = reticulum.get_packet_q(packet.packet_hash)
if packet.rssi != None: self.rssi = packet.rssi if packet.rssi != None:
if packet.snr != None: self.snr = packet.snr self.rssi = packet.rssi
if packet.q != None: self.q = packet.q if packet.snr != None:
self.snr = packet.snr
if packet.q != None:
self.q = packet.q
def __update_keepalive(self): def __update_keepalive(self):
self.keepalive = max(min(self.rtt*(Link.KEEPALIVE_MAX/Link.KEEPALIVE_MAX_RTT), Link.KEEPALIVE_MAX), Link.KEEPALIVE_MIN) self.keepalive = max(min(self.rtt*(Link.KEEPALIVE_MAX/Link.KEEPALIVE_MAX_RTT), Link.KEEPALIVE_MAX), Link.KEEPALIVE_MIN)
@ -942,28 +903,21 @@ class Link:
identity_string = str(self.get_remote_identity()) if self.get_remote_identity() != None else "<Unknown>" identity_string = str(self.get_remote_identity()) if self.get_remote_identity() != None else "<Unknown>"
RNS.log("Request "+RNS.prettyhexrep(request_id)+" from "+identity_string+" not allowed for: "+str(path), RNS.LOG_DEBUG) RNS.log("Request "+RNS.prettyhexrep(request_id)+" from "+identity_string+" not allowed for: "+str(path), RNS.LOG_DEBUG)
def handle_response(self, request_id, response_data, response_size, response_transfer_size, metadata=None, update_sizes=False, check_size=False): def handle_response(self, request_id, response_data, response_size, response_transfer_size, metadata=None):
if self.status == Link.ACTIVE: if self.status == Link.ACTIVE:
remove = None remove = None
for pending_request in self.pending_requests: for pending_request in self.pending_requests:
if pending_request.request_id == request_id: if pending_request.request_id == request_id:
if not check_size or pending_request.max_response_size == None: size_ok = True
else: size_ok = response_size <= pending_request.max_response_size
remove = pending_request remove = pending_request
try: try:
if update_sizes:
pending_request.response_size = response_size pending_request.response_size = response_size
if pending_request.response_transfer_size == None: pending_request.response_transfer_size = 0 if pending_request.response_transfer_size == None:
pending_request.response_transfer_size = 0
pending_request.response_transfer_size += response_transfer_size pending_request.response_transfer_size += response_transfer_size
pending_request.response_received(response_data, metadata)
except Exception as e:
RNS.log("Error occurred while handling response. The contained exception was: "+str(e), RNS.LOG_ERROR)
if size_ok: pending_request.response_received(response_data, metadata)
else:
RNS.log(f"Rejected response with excessive size {RNS.prettysize(response_size)} on {self}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
pending_request.response_rejected()
except Exception as e: RNS.log("Error occurred while handling response. The contained exception was: "+str(e), RNS.LOG_ERROR)
break break
if remove != None: if remove != None:
@ -979,8 +933,8 @@ class Link:
def job(): self.handle_request(request_id, request_data) def job(): self.handle_request(request_id, request_data)
threading.Thread(target=job, daemon=True).start() threading.Thread(target=job, daemon=True).start()
else:
else: RNS.log("Incoming request resource failed with status: "+RNS.hexrep([resource.status]), RNS.LOG_DEBUG) RNS.log("Incoming request resource failed with status: "+RNS.hexrep([resource.status]), RNS.LOG_DEBUG)
def response_resource_concluded(self, resource): def response_resource_concluded(self, resource):
if resource.status == RNS.Resource.COMPLETE: if resource.status == RNS.Resource.COMPLETE:
@ -1011,28 +965,23 @@ class Link:
:return: ``Channel`` object :return: ``Channel`` object
""" """
if self._channel is None: self._channel = Channel(LinkChannelOutlet(self)) if self._channel is None:
self._channel = Channel(LinkChannelOutlet(self))
return self._channel return self._channel
def receive(self, packet): def receive(self, packet):
self.watchdog_lock = True self.watchdog_lock = True
try: self.__receive(packet)
except Exception as e:
RNS.log(f"Error on {self} while receiving packet: {e}", RNS.LOG_ERROR)
RNS.trace_exception(e)
finally: self.watchdog_lock = False
def __receive(self, packet):
if not self.status == Link.CLOSED and not (self.initiator and packet.context == RNS.Packet.KEEPALIVE and packet.data == bytes([0xFF])): if not self.status == Link.CLOSED and not (self.initiator and packet.context == RNS.Packet.KEEPALIVE and packet.data == bytes([0xFF])):
if packet.receiving_interface != self.attached_interface: if packet.receiving_interface != self.attached_interface:
RNS.log(f"Link-associated packet received on unexpected interface {packet.receiving_interface} instead of {self.attached_interface}! Someone might be trying to manipulate your communication!", RNS.LOG_ERROR) RNS.log(f"Link-associated packet received on unexpected interface {packet.receiving_interface} instead of {self.attached_interface}! Someone might be trying to manipulate your communication!", RNS.LOG_ERROR)
else: else:
self.last_inbound = time.time() self.last_inbound = time.time()
if packet.context != RNS.Packet.KEEPALIVE: self.last_data = self.last_inbound if packet.context != RNS.Packet.KEEPALIVE:
self.last_data = self.last_inbound
self.rx += 1 self.rx += 1
self.rxbytes += len(packet.data) self.rxbytes += len(packet.data)
RNS.Transport.link_rxb += len(packet.raw) if self.status == Link.STALE:
if self.status == Link.STALE: self.status = Link.ACTIVE self.status = Link.ACTIVE
if packet.packet_type == RNS.Packet.DATA: if packet.packet_type == RNS.Packet.DATA:
should_query = False should_query = False
@ -1053,8 +1002,10 @@ class Link:
elif self.destination.proof_strategy == RNS.Destination.PROVE_APP: elif self.destination.proof_strategy == RNS.Destination.PROVE_APP:
if self.destination.callbacks.proof_requested: if self.destination.callbacks.proof_requested:
try: try:
if self.destination.callbacks.proof_requested(packet): packet.prove() if self.destination.callbacks.proof_requested(packet):
except Exception as e: RNS.log("Error while executing proof request callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR) packet.prove()
except Exception as e:
RNS.log("Error while executing proof request callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
elif packet.context == RNS.Packet.LINKIDENTIFY: elif packet.context == RNS.Packet.LINKIDENTIFY:
plaintext = self.decrypt(packet.data) plaintext = self.decrypt(packet.data)
@ -1067,16 +1018,12 @@ class Link:
identity.load_public_key(public_key) identity.load_public_key(public_key)
if identity.validate(signature, signed_data): if identity.validate(signature, signed_data):
if RNS.Reticulum.get_instance().is_blackholed(identity.hash):
RNS.log(f"Terminating incoming link from blackholed identity {RNS.prettyhexrep(identity.hash)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
self.teardown()
else:
if self.__remote_identity == None:
self.__remote_identity = identity self.__remote_identity = identity
if self.callbacks.remote_identified != None: if self.callbacks.remote_identified != None:
try: self.callbacks.remote_identified(self, self.__remote_identity) try:
except Exception as e: RNS.log(f"Error while executing remote identified callback from {self}. The contained exception was: "+str(e), RNS.LOG_ERROR) self.callbacks.remote_identified(self, self.__remote_identity)
except Exception as e:
RNS.log("Error while executing remote identified callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
self.__update_phy_stats(packet, query_shared=True) self.__update_phy_stats(packet, query_shared=True)
@ -1085,15 +1032,12 @@ class Link:
request_id = packet.getTruncatedHash() request_id = packet.getTruncatedHash()
packed_request = self.decrypt(packet.data) packed_request = self.decrypt(packet.data)
if packed_request != None: if packed_request != None:
if self.destination.max_request_size == None: size_ok = True
else: size_ok = len(packed_request) <= self.destination.max_request_size
if not size_ok: RNS.log(f"Ignored request with excessive size {RNS.prettysize(len(packed_request))} on {self.destination}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
else:
unpacked_request = umsgpack.unpackb(packed_request) unpacked_request = umsgpack.unpackb(packed_request)
def job(): self.handle_request(request_id, unpacked_request) def job(): self.handle_request(request_id, unpacked_request)
threading.Thread(target=job, daemon=True).start() threading.Thread(target=job, daemon=True).start()
self.__update_phy_stats(packet, query_shared=True) self.__update_phy_stats(packet, query_shared=True)
except Exception as e: RNS.log("Error occurred while handling request. The contained exception was: "+str(e), RNS.LOG_ERROR) except Exception as e:
RNS.log("Error occurred while handling request. The contained exception was: "+str(e), RNS.LOG_ERROR)
elif packet.context == RNS.Packet.RESPONSE: elif packet.context == RNS.Packet.RESPONSE:
try: try:
@ -1103,10 +1047,11 @@ class Link:
request_id = unpacked_response[0] request_id = unpacked_response[0]
response_data = unpacked_response[1] response_data = unpacked_response[1]
transfer_size = len(umsgpack.packb(response_data))-2 transfer_size = len(umsgpack.packb(response_data))-2
def job(): self.handle_response(request_id, response_data, transfer_size, transfer_size, update_sizes=True, check_size=True) def job(): self.handle_response(request_id, response_data, transfer_size, transfer_size)
threading.Thread(target=job, daemon=True).start() threading.Thread(target=job, daemon=True).start()
self.__update_phy_stats(packet, query_shared=True) self.__update_phy_stats(packet, query_shared=True)
except Exception as e: RNS.log("Error occurred while handling response. The contained exception was: "+str(e), RNS.LOG_ERROR) except Exception as e:
RNS.log("Error occurred while handling response. The contained exception was: "+str(e), RNS.LOG_ERROR)
elif packet.context == RNS.Packet.LRRTT: elif packet.context == RNS.Packet.LRRTT:
if not self.initiator: if not self.initiator:
@ -1121,26 +1066,13 @@ class Link:
packet.plaintext = self.decrypt(packet.data) packet.plaintext = self.decrypt(packet.data)
if packet.plaintext != None: if packet.plaintext != None:
self.__update_phy_stats(packet, query_shared=True) self.__update_phy_stats(packet, query_shared=True)
try:
if RNS.ResourceAdvertisement.is_request(packet): if RNS.ResourceAdvertisement.is_request(packet):
if self.destination.request_handlers: RNS.Resource.accept(packet, callback=self.request_resource_concluded)
if self.destination.max_request_size == None: size_ok = True
else: size_ok = RNS.ResourceAdvertisement.read_size(packet) <= self.destination.max_request_size
if size_ok: RNS.Resource.accept(packet, callback=self.request_resource_concluded)
else:
RNS.Resource.reject(packet)
RNS.log(f"Rejected request with excessive size {RNS.prettysize(RNS.ResourceAdvertisement.read_size(packet))} on {self}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
elif RNS.ResourceAdvertisement.is_response(packet): elif RNS.ResourceAdvertisement.is_response(packet):
request_id = RNS.ResourceAdvertisement.read_request_id(packet) request_id = RNS.ResourceAdvertisement.read_request_id(packet)
for pending_request in self.pending_requests: for pending_request in self.pending_requests:
if pending_request.request_id == request_id: if pending_request.request_id == request_id:
if pending_request.max_response_size == None: size_ok = True
else: size_ok = RNS.ResourceAdvertisement.read_size(packet) <= pending_request.max_response_size
if not size_ok:
RNS.Resource.reject(packet)
pending_request.response_rejected()
RNS.log(f"Rejected response with excessive size {RNS.prettysize(RNS.ResourceAdvertisement.read_size(packet))} on {self}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
else:
response_resource = RNS.Resource.accept(packet, callback=self.response_resource_concluded, progress_callback=pending_request.response_resource_progress, request_id = request_id) response_resource = RNS.Resource.accept(packet, callback=self.response_resource_concluded, progress_callback=pending_request.response_resource_progress, request_id = request_id)
if response_resource != None: if response_resource != None:
if pending_request.response_size == None: if pending_request.response_size == None:
@ -1151,6 +1083,7 @@ class Link:
if pending_request.started_at == None: if pending_request.started_at == None:
pending_request.started_at = time.time() pending_request.started_at = time.time()
pending_request.response_resource_progress(response_resource) pending_request.response_resource_progress(response_resource)
elif self.resource_strategy == Link.ACCEPT_NONE: pass elif self.resource_strategy == Link.ACCEPT_NONE: pass
elif self.resource_strategy == Link.ACCEPT_APP: elif self.resource_strategy == Link.ACCEPT_APP:
if self.callbacks.resource != None: if self.callbacks.resource != None:
@ -1163,9 +1096,6 @@ class Link:
RNS.log("Error while executing resource accept callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("Error while executing resource accept callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
elif self.resource_strategy == Link.ACCEPT_ALL: elif self.resource_strategy == Link.ACCEPT_ALL:
RNS.Resource.accept(packet, self.callbacks.resource_concluded) RNS.Resource.accept(packet, self.callbacks.resource_concluded)
except Exception as e:
RNS.log(f"Invalid resource advertisement on {self}: {e}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
self.teardown()
elif packet.context == RNS.Packet.RESOURCE_REQ: elif packet.context == RNS.Packet.RESOURCE_REQ:
plaintext = self.decrypt(packet.data) plaintext = self.decrypt(packet.data)
@ -1218,7 +1148,6 @@ class Link:
elif packet.context == RNS.Packet.KEEPALIVE: elif packet.context == RNS.Packet.KEEPALIVE:
if not self.initiator and packet.data == bytes([0xFF]): if not self.initiator and packet.data == bytes([0xFF]):
if time.time() >= self.last_outbound + self.keepalive:
keepalive_packet = RNS.Packet(self, bytes([0xFE]), context=RNS.Packet.KEEPALIVE) keepalive_packet = RNS.Packet(self, bytes([0xFE]), context=RNS.Packet.KEEPALIVE)
keepalive_packet.send() keepalive_packet.send()
self.had_outbound(is_keepalive = True) self.had_outbound(is_keepalive = True)
@ -1252,6 +1181,9 @@ class Link:
threading.Thread(target=job, daemon=True).start() threading.Thread(target=job, daemon=True).start()
self.__update_phy_stats(packet, query_shared=True) self.__update_phy_stats(packet, query_shared=True)
self.watchdog_lock = False
def encrypt(self, plaintext): def encrypt(self, plaintext):
try: try:
if not self.token: if not self.token:
@ -1266,6 +1198,7 @@ class Link:
RNS.log("Encryption on link "+str(self)+" failed. The contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("Encryption on link "+str(self)+" failed. The contained exception was: "+str(e), RNS.LOG_ERROR)
raise e raise e
def decrypt(self, ciphertext): def decrypt(self, ciphertext):
try: try:
if not self.token: self.token = Token(self.derived_key) if not self.token: self.token = Token(self.derived_key)
@ -1275,6 +1208,7 @@ class Link:
RNS.log("Decryption failed on link "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("Decryption failed on link "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
return None return None
def sign(self, message): def sign(self, message):
return self.sig_prv.sign(message) return self.sig_prv.sign(message)
@ -1282,7 +1216,8 @@ class Link:
try: try:
self.peer_sig_pub.verify(signature, message) self.peer_sig_pub.verify(signature, message)
return True return True
except Exception as e: return False except Exception as e:
return False
def set_link_established_callback(self, callback): def set_link_established_callback(self, callback):
self.callbacks.link_established = callback self.callbacks.link_established = callback
@ -1394,7 +1329,7 @@ class Link:
if len(self.outgoing_resources) > 0: return False if len(self.outgoing_resources) > 0: return False
else: return True else: return True
def __str__(self): return RNS.prettyhexrep(self.link_id) if self.link_id else "<pending link>" def __str__(self): return RNS.prettyhexrep(self.link_id)
class RequestReceipt(): class RequestReceipt():
@ -1410,9 +1345,7 @@ class RequestReceipt():
RECEIVING = 0x03 RECEIVING = 0x03
READY = 0x04 READY = 0x04
def __init__(self, link, packet_receipt=None, resource=None, response_callback=None, failed_callback=None, def __init__(self, link, packet_receipt = None, resource = None, response_callback = None, failed_callback = None, progress_callback = None, timeout = None, request_size = None):
progress_callback=None, timeout=None, request_size=None, max_response_size=None):
self.packet_receipt = packet_receipt self.packet_receipt = packet_receipt
self.resource = resource self.resource = resource
self.started_at = None self.started_at = None
@ -1439,7 +1372,6 @@ class RequestReceipt():
self.progress = 0 self.progress = 0
self.concluded_at = None self.concluded_at = None
self.response_concluded_at = None self.response_concluded_at = None
self.max_response_size = max_response_size
if timeout != None: if timeout != None:
self.timeout = timeout self.timeout = timeout
@ -1453,6 +1385,7 @@ class RequestReceipt():
self.link.pending_requests.append(self) self.link.pending_requests.append(self)
def request_resource_concluded(self, resource): def request_resource_concluded(self, resource):
if resource.status == RNS.Resource.COMPLETE: if resource.status == RNS.Resource.COMPLETE:
RNS.log("Request "+RNS.prettyhexrep(self.request_id)+" successfully sent as resource.", RNS.LOG_DEBUG) RNS.log("Request "+RNS.prettyhexrep(self.request_id)+" successfully sent as resource.", RNS.LOG_DEBUG)
@ -1470,10 +1403,12 @@ class RequestReceipt():
self.link.pending_requests.remove(self) self.link.pending_requests.remove(self)
if self.callbacks.failed != None: if self.callbacks.failed != None:
try: self.callbacks.failed(self) try:
self.callbacks.failed(self)
except Exception as e: except Exception as e:
RNS.log("Error while executing request failed callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("Error while executing request failed callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
def __response_timeout_job(self): def __response_timeout_job(self):
while self.status == RequestReceipt.DELIVERED: while self.status == RequestReceipt.DELIVERED:
now = time.time() now = time.time()
@ -1483,25 +1418,18 @@ class RequestReceipt():
time.sleep(0.1) time.sleep(0.1)
def request_timed_out(self, packet_receipt): def request_timed_out(self, packet_receipt):
if self in self.link.pending_requests and self.status in [RequestReceipt.DELIVERED, RequestReceipt.SENT]: if self in self.link.pending_requests and self.status == RequestReceipt.DELIVERED:
self.status = RequestReceipt.FAILED self.status = RequestReceipt.FAILED
self.concluded_at = time.time() self.concluded_at = time.time()
self.link.pending_requests.remove(self) self.link.pending_requests.remove(self)
if self.callbacks.failed != None: if self.callbacks.failed != None:
try: self.callbacks.failed(self) try: self.callbacks.failed(self)
except Exception as e: RNS.log("Error while executing request timed out callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR) except Exception as e:
RNS.log("Error while executing request timed out callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
def response_rejected(self):
if self in self.link.pending_requests and self.status in [RequestReceipt.DELIVERED, RequestReceipt.SENT]:
self.status = RequestReceipt.FAILED
self.concluded_at = time.time()
self.link.pending_requests.remove(self)
if self.callbacks.failed != None:
try: self.callbacks.failed(self)
except Exception as e: RNS.log("Error while executing request timed out callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
def response_resource_progress(self, resource): def response_resource_progress(self, resource):
if resource != None: if resource != None:
@ -1518,10 +1446,13 @@ class RequestReceipt():
self.progress = resource.get_progress() self.progress = resource.get_progress()
if self.callbacks.progress != None: if self.callbacks.progress != None:
try: self.callbacks.progress(self) try:
except Exception as e: RNS.log("Error while executing response progress callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR) self.callbacks.progress(self)
except Exception as e:
RNS.log("Error while executing response progress callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
else:
resource.cancel()
else: resource.cancel()
def response_received(self, response, metadata=None): def response_received(self, response, metadata=None):
if not self.status == RequestReceipt.FAILED: if not self.status == RequestReceipt.FAILED:
@ -1540,11 +1471,13 @@ class RequestReceipt():
if self.callbacks.progress != None: if self.callbacks.progress != None:
try: self.callbacks.progress(self) try: self.callbacks.progress(self)
except Exception as e: RNS.log("Error while executing response progress callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR) except Exception as e:
RNS.log("Error while executing response progress callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
if self.callbacks.response != None: if self.callbacks.response != None:
try: self.callbacks.response(self) try: self.callbacks.response(self)
except Exception as e: RNS.log("Error while executing response received callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR) except Exception as e:
RNS.log("Error while executing response received callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
def get_request_id(self): def get_request_id(self):
""" """
@ -1568,23 +1501,31 @@ class RequestReceipt():
""" """
:returns: The response as *bytes* if it is ready, otherwise *None*. :returns: The response as *bytes* if it is ready, otherwise *None*.
""" """
if self.status == RequestReceipt.READY: return self.response if self.status == RequestReceipt.READY:
else: return None return self.response
else:
return None
def get_response_time(self): def get_response_time(self):
""" """
:returns: The response time of the request in seconds. :returns: The response time of the request in seconds.
""" """
if self.status == RequestReceipt.READY: return self.response_concluded_at - self.started_at if self.status == RequestReceipt.READY:
else: return None return self.response_concluded_at - self.started_at
else:
return None
def concluded(self): def concluded(self):
""" """
:returns: True if the associated request has concluded (successfully or with a failure), otherwise False. :returns: True if the associated request has concluded (successfully or with a failure), otherwise False.
""" """
if self.status == RequestReceipt.READY: return True if self.status == RequestReceipt.READY:
elif self.status == RequestReceipt.FAILED: return True return True
else: return False elif self.status == RequestReceipt.FAILED:
return True
else:
return False
class RequestReceiptCallbacks: class RequestReceiptCallbacks:

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -103,7 +103,7 @@ class Packet:
# With an MTU of 500, the maximum of data we can # With an MTU of 500, the maximum of data we can
# send in a single encrypted packet is given by # send in a single encrypted packet is given by
# the below calculation; 383 bytes. # the below calculation; 383 bytes.
ENCRYPTED_MDU = math.floor((RNS.Reticulum.MDU-RNS.Identity.TOKEN_OVERHEAD-RNS.Identity.KEYSIZE//16)/RNS.Identity.AES256_BLOCKSIZE)*RNS.Identity.AES256_BLOCKSIZE - 1 ENCRYPTED_MDU = math.floor((RNS.Reticulum.MDU-RNS.Identity.TOKEN_OVERHEAD-RNS.Identity.KEYSIZE//16)/RNS.Identity.AES128_BLOCKSIZE)*RNS.Identity.AES128_BLOCKSIZE - 1
""" """
The maximum size of the payload data in a single encrypted packet The maximum size of the payload data in a single encrypted packet
""" """
@ -116,13 +116,11 @@ class Packet:
__slots__ = "hops", "header", "header_type", "packet_type", "transport_type", "context", "context_flag", "destination" __slots__ = "hops", "header", "header_type", "packet_type", "transport_type", "context", "context_flag", "destination"
__slots__ += "transport_id", "data", "flags", "raw", "packed", "sent", "create_receipt", "receipt", "fromPacked", "MTU" __slots__ += "transport_id", "data", "flags", "raw", "packed", "sent", "create_receipt", "receipt", "fromPacked", "MTU"
__slots__ += "sent_at", "packet_hash", "ratchet_id", "receiving_interface", "attached_interface", "blocked_interfaces" __slots__ += "sent_at", "packet_hash", "ratchet_id", "attached_interface", "receiving_interface", "rssi", "snr", "q"
__slots__ += "rssi", "snr", "q", "ciphertext", "plaintext", "destination_hash", "destination_type", "link", "map_hash" __slots__ += "ciphertext", "plaintext", "destination_hash", "destination_type", "link", "map_hash", "is_outbound_pr"
__slots__ += "is_outbound_pr", "traffic_class", "announce_signature_validated"
def __init__(self, destination, data, packet_type = DATA, context = NONE, transport_type = RNS.Transport.BROADCAST, def __init__(self, destination, data, packet_type = DATA, context = NONE, transport_type = RNS.Transport.BROADCAST,
header_type=HEADER_1, transport_id=None, attached_interface=None, blocked_interfaces=None, header_type = HEADER_1, transport_id = None, attached_interface = None, create_receipt = True, context_flag=FLAG_UNSET):
create_receipt=True, context_flag=FLAG_UNSET):
if destination != None: if destination != None:
if transport_type == None: if transport_type == None:
@ -161,10 +159,7 @@ class Packet:
self.packet_hash = None self.packet_hash = None
self.ratchet_id = None self.ratchet_id = None
self.traffic_class = None
self.announce_signature_validated = None
self.attached_interface = attached_interface self.attached_interface = attached_interface
self.blocked_interfaces = blocked_interfaces
self.receiving_interface = None self.receiving_interface = None
self.is_outbound_pr = False self.is_outbound_pr = False
self.rssi = None self.rssi = None
@ -237,13 +232,8 @@ class Packet:
self.header += bytes([self.context]) self.header += bytes([self.context])
self.raw = self.header + self.ciphertext self.raw = self.header + self.ciphertext
if self.packet_type == Packet.ANNOUNCE and self.header_type == Packet.HEADER_1:
transport_id_length = RNS.Reticulum.TRUNCATED_HASHLENGTH//8
if len(self.raw)+transport_id_length > self.MTU:
raise IOError(f"Transport-inserted announce packet size of {len(self.raw)+transport_id_length} exceeds MTU of {self.MTU} bytes")
if len(self.raw) > self.MTU: if len(self.raw) > self.MTU:
raise IOError(f"Packet size of {len(self.raw)} exceeds MTU of {self.MTU} bytes") raise IOError("Packet size of "+str(len(self.raw))+" exceeds MTU of "+str(self.MTU)+" bytes")
self.packed = True self.packed = True
self.update_hash() self.update_hash()
@ -254,9 +244,6 @@ class Packet:
self.flags = self.raw[0] self.flags = self.raw[0]
self.hops = self.raw[1] self.hops = self.raw[1]
if self.hops >= RNS.Transport.PATHFINDER_M:
raise ValueError(f"Invalid hop count {self.hops}")
self.header_type = (self.flags & 0b01000000) >> 6 self.header_type = (self.flags & 0b01000000) >> 6
self.context_flag = (self.flags & 0b00100000) >> 5 self.context_flag = (self.flags & 0b00100000) >> 5
self.transport_type = (self.flags & 0b00010000) >> 4 self.transport_type = (self.flags & 0b00010000) >> 4
@ -270,23 +257,18 @@ class Packet:
self.destination_hash = self.raw[DST_LEN+2:2*DST_LEN+2] self.destination_hash = self.raw[DST_LEN+2:2*DST_LEN+2]
self.context = ord(self.raw[2*DST_LEN+2:2*DST_LEN+3]) self.context = ord(self.raw[2*DST_LEN+2:2*DST_LEN+3])
self.data = self.raw[2*DST_LEN+3:] self.data = self.raw[2*DST_LEN+3:]
if len(self.transport_id) != DST_LEN: raise ValueError("Malformed Transport ID field")
if len(self.destination_hash) != DST_LEN: raise ValueError("Malformed destination hash field")
else: else:
self.transport_id = None self.transport_id = None
self.destination_hash = self.raw[2:DST_LEN+2] self.destination_hash = self.raw[2:DST_LEN+2]
self.context = ord(self.raw[DST_LEN+2:DST_LEN+3]) self.context = ord(self.raw[DST_LEN+2:DST_LEN+3])
self.data = self.raw[DST_LEN+3:] self.data = self.raw[DST_LEN+3:]
if len(self.destination_hash) != DST_LEN: raise ValueError("Malformed destination hash field")
if len(self.data) == 0: raise ValueError("Zero-length data field")
self.packed = False self.packed = False
self.update_hash() self.update_hash()
return True return True
except Exception as e: except Exception as e:
RNS.log(f"Dropping malformed packet. The contained exception was: {e}", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None RNS.log("Received malformed packet, dropping it. The contained exception was: "+str(e), RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
return False return False
def send(self): def send(self):
@ -296,8 +278,6 @@ class Packet:
:returns: A :ref:`RNS.PacketReceipt<api-packetreceipt>` instance if *create_receipt* was set to *True* when the packet was instantiated, if not returns *None*. If the packet could not be sent *False* is returned. :returns: A :ref:`RNS.PacketReceipt<api-packetreceipt>` instance if *create_receipt* was set to *True* when the packet was instantiated, if not returns *None*. If the packet could not be sent *False* is returned.
""" """
if not self.sent: if not self.sent:
if self.hops >= RNS.Transport.PATHFINDER_M: return False
if not self.packed: self.pack()
if self.destination.type == RNS.Destination.LINK: if self.destination.type == RNS.Destination.LINK:
if self.destination.status == RNS.Link.CLOSED: if self.destination.status == RNS.Link.CLOSED:
RNS.log("Attempt to transmit over a closed link, dropping packet", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log("Attempt to transmit over a closed link, dropping packet", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
@ -308,14 +288,13 @@ class Packet:
else: else:
self.destination.last_outbound = time.time() self.destination.last_outbound = time.time()
self.destination.tx += 1 self.destination.tx += 1
self.destination.txbytes += len(self.ciphertext) self.destination.txbytes += len(self.data)
RNS.Transport.link_txb += len(self.raw)
if not self.packed: self.pack()
if RNS.Transport.outbound(self): return self.receipt if RNS.Transport.outbound(self): return self.receipt
else: else:
if not self.attached_interface: RNS.log("No interfaces could process the outbound packet", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
RNS.log(f"No interfaces could process the outbound {self.hops} hop, type {self.packet_type} packet for {self.destination}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
self.sent = False self.sent = False
self.receipt = None self.receipt = None
return False return False
@ -334,44 +313,51 @@ class Packet:
# encrypted destinations # encrypted destinations
self.pack() self.pack()
if RNS.Transport.outbound(self): return self.receipt if RNS.Transport.outbound(self):
return self.receipt
else: else:
RNS.log("Re-send failed. No interfaces could process the outbound packet", RNS.LOG_WARNING) RNS.log("Re-send failed. No interfaces could process the outbound packet", RNS.LOG_WARNING)
self.sent = False self.sent = False
self.receipt = None self.receipt = None
return False return False
else:
else: raise IOError("Packet was not sent yet") raise IOError("Packet was not sent yet")
def prove(self, destination=None): def prove(self, destination=None):
if self.fromPacked and hasattr(self, "destination") and self.destination: if self.fromPacked and hasattr(self, "destination") and self.destination:
if self.destination.identity and self.destination.identity.prv: if self.destination.identity and self.destination.identity.prv:
self.destination.identity.prove(self, destination) self.destination.identity.prove(self, destination)
elif self.fromPacked and hasattr(self, "link") and self.link: self.link.prove_packet(self) elif self.fromPacked and hasattr(self, "link") and self.link:
else: RNS.log("Could not prove packet associated with neither a destination nor a link", RNS.LOG_ERROR) self.link.prove_packet(self)
else:
RNS.log("Could not prove packet associated with neither a destination nor a link", RNS.LOG_ERROR)
# Generates a special destination that allows Reticulum # Generates a special destination that allows Reticulum
# to direct the proof back to the proved packet's sender # to direct the proof back to the proved packet's sender
def generate_proof_destination(self): return ProofDestination(self) def generate_proof_destination(self):
return ProofDestination(self)
def validate_proof_packet(self, proof_packet): return self.receipt.validate_proof_packet(proof_packet) def validate_proof_packet(self, proof_packet):
return self.receipt.validate_proof_packet(proof_packet)
def validate_proof(self, proof): return self.receipt.validate_proof(proof) def validate_proof(self, proof):
return self.receipt.validate_proof(proof)
def update_hash(self): self.packet_hash = self.get_hash() def update_hash(self):
self.packet_hash = self.get_hash()
def get_hash(self): return RNS.Identity.full_hash(self.get_hashable_part()) def get_hash(self):
return RNS.Identity.full_hash(self.get_hashable_part())
def getTruncatedHash(self): return RNS.Identity.truncated_hash(self.get_hashable_part()) def getTruncatedHash(self):
return RNS.Identity.truncated_hash(self.get_hashable_part())
@property
def truncated_packet_hash(self):
return self.packet_hash[:RNS.Reticulum.TRUNCATED_HASHLENGTH//8]
def get_hashable_part(self): def get_hashable_part(self):
hashable_part = bytes([self.raw[0] & 0b00001111]) hashable_part = bytes([self.raw[0] & 0b00001111])
if self.header_type == Packet.HEADER_2: hashable_part += self.raw[(RNS.Identity.TRUNCATED_HASHLENGTH//8)+2:] if self.header_type == Packet.HEADER_2:
else: hashable_part += self.raw[2:] hashable_part += self.raw[(RNS.Identity.TRUNCATED_HASHLENGTH//8)+2:]
else:
hashable_part += self.raw[2:]
return hashable_part return hashable_part
@ -379,29 +365,36 @@ class Packet:
""" """
:returns: The physical layer *Received Signal Strength Indication* if available, otherwise ``None``. :returns: The physical layer *Received Signal Strength Indication* if available, otherwise ``None``.
""" """
if self.rssi != None: return self.rssi if self.rssi != None:
else: return RNS.Reticulum.get_instance().get_packet_rssi(self.packet_hash) return self.rssi
else:
return reticulum.get_packet_rssi(self.packet_hash)
def get_snr(self): def get_snr(self):
""" """
:returns: The physical layer *Signal-to-Noise Ratio* if available, otherwise ``None``. :returns: The physical layer *Signal-to-Noise Ratio* if available, otherwise ``None``.
""" """
if self.snr != None: return self.snr if self.snr != None:
else: return RNS.Reticulum.get_instance().get_packet_snr(self.packet_hash) return self.snr
else:
return reticulum.get_packet_snr(self.packet_hash)
def get_q(self): def get_q(self):
""" """
:returns: The physical layer *Link Quality* if available, otherwise ``None``. :returns: The physical layer *Link Quality* if available, otherwise ``None``.
""" """
if self.q != None: return self.q if self.q != None:
else: return RNS.Reticulum.get_instance().get_packet_q(self.packet_hash) return self.q
else:
return reticulum.get_packet_q(self.packet_hash)
class ProofDestination: class ProofDestination:
def __init__(self, packet): def __init__(self, packet):
self.hash = packet.truncated_packet_hash; self.hash = packet.get_hash()[:RNS.Reticulum.TRUNCATED_HASHLENGTH//8];
self.type = RNS.Destination.SINGLE self.type = RNS.Destination.SINGLE
def encrypt(self, plaintext): return plaintext def encrypt(self, plaintext):
return plaintext
class PacketReceipt: class PacketReceipt:
@ -423,8 +416,8 @@ class PacketReceipt:
# Creates a new packet receipt from a sent packet # Creates a new packet receipt from a sent packet
def __init__(self, packet): def __init__(self, packet):
self.hash = packet.packet_hash self.hash = packet.get_hash()
self.truncated_hash = packet.truncated_packet_hash self.truncated_hash = packet.getTruncatedHash()
self.sent = True self.sent = True
self.sent_at = time.time() self.sent_at = time.time()
self.proved = False self.proved = False
@ -470,17 +463,18 @@ class PacketReceipt:
link.last_proof = self.concluded_at link.last_proof = self.concluded_at
if self.callbacks.delivery != None: if self.callbacks.delivery != None:
try: self.callbacks.delivery(self) try:
self.callbacks.delivery(self)
except Exception as e: except Exception as e:
RNS.log("An error occurred while evaluating external delivery callback for "+str(link), RNS.LOG_ERROR) RNS.log("An error occurred while evaluating external delivery callback for "+str(link), RNS.LOG_ERROR)
RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
RNS.trace_exception(e) RNS.trace_exception(e)
return True return True
else:
else: return False return False
else: return False else:
return False
elif len(proof) == PacketReceipt.IMPL_LENGTH: elif len(proof) == PacketReceipt.IMPL_LENGTH:
pass pass
# TODO: Why is this disabled? # TODO: Why is this disabled?
@ -497,7 +491,8 @@ class PacketReceipt:
# else: # else:
# RNS.log("invalid") # RNS.log("invalid")
# return False # return False
else: return False else:
return False
# Validate a raw proof # Validate a raw proof
def validate_proof(self, proof, proof_packet=None): def validate_proof(self, proof, proof_packet=None):
@ -514,20 +509,24 @@ class PacketReceipt:
self.proof_packet = proof_packet self.proof_packet = proof_packet
if self.callbacks.delivery != None: if self.callbacks.delivery != None:
try: self.callbacks.delivery(self) try:
self.callbacks.delivery(self)
except Exception as e: except Exception as e:
RNS.log("Error while executing proof validated callback. The contained exception was: "+str(e), RNS.LOG_ERROR) RNS.log("Error while executing proof validated callback. The contained exception was: "+str(e), RNS.LOG_ERROR)
return True return True
else:
else: return False return False
else: return False else:
return False
elif len(proof) == PacketReceipt.IMPL_LENGTH: elif len(proof) == PacketReceipt.IMPL_LENGTH:
# This is an implicit proof # This is an implicit proof
if not hasattr(self.destination, "identity"): return False if not hasattr(self.destination, "identity"):
if self.destination.identity == None: return False return False
if self.destination.identity == None:
return False
signature = proof[:RNS.Identity.SIGLENGTH//8] signature = proof[:RNS.Identity.SIGLENGTH//8]
proof_valid = self.destination.identity.validate(signature, self.hash) proof_valid = self.destination.identity.validate(signature, self.hash)
@ -538,13 +537,16 @@ class PacketReceipt:
self.proof_packet = proof_packet self.proof_packet = proof_packet
if self.callbacks.delivery != None: if self.callbacks.delivery != None:
try: self.callbacks.delivery(self) try:
except Exception as e: RNS.log("Error while executing proof validated callback. The contained exception was: "+str(e), RNS.LOG_ERROR) self.callbacks.delivery(self)
except Exception as e:
RNS.log("Error while executing proof validated callback. The contained exception was: "+str(e), RNS.LOG_ERROR)
return True return True
else:
else: return False return False
else: return False else:
return False
def get_rtt(self): def get_rtt(self):
""" """
@ -557,8 +559,10 @@ class PacketReceipt:
def check_timeout(self): def check_timeout(self):
if self.status == PacketReceipt.SENT and self.is_timed_out(): if self.status == PacketReceipt.SENT and self.is_timed_out():
if self.timeout == -1: self.status = PacketReceipt.CULLED if self.timeout == -1:
else: self.status = PacketReceipt.FAILED self.status = PacketReceipt.CULLED
else:
self.status = PacketReceipt.FAILED
self.concluded_at = time.time() self.concluded_at = time.time()

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -31,4 +31,5 @@
class Resolver: class Resolver:
@staticmethod @staticmethod
def resolve_identity(full_name): pass def resolve_identity(full_name):
pass

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -30,7 +30,6 @@
import RNS import RNS
import os import os
import io
import bz2 import bz2
import math import math
import time import time
@ -139,7 +138,6 @@ class Resource:
HASHMAP_IS_NOT_EXHAUSTED = 0x00 HASHMAP_IS_NOT_EXHAUSTED = 0x00
HASHMAP_IS_EXHAUSTED = 0xFF HASHMAP_IS_EXHAUSTED = 0xFF
HASHMAP_IS_EMPTY = 0xFE
# Status constants # Status constants
NONE = 0x00 NONE = 0x00
@ -151,7 +149,7 @@ class Resource:
COMPLETE = 0x06 COMPLETE = 0x06
FAILED = 0x07 FAILED = 0x07
CORRUPT = 0x08 CORRUPT = 0x08
REJECTED = 0x09 REJECTED = 0x00
@staticmethod @staticmethod
def reject(advertisement_packet): def reject(advertisement_packet):
@ -183,7 +181,6 @@ class Resource:
resource.hashmap_raw = adv.m resource.hashmap_raw = adv.m
resource.encrypted = True if resource.flags & 0x01 else False resource.encrypted = True if resource.flags & 0x01 else False
resource.compressed = True if resource.flags >> 1 & 0x01 else False resource.compressed = True if resource.flags >> 1 & 0x01 else False
resource.app_flag = True if resource.flags >> 6 & 0x01 else False
resource.initiator = False resource.initiator = False
resource.callback = callback resource.callback = callback
resource.__progress_callback = progress_callback resource.__progress_callback = progress_callback
@ -242,14 +239,14 @@ class Resource:
return None return None
except Exception as e: except Exception as e:
RNS.log(f"Could not decode resource advertisement, dropping resource: {e}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log("Could not decode resource advertisement, dropping resource", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return None return None
# Create a resource for transmission to a remote destination # Create a resource for transmission to a remote destination
# The data passed can be either a bytes-array or a file opened # The data passed can be either a bytes-array or a file opened
# in binary read mode. # in binary read mode.
def __init__(self, data, link, metadata=None, advertise=True, auto_compress=True, callback=None, progress_callback=None, def __init__(self, data, link, metadata=None, advertise=True, auto_compress=True, callback=None, progress_callback=None,
timeout=None, segment_index = 1, original_hash=None, request_id=None, is_response=False, app_flag=False, sent_metadata_size=0): timeout = None, segment_index = 1, original_hash = None, request_id = None, is_response = False, sent_metadata_size=0):
data_size = None data_size = None
resource_data = None resource_data = None
@ -259,7 +256,6 @@ class Resource:
self.metadata = None self.metadata = None
self.has_metadata = False self.has_metadata = False
self.metadata_size = sent_metadata_size self.metadata_size = sent_metadata_size
self.app_flag = app_flag
if metadata != None: if metadata != None:
packed_metadata = umsgpack.packb(metadata) packed_metadata = umsgpack.packb(metadata)
@ -284,19 +280,6 @@ class Resource:
if hasattr(data, "read"): if hasattr(data, "read"):
if data_size == None: data_size = os.stat(data.name).st_size if data_size == None: data_size = os.stat(data.name).st_size
if data_size == 0:
if isinstance(data, io.BufferedReader):
RNS.log(f"Attempting to proxy stream data...", RNS.LOG_DEBUG)
stream_proxy = tempfile.TemporaryFile()
stream_proxy.write(data.read())
stream_proxy.flush(); stream_proxy.seek(0)
data_size = os.stat(stream_proxy.name).st_size
if data_size: RNS.log(f"Stream proxy succeeded, final data size is {RNS.prettysize(data_size)}", RNS.LOG_DEBUG)
else: RNS.log(f"Resource initialisation received an invalid readable input, or stream data that could not be proxied, the transfer will likely fail.", RNS.LOG_WARNING)
data = stream_proxy
else: RNS.log(f"Got zero-sized readable at resource initialisation, the transfer will likely.", RNS.LOG_WARNING)
self.total_size = data_size + self.metadata_size self.total_size = data_size + self.metadata_size
if self.total_size <= Resource.MAX_EFFICIENT_SIZE: if self.total_size <= Resource.MAX_EFFICIENT_SIZE:
@ -307,6 +290,12 @@ class Resource:
data.close() data.close()
else: else:
# self.total_segments = ((data_size-1)//Resource.MAX_EFFICIENT_SIZE)+1
# self.segment_index = segment_index
# self.split = True
# seek_index = segment_index-1
# seek_position = seek_index*Resource.MAX_EFFICIENT_SIZE
self.total_segments = ((self.total_size-1)//Resource.MAX_EFFICIENT_SIZE)+1 self.total_segments = ((self.total_size-1)//Resource.MAX_EFFICIENT_SIZE)+1
self.segment_index = segment_index self.segment_index = segment_index
self.split = True self.split = True
@ -333,9 +322,11 @@ class Resource:
self.segment_index = 1 self.segment_index = 1
self.split = False self.split = False
elif data == None: pass elif data == None:
pass
else: raise TypeError("Invalid data instance type passed to resource initialisation") else:
raise TypeError("Invalid data instance type passed to resource initialisation")
if resource_data: if resource_data:
if self.has_metadata: data = self.metadata + resource_data if self.has_metadata: data = self.metadata + resource_data
@ -343,8 +334,10 @@ class Resource:
self.status = Resource.NONE self.status = Resource.NONE
self.link = link self.link = link
if self.link.mtu: self.sdu = self.link.mtu - RNS.Reticulum.HEADER_MAXSIZE - RNS.Reticulum.IFAC_MIN_SIZE if self.link.mtu:
else: self.sdu = link.mdu or Resource.SDU self.sdu = self.link.mtu - RNS.Reticulum.HEADER_MAXSIZE - RNS.Reticulum.IFAC_MIN_SIZE
else:
self.sdu = link.mdu or Resource.SDU
self.max_retries = Resource.MAX_RETRIES self.max_retries = Resource.MAX_RETRIES
self.max_adv_retries = Resource.MAX_ADV_RETRIES self.max_adv_retries = Resource.MAX_ADV_RETRIES
self.retries_left = self.max_retries self.retries_left = self.max_retries
@ -383,8 +376,10 @@ class Resource:
self.req_hashlist = [] self.req_hashlist = []
self.receiver_min_consecutive_height = 0 self.receiver_min_consecutive_height = 0
if timeout != None: self.timeout = timeout if timeout != None:
else: self.timeout = self.link.rtt * self.link.traffic_timeout_factor self.timeout = timeout
else:
self.timeout = self.link.rtt * self.link.traffic_timeout_factor
if data != None: if data != None:
self.initiator = True self.initiator = True
@ -393,21 +388,18 @@ class Resource:
compression_began = time.time() compression_began = time.time()
if self.auto_compress and data_size <= self.auto_compress_limit: if self.auto_compress and data_size <= self.auto_compress_limit:
RNS.log("Compressing resource data...", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log("Compressing resource data...", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
try:
self.compressed_data = bz2.compress(self.uncompressed_data) self.compressed_data = bz2.compress(self.uncompressed_data)
RNS.log("Compression completed in "+str(round(time.time()-compression_began, 3))+" seconds", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log("Compression completed in "+str(round(time.time()-compression_began, 3))+" seconds", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
except Exception as e: else:
RNS.log(f"Could not auto-compress resource data, falling back to uncompressed transfer: {e}", RNS.LOG_DEBUG)
self.compressed_data = self.uncompressed_data self.compressed_data = self.uncompressed_data
else: self.compressed_data = self.uncompressed_data
self.uncompressed_size = len(self.uncompressed_data) self.uncompressed_size = len(self.uncompressed_data)
self.compressed_size = len(self.compressed_data) self.compressed_size = len(self.compressed_data)
if (self.compressed_size < self.uncompressed_size and auto_compress): if (self.compressed_size < self.uncompressed_size and auto_compress):
saved_bytes = len(self.uncompressed_data) - len(self.compressed_data) saved_bytes = len(self.uncompressed_data) - len(self.compressed_data)
RNS.log("Compression saved "+str(saved_bytes)+" bytes, sending compressed", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log("Compression saved "+str(saved_bytes)+" bytes, sending compressed", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
self.data = b"" self.data = b""
self.data += RNS.Identity.get_random_hash()[:Resource.RANDOM_HASH_SIZE] self.data += RNS.Identity.get_random_hash()[:Resource.RANDOM_HASH_SIZE]
@ -459,8 +451,8 @@ class Resource:
self.hashmap = b"" self.hashmap = b""
collision_guard_list = [] collision_guard_list = []
for i in range(0,hashmap_entries): for i in range(0,hashmap_entries):
part_data = self.data[i*self.sdu:(i+1)*self.sdu] data = self.data[i*self.sdu:(i+1)*self.sdu]
map_hash = self.get_map_hash(part_data) map_hash = self.get_map_hash(data)
if map_hash in collision_guard_list: if map_hash in collision_guard_list:
RNS.log("Found hash collision in resource map, remapping...", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log("Found hash collision in resource map, remapping...", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
@ -472,7 +464,7 @@ class Resource:
if len(collision_guard_list) > ResourceAdvertisement.COLLISION_GUARD_SIZE: if len(collision_guard_list) > ResourceAdvertisement.COLLISION_GUARD_SIZE:
collision_guard_list.pop(0) collision_guard_list.pop(0)
part = RNS.Packet(link, part_data, context=RNS.Packet.RESOURCE) part = RNS.Packet(link, data, context=RNS.Packet.RESOURCE)
part.pack() part.pack()
part.map_hash = map_hash part.map_hash = map_hash
@ -490,28 +482,23 @@ class Resource:
def hashmap_update_packet(self, plaintext): def hashmap_update_packet(self, plaintext):
if not self.status == Resource.FAILED: if not self.status == Resource.FAILED:
if self.waiting_for_hmu:
self.last_activity = time.time() self.last_activity = time.time()
self.retries_left = self.max_retries self.retries_left = self.max_retries
update = umsgpack.unpackb(plaintext[RNS.Identity.HASHLENGTH//8:]) update = umsgpack.unpackb(plaintext[RNS.Identity.HASHLENGTH//8:])
self.hashmap_update(update[0], update[1]) self.hashmap_update(update[0], update[1])
def hashmap_update(self, segment, hashmap): def hashmap_update(self, segment, hashmap):
if not self.status == Resource.FAILED: if not self.status == Resource.FAILED:
self.status = Resource.TRANSFERRING self.status = Resource.TRANSFERRING
seg_len = ResourceAdvertisement.HASHMAP_MAX_LEN seg_len = ResourceAdvertisement.HASHMAP_MAX_LEN
hashes = len(hashmap)//Resource.MAPHASH_LEN hashes = len(hashmap)//Resource.MAPHASH_LEN
for i in range(0,hashes): for i in range(0,hashes):
if self.hashmap[i+segment*seg_len] == None: self.hashmap_height += 1 if self.hashmap[i+segment*seg_len] == None:
self.hashmap_height += 1
self.hashmap[i+segment*seg_len] = hashmap[i*Resource.MAPHASH_LEN:(i+1)*Resource.MAPHASH_LEN] self.hashmap[i+segment*seg_len] = hashmap[i*Resource.MAPHASH_LEN:(i+1)*Resource.MAPHASH_LEN]
if hashes < 1 and segment != 0:
RNS.log("Invalid HMU received, cancelling transfer", RNS.LOG_ERROR)
self.cancel()
else:
if hashes == 0 and segment == 0: RNS.log(f"Received HMU-less resource advertisement, requesting initial hashmap", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
self.waiting_for_hmu = False self.waiting_for_hmu = False
self.request_next() self.request_next()
@ -530,22 +517,13 @@ class Resource:
prepare_thread = threading.Thread(target=self.__prepare_next_segment, daemon=True) prepare_thread = threading.Thread(target=self.__prepare_next_segment, daemon=True)
prepare_thread.start() prepare_thread.start()
def ensure_link(self):
if not self.link or self.link.status != RNS.Link.ACTIVE:
RNS.log(f"Invalid link state for {self}, aborting transfer", RNS.LOG_VERBOSE)
try: self.cancel()
except Exception as e: RNS.log(f"Error while cancelling resource on link-state abort: {e}", RNS.LOG_ERROR)
return False
else: return True
def __advertise_job(self): def __advertise_job(self):
self.advertisement_packet = RNS.Packet(self.link, ResourceAdvertisement(self, mdu=self.link.mdu).pack(), context=RNS.Packet.RESOURCE_ADV) self.advertisement_packet = RNS.Packet(self.link, ResourceAdvertisement(self).pack(), context=RNS.Packet.RESOURCE_ADV)
while not self.link.ready_for_new_resource(): while not self.link.ready_for_new_resource():
self.status = Resource.QUEUED self.status = Resource.QUEUED
sleep(0.25) sleep(0.25)
try: try:
if not self.ensure_link(): return
self.advertisement_packet.send() self.advertisement_packet.send()
self.last_activity = time.time() self.last_activity = time.time()
self.started_transferring = self.last_activity self.started_transferring = self.last_activity
@ -563,13 +541,18 @@ class Resource:
self.watchdog_job() self.watchdog_job()
def update_eifr(self): def update_eifr(self):
if self.rtt == None: rtt = self.link.rtt if self.rtt == None:
else: rtt = self.rtt rtt = self.link.rtt
if self.req_data_rtt_rate != 0: expected_inflight_rate = self.req_data_rtt_rate*8
else: else:
if self.previous_eifr != None: expected_inflight_rate = self.previous_eifr rtt = self.rtt
else: expected_inflight_rate = self.link.establishment_cost*8 / rtt
if self.req_data_rtt_rate != 0:
expected_inflight_rate = self.req_data_rtt_rate*8
else:
if self.previous_eifr != None:
expected_inflight_rate = self.previous_eifr
else:
expected_inflight_rate = self.link.establishment_cost*8 / rtt
self.eifr = expected_inflight_rate self.eifr = expected_inflight_rate
if self.link: self.link.expected_rate = self.eifr if self.link: self.link.expected_rate = self.eifr
@ -583,7 +566,8 @@ class Resource:
this_job_id = self.__watchdog_job_id this_job_id = self.__watchdog_job_id
while self.status < Resource.ASSEMBLING and this_job_id == self.__watchdog_job_id: while self.status < Resource.ASSEMBLING and this_job_id == self.__watchdog_job_id:
while self.watchdog_lock: sleep(0.025) while self.watchdog_lock:
sleep(0.025)
sleep_time = None sleep_time = None
if self.status == Resource.ADVERTISED: if self.status == Resource.ADVERTISED:
@ -597,8 +581,7 @@ class Resource:
try: try:
RNS.log("No part requests received, retrying resource advertisement...", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log("No part requests received, retrying resource advertisement...", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
self.retries_left -= 1 self.retries_left -= 1
if not self.ensure_link(): return self.advertisement_packet = RNS.Packet(self.link, ResourceAdvertisement(self).pack(), context=RNS.Packet.RESOURCE_ADV)
self.advertisement_packet = RNS.Packet(self.link, ResourceAdvertisement(self, mdu=self.link.mdu).pack(), context=RNS.Packet.RESOURCE_ADV)
self.advertisement_packet.send() self.advertisement_packet.send()
self.last_activity = time.time() self.last_activity = time.time()
self.adv_sent = self.last_activity self.adv_sent = self.last_activity
@ -674,13 +657,13 @@ class Resource:
self.last_part_sent = time.time() self.last_part_sent = time.time()
sleep_time = 0.001 sleep_time = 0.001
elif self.status >= Resource.ASSEMBLING: elif self.status == Resource.REJECTED:
sleep_time = 0.001 sleep_time = 0.001
if sleep_time == 0: if sleep_time == 0:
RNS.log("Warning! Link watchdog sleep time of 0!", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log("Warning! Link watchdog sleep time of 0!", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
if sleep_time == None or sleep_time < 0: if sleep_time == None or sleep_time < 0:
RNS.log(f"Timing error ({sleep_time}/{self.status}), cancelling resource transfer.", RNS.LOG_ERROR) RNS.log("Timing error, cancelling resource transfer.", RNS.LOG_ERROR)
self.cancel() self.cancel()
if sleep_time != None: if sleep_time != None:
@ -771,7 +754,6 @@ class Resource:
try: try:
proof = RNS.Identity.full_hash(self.data+self.hash) proof = RNS.Identity.full_hash(self.data+self.hash)
proof_data = self.hash+proof proof_data = self.hash+proof
if not self.ensure_link(): return
proof_packet = RNS.Packet(self.link, proof_data, packet_type=RNS.Packet.PROOF, context=RNS.Packet.RESOURCE_PRF) proof_packet = RNS.Packet(self.link, proof_data, packet_type=RNS.Packet.PROOF, context=RNS.Packet.RESOURCE_PRF)
proof_packet.send() proof_packet.send()
RNS.Transport.cache(proof_packet, force_cache=True) RNS.Transport.cache(proof_packet, force_cache=True)
@ -793,10 +775,9 @@ class Resource:
is_response = self.is_response, is_response = self.is_response,
advertise = False, advertise = False,
auto_compress = self.auto_compress_option, auto_compress = self.auto_compress_option,
app_flag = self.app_flag,
sent_metadata_size = self.metadata_size) sent_metadata_size = self.metadata_size)
if self.__progress_callback:
if self.__progress_callback: self.next_segment.progress_callback(self.__progress_callback) self.next_segment.progress_callback(self.__progress_callback)
def validate_proof(self, proof_data): def validate_proof(self, proof_data):
if not self.status == Resource.FAILED: if not self.status == Resource.FAILED:
@ -879,9 +860,9 @@ class Resource:
part_data = packet.data part_data = packet.data
part_hash = self.get_map_hash(part_data) part_hash = self.get_map_hash(part_data)
search_start = self.consecutive_completed_height+1 consecutive_index = self.consecutive_completed_height if self.consecutive_completed_height >= 0 else 0
i = search_start i = consecutive_index
for map_hash in self.hashmap[search_start:search_start+self.window]: for map_hash in self.hashmap[consecutive_index:consecutive_index+self.window]:
if map_hash == part_hash: if map_hash == part_hash:
if self.parts[i] == None: if self.parts[i] == None:
@ -892,6 +873,9 @@ class Resource:
self.outstanding_parts -= 1 self.outstanding_parts -= 1
# Update consecutive completed pointer # Update consecutive completed pointer
if i == self.consecutive_completed_height + 1:
self.consecutive_completed_height = i
cp = self.consecutive_completed_height + 1 cp = self.consecutive_completed_height + 1
while cp < len(self.parts) and self.parts[cp] != None: while cp < len(self.parts) and self.parts[cp] != None:
self.consecutive_completed_height = cp self.consecutive_completed_height = cp
@ -954,13 +938,6 @@ class Resource:
hashmap_exhausted = Resource.HASHMAP_IS_NOT_EXHAUSTED hashmap_exhausted = Resource.HASHMAP_IS_NOT_EXHAUSTED
requested_hashes = b"" requested_hashes = b""
# Initial advertisement-carried hashmap was empty,
# we can't request any real parts, and need an
# actual hashmap update first
if self.hashmap[0] == None:
hashmap_exhausted = Resource.HASHMAP_IS_EMPTY
else:
i = 0; pn = self.consecutive_completed_height+1 i = 0; pn = self.consecutive_completed_height+1
search_start = pn search_start = pn
search_size = self.window search_size = self.window
@ -985,14 +962,10 @@ class Resource:
hmu_part += last_map_hash hmu_part += last_map_hash
self.waiting_for_hmu = True self.waiting_for_hmu = True
if hashmap_exhausted == Resource.HASHMAP_IS_EMPTY:
self.waiting_for_hmu = True
request_data = hmu_part + self.hash + requested_hashes request_data = hmu_part + self.hash + requested_hashes
request_packet = RNS.Packet(self.link, request_data, context = RNS.Packet.RESOURCE_REQ) request_packet = RNS.Packet(self.link, request_data, context = RNS.Packet.RESOURCE_REQ)
try: try:
if not self.ensure_link(): return
request_packet.send() request_packet.send()
self.last_activity = time.time() self.last_activity = time.time()
self.req_sent = self.last_activity self.req_sent = self.last_activity
@ -1018,16 +991,8 @@ class Resource:
self.retries_left = self.max_retries self.retries_left = self.max_retries
pad = 1 wants_more_hashmap = True if request_data[0] == Resource.HASHMAP_IS_EXHAUSTED else False
hashmap_is_empty = False pad = 1+Resource.MAPHASH_LEN if wants_more_hashmap else 1
wants_more_hashmap = False
if request_data[0] == Resource.HASHMAP_IS_EMPTY:
wants_more_hashmap = True
hashmap_is_empty = True
if request_data[0] == Resource.HASHMAP_IS_EXHAUSTED:
wants_more_hashmap = True
pad += Resource.MAPHASH_LEN
requested_hashes = request_data[pad+RNS.Identity.HASHLENGTH//8:] requested_hashes = request_data[pad+RNS.Identity.HASHLENGTH//8:]
@ -1045,11 +1010,11 @@ class Resource:
for part in requested_parts: for part in requested_parts:
try: try:
if not self.ensure_link(): return
if not part.sent: if not part.sent:
part.send() part.send()
self.sent_parts += 1 self.sent_parts += 1
else: part.resend() else:
part.resend()
self.last_activity = time.time() self.last_activity = time.time()
self.last_part_sent = self.last_activity self.last_part_sent = self.last_activity
@ -1063,19 +1028,22 @@ class Resource:
last_map_hash = request_data[1:Resource.MAPHASH_LEN+1] last_map_hash = request_data[1:Resource.MAPHASH_LEN+1]
part_index = self.receiver_min_consecutive_height part_index = self.receiver_min_consecutive_height
search_start = part_index if not hashmap_is_empty else 0 search_start = part_index
search_end = self.receiver_min_consecutive_height+ResourceAdvertisement.COLLISION_GUARD_SIZE if not hashmap_is_empty else 0 search_end = self.receiver_min_consecutive_height+ResourceAdvertisement.COLLISION_GUARD_SIZE
for part in self.parts[search_start:search_end]: for part in self.parts[search_start:search_end]:
part_index += 1 part_index += 1
if part.map_hash == last_map_hash: break if part.map_hash == last_map_hash:
break
self.receiver_min_consecutive_height = max(part_index-1-Resource.WINDOW_MAX, 0) self.receiver_min_consecutive_height = max(part_index-1-Resource.WINDOW_MAX, 0)
if not hashmap_is_empty and part_index % ResourceAdvertisement.HASHMAP_MAX_LEN != 0: if part_index % ResourceAdvertisement.HASHMAP_MAX_LEN != 0:
RNS.log("Resource sequencing error, cancelling transfer!", RNS.LOG_ERROR) RNS.log("Resource sequencing error, cancelling transfer!", RNS.LOG_ERROR)
self.cancel() self.cancel()
return return
else: segment = part_index // ResourceAdvertisement.HASHMAP_MAX_LEN else:
segment = part_index // ResourceAdvertisement.HASHMAP_MAX_LEN
hashmap_start = segment*ResourceAdvertisement.HASHMAP_MAX_LEN hashmap_start = segment*ResourceAdvertisement.HASHMAP_MAX_LEN
hashmap_end = min((segment+1)*ResourceAdvertisement.HASHMAP_MAX_LEN, len(self.parts)) hashmap_end = min((segment+1)*ResourceAdvertisement.HASHMAP_MAX_LEN, len(self.parts))
@ -1084,16 +1052,10 @@ class Resource:
for i in range(hashmap_start,hashmap_end): for i in range(hashmap_start,hashmap_end):
hashmap += self.hashmap[i*Resource.MAPHASH_LEN:(i+1)*Resource.MAPHASH_LEN] hashmap += self.hashmap[i*Resource.MAPHASH_LEN:(i+1)*Resource.MAPHASH_LEN]
if not hashmap:
RNS.log("Resource HMU error, cancelling transfer!", RNS.LOG_ERROR)
self.cancel()
return
hmu = self.hash+umsgpack.packb([segment, hashmap]) hmu = self.hash+umsgpack.packb([segment, hashmap])
hmu_packet = RNS.Packet(self.link, hmu, context = RNS.Packet.RESOURCE_HMU) hmu_packet = RNS.Packet(self.link, hmu, context = RNS.Packet.RESOURCE_HMU)
try: try:
if not self.ensure_link(): return
hmu_packet.send() hmu_packet.send()
self.last_activity = time.time() self.last_activity = time.time()
except Exception as e: except Exception as e:
@ -1117,7 +1079,7 @@ class Resource:
if self.next_segment: self.next_segment.cancel() if self.next_segment: self.next_segment.cancel()
if self.status == Resource.CORRUPT: if self.status == Resource.CORRUPT:
if self in self.link.incoming_resources: self.link.cancel_incoming_resource(self) self.link.cancel_incoming_resource(self)
self.reject(self.advertisement_packet) self.reject(self.advertisement_packet)
self.link.teardown() self.link.teardown()
@ -1128,15 +1090,11 @@ class Resource:
try: try:
cancel_packet = RNS.Packet(self.link, self.hash, context=RNS.Packet.RESOURCE_ICL) cancel_packet = RNS.Packet(self.link, self.hash, context=RNS.Packet.RESOURCE_ICL)
cancel_packet.send() cancel_packet.send()
except Exception as e: RNS.log("Could not send resource cancel packet, the contained exception was: "+str(e), RNS.LOG_ERROR) except Exception as e:
if self in self.link.outgoing_resources: self.link.cancel_outgoing_resource(self) RNS.log("Could not send resource cancel packet, the contained exception was: "+str(e), RNS.LOG_ERROR)
self.link.cancel_outgoing_resource(self)
else: else:
if self.link.status == RNS.Link.ACTIVE: self.link.cancel_incoming_resource(self)
try:
cancel_packet = RNS.Packet(self.link, self.hash, context=RNS.Packet.RESOURCE_RCL)
cancel_packet.send()
except Exception as e: RNS.log("Could not send resource cancel packet, the contained exception was: "+str(e), RNS.LOG_ERROR)
if self in self.link.incoming_resources: self.link.cancel_incoming_resource(self)
if self.callback != None: if self.callback != None:
try: try:
@ -1149,7 +1107,7 @@ class Resource:
if self.status < Resource.COMPLETE: if self.status < Resource.COMPLETE:
if self.initiator: if self.initiator:
self.status = Resource.REJECTED self.status = Resource.REJECTED
if self in self.link.outgoing_resources: self.link.cancel_outgoing_resource(self) self.link.cancel_outgoing_resource(self)
if self.callback != None: if self.callback != None:
try: try:
self.link.resource_concluded(self) self.link.resource_concluded(self)
@ -1270,9 +1228,7 @@ class Resource:
return self.compressed return self.compressed
def __str__(self): def __str__(self):
if hasattr(self, "hash") and self.hash and hasattr(self, "link") and self.link:
return "<"+RNS.hexrep(self.hash,delimit=False)+"/"+RNS.hexrep(self.link.link_id,delimit=False)+">" return "<"+RNS.hexrep(self.hash,delimit=False)+"/"+RNS.hexrep(self.link.link_id,delimit=False)+">"
else: return "<initializing_resource/unknown>"
class ResourceAdvertisement: class ResourceAdvertisement:
@ -1285,33 +1241,42 @@ class ResourceAdvertisement:
@staticmethod @staticmethod
def is_request(advertisement_packet): def is_request(advertisement_packet):
adv = ResourceAdvertisement.unpack(advertisement_packet.plaintext) adv = ResourceAdvertisement.unpack(advertisement_packet.plaintext)
if adv.q != None and adv.u: return True if adv.q != None and adv.u:
else: return False return True
else:
return False
@staticmethod @staticmethod
def is_response(advertisement_packet): def is_response(advertisement_packet):
adv = ResourceAdvertisement.unpack(advertisement_packet.plaintext) adv = ResourceAdvertisement.unpack(advertisement_packet.plaintext)
if adv.q != None and adv.p: return True
else: return False if adv.q != None and adv.p:
return True
else:
return False
@staticmethod @staticmethod
def read_request_id(advertisement_packet): def read_request_id(advertisement_packet):
adv = ResourceAdvertisement.unpack(advertisement_packet.plaintext) adv = ResourceAdvertisement.unpack(advertisement_packet.plaintext)
return adv.q return adv.q
@staticmethod @staticmethod
def read_transfer_size(advertisement_packet): def read_transfer_size(advertisement_packet):
adv = ResourceAdvertisement.unpack(advertisement_packet.plaintext) adv = ResourceAdvertisement.unpack(advertisement_packet.plaintext)
return adv.t return adv.t
@staticmethod @staticmethod
def read_size(advertisement_packet): def read_size(advertisement_packet):
adv = ResourceAdvertisement.unpack(advertisement_packet.plaintext) adv = ResourceAdvertisement.unpack(advertisement_packet.plaintext)
return adv.d return adv.d
def __init__(self, resource=None, request_id=None, is_response=False, mdu=None):
def __init__(self, resource=None, request_id=None, is_response=False):
self.link = None self.link = None
self.mdu = mdu or RNS.Link.MDU
if resource != None: if resource != None:
self.t = resource.size # Transfer size self.t = resource.size # Transfer size
self.d = resource.total_size # Total uncompressed data size self.d = resource.total_size # Total uncompressed data size
@ -1322,7 +1287,6 @@ class ResourceAdvertisement:
self.m = resource.hashmap # Resource hashmap self.m = resource.hashmap # Resource hashmap
self.c = resource.compressed # Compression flag self.c = resource.compressed # Compression flag
self.e = resource.encrypted # Encryption flag self.e = resource.encrypted # Encryption flag
self.a = resource.app_flag # Application flag
self.s = resource.split # Split flag self.s = resource.split # Split flag
self.x = resource.has_metadata # Metadata flag self.x = resource.has_metadata # Metadata flag
self.i = resource.segment_index # Segment index self.i = resource.segment_index # Segment index
@ -1340,17 +1304,31 @@ class ResourceAdvertisement:
self.p = True self.p = True
# Flags # Flags
self.f = 0x00 | self.a << 6 | self.x << 5 | self.p << 4 | self.u << 3 | self.s << 2 | self.c << 1 | self.e self.f = 0x00 | self.x << 5 | self.p << 4 | self.u << 3 | self.s << 2 | self.c << 1 | self.e
def get_transfer_size(self): return self.t def get_transfer_size(self):
def get_data_size(self): return self.d return self.t
def get_parts(self): return self.n
def get_segments(self): return self.l def get_data_size(self):
def get_hash(self): return self.h return self.d
def is_compressed(self): return self.c
def has_metadata(self): return self.x def get_parts(self):
def get_link(self): return self.link return self.n
def app_flag_set(self): return self.f >> 6 & 0x01
def get_segments(self):
return self.l
def get_hash(self):
return self.h
def is_compressed(self):
return self.c
def has_metadata(self):
return self.x
def get_link(self):
return self.link
def pack(self, segment=0): def pack(self, segment=0):
hashmap_start = segment*ResourceAdvertisement.HASHMAP_MAX_LEN hashmap_start = segment*ResourceAdvertisement.HASHMAP_MAX_LEN
@ -1360,7 +1338,8 @@ class ResourceAdvertisement:
for i in range(hashmap_start,hashmap_end): for i in range(hashmap_start,hashmap_end):
hashmap += self.m[i*Resource.MAPHASH_LEN:(i+1)*Resource.MAPHASH_LEN] hashmap += self.m[i*Resource.MAPHASH_LEN:(i+1)*Resource.MAPHASH_LEN]
dictionary = { "t": self.t, # Transfer size dictionary = {
"t": self.t, # Transfer size
"d": self.d, # Data size "d": self.d, # Data size
"n": self.n, # Number of parts "n": self.n, # Number of parts
"h": self.h, # Resource hash "h": self.h, # Resource hash
@ -1370,15 +1349,11 @@ class ResourceAdvertisement:
"l": self.l, # Total segments "l": self.l, # Total segments
"q": self.q, # Request ID "q": self.q, # Request ID
"f": self.f, # Resource flags "f": self.f, # Resource flags
"m": hashmap } "m": hashmap
}
packed_adv = umsgpack.packb(dictionary) return umsgpack.packb(dictionary)
if len(packed_adv) > self.mdu:
RNS.log(f"Resource advertisement ({RNS.prettysize(len(packed_adv))}) exceeds link MDU, deferring initial HMU", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
dictionary["m"] = b""
packed_adv = umsgpack.packb(dictionary)
return packed_adv
@staticmethod @staticmethod
def unpack(data): def unpack(data):
@ -1402,8 +1377,5 @@ class ResourceAdvertisement:
adv.u = True if ((adv.f >> 3) & 0x01) == 0x01 else False adv.u = True if ((adv.f >> 3) & 0x01) == 0x01 else False
adv.p = True if ((adv.f >> 4) & 0x01) == 0x01 else False adv.p = True if ((adv.f >> 4) & 0x01) == 0x01 else False
adv.x = True if ((adv.f >> 5) & 0x01) == 0x01 else False adv.x = True if ((adv.f >> 5) & 0x01) == 0x01 else False
adv.a = True if ((adv.f >> 6) & 0x01) == 0x01 else False
if adv.t > Resource.MAX_EFFICIENT_SIZE*3: raise ValueError("Invalid transfer size")
return adv return adv

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -1,851 +0,0 @@
# Reticulum License
#
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# - The Software shall not be used in any kind of system which includes amongst
# its functions the ability to purposefully do harm to human beings.
#
# - The Software shall not be used, directly or indirectly, in the creation of
# an artificial intelligence, machine learning or language model training
# dataset, including but not limited to any use that contributes to the
# training or development of such a model or algorithm.
#
# - The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import RNS
import hashlib
import struct
import sys
import threading
import os
import time
from array import array
class AnnounceRateTable:
# Columnar store for announce rate tracking. Layout:
# _index : dict[destination_hash -> slot]
# _keys : list[slot] -> destination_hash (for swap-remove bookkeeping)
# _last : array('d') last non-violating announce time
# _blocked: array('d') blocked_until
# _viol : array('i') rate_violations
# _ts : array('d') flat ring of max_timestamps doubles per slot
# _ts_len : array('B') valid timestamp count per slot
# _ts_head: array('B') ring write cursor per slot
def __init__(self, max_timestamps=16):
assert max_timestamps <= 255, "Timestamp ring size must fit in a single byte"
self.max_ts = max_timestamps
self._index = {}
self._keys = []
self._last = array('d')
self._blocked = array('d')
self._viol = array('i')
self._ts = array('d')
self._ts_len = array('B')
self._ts_head = array('B')
self._cursor = 0
def __len__(self): return len(self._keys)
def __contains__(self, destination_hash): return destination_hash in self._index
# Perform the announce-rate check-and-update for one announce.
# Caller must hold the announce_rate_table_lock.
def check_and_update(self, destination_hash, now, target, grace, penalty):
slot = self._index.get(destination_hash)
if slot is None:
slot = len(self._keys)
self._index[destination_hash] = slot
self._keys.append(destination_hash)
self._last.append(now)
self._blocked.append(0.0)
self._viol.append(0)
self._ts.extend([0.0]*self.max_ts)
self._ts[slot*self.max_ts] = now
self._ts_len.append(1)
self._ts_head.append(1 % self.max_ts)
return False
base = slot*self.max_ts
head = self._ts_head[slot]
self._ts[base+head] = now
self._ts_head[slot] = (head+1) % self.max_ts
if self._ts_len[slot] < self.max_ts: self._ts_len[slot] += 1
current_rate = now - self._last[slot]
rate_blocked = False
if now > self._blocked[slot]:
if current_rate < target: self._viol[slot] += 1
else:
v = self._viol[slot] - 1
self._viol[slot] = v if v > 0 else 0
if self._viol[slot] > grace:
self._blocked[slot] = self._last[slot] + target + penalty
rate_blocked = True
else: self._last[slot] = now
else: rate_blocked = True
return rate_blocked
# Caller must hold the table lock
def cull(self, now, ttl, max_scan=1024, max_cull=None):
if max_cull is None: max_cull = max_scan
horizon = now - ttl
culled = 0
scanned = 0
n = len(self._keys)
while scanned < max_scan and culled < max_cull and n > 0:
if self._cursor >= n: self._cursor = 0
slot = self._cursor
if self._last[slot] < horizon and self._blocked[slot] <= now:
last_i = n - 1
stale_key = self._keys[slot]
if slot != last_i:
moved_key = self._keys[last_i]
self._keys[slot] = moved_key
self._index[moved_key] = slot
self._last[slot] = self._last[last_i]
self._blocked[slot] = self._blocked[last_i]
self._viol[slot] = self._viol[last_i]
self._ts_len[slot] = self._ts_len[last_i]
self._ts_head[slot] = self._ts_head[last_i]
f = last_i*self.max_ts; t = slot*self.max_ts
self._ts[t:t+self.max_ts] = self._ts[f:f+self.max_ts]
self._keys.pop()
self._last.pop(); self._blocked.pop(); self._viol.pop()
self._ts_len.pop(); self._ts_head.pop()
del self._ts[len(self._ts)-self.max_ts:] # drop tail block of removed slot
del self._index[stale_key]
n -= 1
culled += 1
# Do not advance cursor, re-examine the slot a moved entry landed in
else:
self._cursor = slot + 1
scanned += 1
if self._cursor >= n: self._cursor = 0
return culled
# Hold table lock, or accept a possibly inconsistent snapshot
def items(self):
for slot, key in enumerate(self._keys):
n = self._ts_len[slot]
head = self._ts_head[slot]
base = slot*self.max_ts
if n < self.max_ts: ts = list(self._ts[base:base+n])
else: ts = list(self._ts[base+head:base+self.max_ts]) + list(self._ts[base:base+head])
yield key, { "last": self._last[slot], "rate_violations": self._viol[slot],
"blocked_until": self._blocked[slot], "timestamps": ts }
# Persists the announce rate table to a file, in
# compacted, little-endian binary format:
#
# Header (16 bytes)
# 0x00 6 Magic bytes
# 0x06 1 Format version
# 0x07 1 Flags, reserved, must be zero
# 0x08 1 Timestamp ring size (max_ts)
# 0x09 1 Destination hash length in bytes (keylen)
# 0x0A 2 Reserved, zero
# 0x0C 4 Entry count, uint32, little-endian
#
# Entries (columnar, in native slot order)
# Destination hash keys, followed by the _last, _blocked,
# _viol, _ts_len, _ts_head and _ts columns, all encoded as
# little-endian fixed-width values.
#
# Caller must hold the announce rate table lock
PERSIST_MAGIC = b"RNSART"
PERSIST_VERSION = 0x01
def persist(self, path, retain=None):
try:
count = len(self._keys)
if count > 0:
keylen = len(self._keys[0])
for key in self._keys:
if len(key) != keylen: raise ValueError("Inconsistent destination hash lengths in table")
else:
keylen = RNS.Identity.TRUNCATED_HASHLENGTH//8
if retain != None and count > 0:
now = time.time()
horizon = now - retain
keep = [slot for slot in range(count) if not (self._last[slot] < horizon and self._blocked[slot] <= now)]
else:
keep = None
if keep != None and len(keep) != count:
count = len(keep)
keys = [self._keys[slot] for slot in keep]
last = array("d", [self._last[slot] for slot in keep])
blocked = array("d", [self._blocked[slot] for slot in keep])
viol = array("i", [self._viol[slot] for slot in keep])
ts_len = array("B", [self._ts_len[slot] for slot in keep])
ts_head = array("B", [self._ts_head[slot] for slot in keep])
ts = array("d")
for slot in keep: ts.extend(self._ts[slot*self.max_ts:(slot+1)*self.max_ts])
else:
keys = self._keys
last = self._last
blocked = self._blocked
viol = self._viol
ts_len = self._ts_len
ts_head = self._ts_head
ts = self._ts
if last.itemsize != 8 or viol.itemsize != 4:
raise ValueError(f"Unsupported array itemsize at persist: d={last.itemsize}, i={viol.itemsize}")
header = self.PERSIST_MAGIC
header += bytes([self.PERSIST_VERSION, 0x00, self.max_ts & 0xFF, keylen & 0xFF])
header += struct.pack("<H", 0)
header += struct.pack("<I", count)
blob = header
blob += b"".join(keys)
blob += self._column_le(last)
blob += self._column_le(blocked)
blob += self._column_le(viol)
blob += ts_len.tobytes()
blob += ts_head.tobytes()
blob += self._column_le(ts)
tmp_path = path+".tmp"
with open(tmp_path, "wb") as file: file.write(blob)
os.replace(tmp_path, path)
return True
except Exception as e:
RNS.log(f"Could not persist announce rate table to {path}: {e}", RNS.LOG_ERROR)
return False
# Caller must hold the announce rate table lock
def load(self, path):
try:
with open(path, "rb") as file: blob = file.read()
except Exception: return False
try:
if len(blob) < 16: return False
if blob[0:6] != self.PERSIST_MAGIC: return False
if blob[6] != self.PERSIST_VERSION: return False
if blob[7] != 0x00: return False
file_max_ts = blob[8]
keylen = blob[9]
count = struct.unpack_from("<I", blob, 12)[0]
if file_max_ts < 1 or file_max_ts > 255: return False
if keylen < 1: return False
entry_size = keylen+22+file_max_ts*8
if len(blob) != 16+count*entry_size: return False
if file_max_ts == self.max_ts:
# Ring size matches, load directly
keys = [blob[16+slot*keylen : 16+(slot+1)*keylen] for slot in range(count)]
index = {}
for slot in range(count):
if keys[slot] in index: return False
index[keys[slot]] = slot
offset = 16+count*keylen
last = self._column_from_le("d", blob[offset:offset+count*8]); offset += count*8
blocked = self._column_from_le("d", blob[offset:offset+count*8]); offset += count*8
viol = self._column_from_le("i", blob[offset:offset+count*4]); offset += count*4
ts_len = array("B", blob[offset:offset+count]); offset += count
ts_head = array("B", blob[offset:offset+count]); offset += count
ts = self._column_from_le("d", blob[offset:offset+count*self.max_ts*8]); offset += count*self.max_ts*8
for slot in range(count):
if ts_len[slot] > self.max_ts or ts_head[slot] >= self.max_ts: return False
else:
# Different ring sizes from current running config,
# rebuild entries best effort.
keys = [blob[16+slot*keylen : 16+(slot+1)*keylen] for slot in range(count)]
index = {}
for slot in range(count):
if keys[slot] in index: return False
index[keys[slot]] = slot
offset = 16+count*keylen
last = self._column_from_le("d", blob[offset:offset+count*8]); offset += count*8
blocked = self._column_from_le("d", blob[offset:offset+count*8]); offset += count*8
viol = self._column_from_le("i", blob[offset:offset+count*4]); offset += count*4
f_ts_len = array("B", blob[offset:offset+count]); offset += count
f_ts_head = array("B", blob[offset:offset+count]); offset += count
f_ts = self._column_from_le("d", blob[offset:offset+count*file_max_ts*8]); offset += count*file_max_ts*8
for slot in range(count):
if f_ts_len[slot] > file_max_ts or f_ts_head[slot] >= file_max_ts: return False
ts_len = array("B"); ts_head = array("B"); ts = array("d")
for slot in range(count):
base = slot*file_max_ts
if f_ts_len[slot] < file_max_ts: chronological = list(f_ts[base:base+f_ts_len[slot]])
else: chronological = list(f_ts[base+f_ts_head[slot]:base+file_max_ts])+list(f_ts[base:base+f_ts_head[slot]])
chronological = chronological[-self.max_ts:]
ts_count = len(chronological)
ts_len.append(ts_count)
ts.extend(chronological)
if ts_count < self.max_ts: ts.extend([0.0]*(self.max_ts-ts_count))
ts_head.append(ts_count % self.max_ts)
self._index = index
self._keys = keys
self._last = last
self._blocked = blocked
self._viol = viol
self._ts = ts
self._ts_len = ts_len
self._ts_head = ts_head
self._cursor = 0
return True
except Exception as e:
RNS.log(f"Could not load announce rate table from {path}: {e}", RNS.LOG_ERROR)
return False
def _column_le(self, column):
if sys.byteorder == "big": swapped = array(column.typecode, column); swapped.byteswap(); return swapped.tobytes()
else: return column.tobytes()
def _column_from_le(self, typecode, data):
column = array(typecode); column.frombytes(data)
if sys.byteorder == "big": column.byteswap()
return column
class PacketHashlist:
# Bucketed byte-blob packet filter hashlist, with the primary
# design goal of memory coherence and density. All operations
# are safe to call without holding a lock.
def __init__(self, hash_width=32, bucket_count=16384):
assert bucket_count > 0 and bucket_count & (bucket_count-1) == 0, "Bucket count must be a power of two"
self.width = hash_width
self._nb = bucket_count
self._mask = bucket_count - 1
self._cur = [bytearray() for _ in range(bucket_count)]
self._prv = [bytearray() for _ in range(bucket_count)]
self._count = 0
def __len__(self):
return self._count
def __contains__(self, packet_hash):
bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask
return packet_hash in self._cur[bucket] or packet_hash in self._prv[bucket]
def add(self, packet_hash):
assert len(packet_hash) == self.width, "Packet hash width mismatch"
bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask
if packet_hash not in self._cur[bucket]:
self._cur[bucket].extend(packet_hash)
self._count += 1
def discard(self, packet_hash):
w = self.width
bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask
for generation in (self._cur, self._prv):
entries = generation[bucket]
pos = entries.find(packet_hash)
while pos != -1:
if pos % w == 0:
del entries[pos:pos+w]
if generation is self._cur: self._count -= 1
pos = entries.find(packet_hash, pos)
else:
pos = entries.find(packet_hash, pos+1)
def swap(self):
self._prv = self._cur
self._cur = [bytearray() for _ in range(self._nb)]
self._count = 0
# Current-generation buckets for persist operations
def buckets(self): return tuple(self._cur)
class KnownDestinationsStore:
# Write-behind SQLite storage backend for known destinations.
# The actual in-memory table is used directly for all transport
# operations; this serves purely as the store, and no read operations
# utilize the database directly.
def __init__(self, db_path, hash_width=16, read_only=False):
import sqlite3
self.sqlite3 = sqlite3
self.db_path = db_path
self.hash_width = hash_width
self.read_only = read_only
self._dirty_upsert = set()
self._dirty_delete = set()
self._ratchet_upserts = {}
self._ratchet_deletes = set()
self._kd_quarantine = {}
self._ratchet_quarantine = {}
self._flush_lock = threading.Lock()
if read_only: self._conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, check_same_thread=False, isolation_level=None, cached_statements=0)
else:
self._conn = sqlite3.connect(db_path, check_same_thread=False, isolation_level=None, cached_statements=0)
self._conn.execute("PRAGMA journal_mode=WAL")
self._conn.execute("PRAGMA synchronous=NORMAL")
self._conn.execute("PRAGMA journal_size_limit=8388608")
self.__create_schema()
def close(self):
if self._conn != None:
self._conn.close()
self._conn = None
def __create_schema(self):
db_version = self._conn.execute("PRAGMA user_version").fetchone()[0]
# Pre-release database cleanup
# TODO: Remove this in 1.6.1
if db_version > 0:
RNS.log(f"Pre-release database detected, rebuilding known destinations DB to release format", RNS.LOG_WARNING)
self._conn.execute("DROP TABLE IF EXISTS known_destinations")
self._conn.execute("""CREATE TABLE IF NOT EXISTS known_destinations (
destination_hash BLOB PRIMARY KEY,
last_heard REAL NOT NULL,
packet_hash BLOB,
public_key BLOB NOT NULL,
app_data BLOB,
used REAL NOT NULL DEFAULT 0,
stamp BLOB,
stamp_value REAL,
lr_cost REAL,
identity_hash BLOB NOT NULL)""")
self._conn.execute("""CREATE TABLE IF NOT EXISTS ratchets (
destination_hash BLOB PRIMARY KEY,
ratchet BLOB NOT NULL,
received REAL NOT NULL)""")
# Schema migrations
try:
self._conn.execute("BEGIN")
# No migrations yet
self._conn.execute("PRAGMA user_version = 0")
self._conn.execute("COMMIT")
self._conn.commit()
except Exception as e:
RNS.log(f"Known destinations database version migration failed: {e}", RNS.LOG_CRITICAL)
RNS.log(f"You will need to remove the known_destinations.db* files from your storage directory so it can be rebuilt", RNS.LOG_CRITICAL)
def __entry_row(self, destination_hash, entry):
stamp = entry[5] if len(entry) > 5 else None
stamp_value = entry[6] if len(entry) > 6 else 0
lr_cost = entry[7] if len(entry) > 7 else None
identity_hash = hashlib.sha256(entry[2]).digest()[:self.hash_width]
return (destination_hash, entry[0], entry[1], entry[2], entry[3], entry[4], stamp, stamp_value, lr_cost, identity_hash)
# Caller must hold Identity.known_destinations_lock
def mark_dirty(self, destination_hash):
if self.read_only: return
self._dirty_delete.discard(destination_hash)
self._kd_quarantine.pop(destination_hash, None)
self._dirty_upsert.add(destination_hash)
# Caller must hold Identity.known_destinations_lock
def mark_deleted(self, destination_hash):
if self.read_only: return
self._dirty_upsert.discard(destination_hash)
self._dirty_delete.add(destination_hash)
self._kd_quarantine.pop(destination_hash, None)
self._ratchet_quarantine.pop(destination_hash, None)
# Caller must hold Identity.known_destinations_lock
def take_pending(self):
if self.read_only: return set(), set(), {}, set()
upserts = self._dirty_upsert
deletes = self._dirty_delete
ratchet_upserts = self._ratchet_upserts
ratchet_deletes = self._ratchet_deletes
self._dirty_upsert = set()
self._dirty_delete = set()
self._ratchet_upserts = {}
self._ratchet_deletes = set()
return upserts, deletes, ratchet_upserts, ratchet_deletes
# Caller must hold the flush lock and have an already open transaction
def __wrap_exec(self, sql, rows):
rejects = []
# By default, run batched queries in chunks.
for i in range(0, len(rows), 4096):
chunk = rows[i:i+4096]
try:
self._conn.execute("SAVEPOINT chunk")
self._conn.executemany(sql, chunk)
self._conn.execute("RELEASE chunk")
# But on integrity and data errors, roll back the chunk
# and iterate per-row, so we can catch any violating
# entries and quarantine them. A later, sane update
# will then unquarantine and heal upon arrival.
#
# Other exceptions are intentionally not caught here,
# so they can propagate directly.
except (self.sqlite3.IntegrityError, self.sqlite3.DataError):
self._conn.execute("ROLLBACK TO chunk")
for row in chunk:
try:
self._conn.execute("SAVEPOINT row")
self._conn.execute(sql, row)
self._conn.execute("RELEASE row")
except (self.sqlite3.IntegrityError, self.sqlite3.DataError) as e:
self._conn.execute("ROLLBACK TO row")
rejects.append((row[0], str(e)))
self._conn.execute("RELEASE chunk")
return rejects
# May be called without any locks held
def apply(self, upserts, deletes, ratchet_upserts, ratchet_deletes, known_destinations):
if self.read_only: return 0
if len(upserts) == 0 and len(deletes) == 0 and len(ratchet_upserts) == 0 and len(ratchet_deletes) == 0: return 0
with self._flush_lock:
rows = []
removables = []
ratchet_rows = []
ratchet_removables = []
for destination_hash in upserts:
if destination_hash in self._kd_quarantine: continue
entry = known_destinations.get(destination_hash)
if entry != None: rows.append(self.__entry_row(destination_hash, entry))
for destination_hash in deletes:
if destination_hash not in known_destinations: removables.append((destination_hash,))
for destination_hash, ratchet_entry in ratchet_upserts.items():
# Ratchets of purged destinations must be removed as well
if destination_hash not in known_destinations: ratchet_removables.append((destination_hash,))
elif destination_hash not in self._ratchet_quarantine: ratchet_rows.append((destination_hash, *ratchet_entry))
for destination_hash in ratchet_deletes:
if destination_hash not in known_destinations: ratchet_removables.append((destination_hash,))
kd_rejects = []
ratchet_rejects = []
try:
self._conn.execute("BEGIN")
kd_rejects += self.__wrap_exec("INSERT INTO known_destinations (destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value, lr_cost, identity_hash) VALUES (?,?,?,?,?,?,?,?,?,?) ON CONFLICT(destination_hash) DO UPDATE SET last_heard=excluded.last_heard, packet_hash=excluded.packet_hash, public_key=excluded.public_key, app_data=excluded.app_data, used=excluded.used, stamp=excluded.stamp, stamp_value=excluded.stamp_value, lr_cost=excluded.lr_cost, identity_hash=excluded.identity_hash", rows)
kd_rejects += self.__wrap_exec("DELETE FROM known_destinations WHERE destination_hash=?", removables)
ratchet_rejects += self.__wrap_exec("INSERT INTO ratchets VALUES (?,?,?) ON CONFLICT(destination_hash) DO UPDATE SET ratchet=excluded.ratchet, received=excluded.received", ratchet_rows)
ratchet_rejects += self.__wrap_exec("DELETE FROM ratchets WHERE destination_hash=?", ratchet_removables)
self._conn.commit()
except Exception:
# If an exception is not handled in the batch query wrapper,
# we will have to roll back the transaction, and mark all
# attempted entries as dirty. A failure state like such
# should not happen, and this is a last resort; at this
# point, writes may stall until an updated entry for
# the violating row(s) arrive and heal the invalid
# dirty entries, so this is raised for logging.
self._dirty_upsert |= {row[0] for row in rows}
self._dirty_delete |= {row[0] for row in removables}
self._ratchet_upserts |= {row[0]: row[1:] for row in ratchet_rows}
self._ratchet_deletes |= {row[0] for row in ratchet_removables}
self._conn.rollback()
raise
# Track failed persists and one-time log per failure. A later
# update of the destination clears the quarantine state and
# allows persist to retry with the (hopefully) valid data.
if len(kd_rejects) > 0 or len(ratchet_rejects) > 0:
for destination_hash, error in kd_rejects:
self._kd_quarantine[destination_hash] = (error, time.time())
RNS.log(f"Invalid known destination data for {RNS.prettyhexrep(destination_hash)} at persist. The contained exception was: {error}", RNS.LOG_ERROR)
for destination_hash, error in ratchet_rejects:
self._ratchet_quarantine[destination_hash] = (error, time.time())
RNS.log(f"Invalid ratchet data for {RNS.prettyhexrep(destination_hash)} at persist. The contained exception was: {error}", RNS.LOG_ERROR)
return len(rows)+len(removables)+len(ratchet_rows)+len(ratchet_removables)-len(kd_rejects)-len(ratchet_rejects)
def load_all(self):
cursor = self._conn.execute("SELECT destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value, lr_cost FROM known_destinations")
while True:
rows = cursor.fetchmany(4096)
if not rows: break
for row in rows: yield row[0], [row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8]]
def remember_ratchet(self, destination_hash, ratchet, now):
if self.read_only: return False
self._ratchet_quarantine.pop(destination_hash, None)
pending = self._ratchet_upserts.get(destination_hash)
if pending != None:
if pending[0] == ratchet:
self._ratchet_deletes.discard(destination_hash)
return False
else:
with self._flush_lock:
row = self._conn.execute("SELECT ratchet FROM ratchets WHERE destination_hash=?", (destination_hash,)).fetchone()
if row != None and row[0] == ratchet:
# Discard any pending delete mark queued
# before this ratchet arrived
self._ratchet_deletes.discard(destination_hash)
return False
self._ratchet_upserts[destination_hash] = (ratchet, now)
self._ratchet_deletes.discard(destination_hash)
return True
def fetch_ratchet(self, destination_hash):
pending = self._ratchet_upserts.get(destination_hash)
if pending != None: return pending[0], pending[1]
if destination_hash in self._ratchet_deletes: return None
with self._flush_lock:
row = self._conn.execute("SELECT ratchet, received FROM ratchets WHERE destination_hash=?", (destination_hash,)).fetchone()
if row != None: return row[0], row[1]
return None
# Caller must hold Identity.known_destinations_lock
def mark_ratchet_deleted(self, destination_hash):
if self.read_only: return
self._ratchet_upserts.pop(destination_hash, None)
self._ratchet_deletes.add(destination_hash)
self._ratchet_quarantine.pop(destination_hash, None)
def expire_ratchets(self, cutoff):
if self.read_only: return []
with self._flush_lock:
# Drop expired entries in pending as well
expired = set(row[0] for row in self._conn.execute("SELECT destination_hash FROM ratchets WHERE received<?", (cutoff,)))
for destination_hash, entry in list(self._ratchet_upserts.items()):
if entry[1] < cutoff:
expired.add(destination_hash)
self._ratchet_upserts.pop(destination_hash, None)
expired.difference_update(self._ratchet_deletes)
if len(expired) > 0:
self._conn.execute("BEGIN")
for i in range(0, len(expired), 4096):
self._conn.executemany("DELETE FROM ratchets WHERE destination_hash=?", [(h,) for h in list(expired)[i:i+4096]])
self._conn.commit()
return list(expired)
def load_ratchets(self):
cursor = self._conn.execute("SELECT destination_hash, ratchet, received FROM ratchets")
while True:
rows = cursor.fetchmany(4096)
if not rows: break
for row in rows: yield row[0], row[1], row[2]
def import_legacy_ratchets(self, dir_path, expiry_cutoff, known_destinations, ratchet_width=32):
from .vendor import umsgpack
imported = 0
for filename in os.listdir(dir_path):
file_path = os.path.join(dir_path, filename)
try:
if not os.path.isfile(file_path): continue
with open(file_path, "rb") as rf:
ratchet_data = umsgpack.unpackb(rf.read())
destination_hash = bytes.fromhex(filename)
ratchet = ratchet_data["ratchet"]; received = ratchet_data["received"]
if len(ratchet) != ratchet_width: raise ValueError("Invalid ratchet width")
if received < expiry_cutoff: raise ValueError("Ratchet expired")
if destination_hash not in known_destinations: raise ValueError("Destination unknown")
self._ratchet_upserts[destination_hash] = (ratchet, received)
imported += 1
os.unlink(file_path)
except Exception as e:
RNS.log(f"Could not import ratchet for {filename}: {e}", RNS.LOG_WARNING)
try: os.unlink(file_path)
except: pass
pending = self._ratchet_upserts
self._ratchet_upserts = {}
self.apply(set(), set(), pending, set(), known_destinations)
return imported
def import_legacy(self, path):
from .vendor import umsgpack
with open(path, "rb") as file: legacy = umsgpack.load(file)
rows = []
now = time.time()
for destination_hash, entry in legacy.items():
if len(entry) < 5: continue
entry = list(entry)
while len(entry) < 8: entry.append(None if len(entry) in (5, 7) else 0)
if entry[0] == None or entry[0] <= 0: entry[0] = now
rows.append(self.__entry_row(destination_hash, entry))
self._conn.execute("BEGIN")
for i in range(0, len(rows), 4096): self._conn.executemany("INSERT OR REPLACE INTO known_destinations (destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value, lr_cost, identity_hash) VALUES (?,?,?,?,?,?,?,?,?,?)", rows[i:i+4096])
self._conn.commit()
return len(rows)
class AnnounceCache:
def __init__(self, db_path):
import sqlite3
self.db_path = db_path
self._pending = {}
self._flush_lock = threading.Lock()
self._conn = sqlite3.connect(db_path, check_same_thread=False, isolation_level=None, cached_statements=0)
self._conn.execute("PRAGMA auto_vacuum=INCREMENTAL")
self._conn.execute("PRAGMA journal_mode=WAL")
self._conn.execute("PRAGMA synchronous=NORMAL")
self._conn.execute("PRAGMA journal_size_limit=8388608")
self.__create_schema()
def __create_schema(self):
self._conn.execute("""CREATE TABLE IF NOT EXISTS cached_announces (
packet_hash BLOB PRIMARY KEY,
received REAL NOT NULL,
raw BLOB NOT NULL,
interface_reference TEXT)""")
self._conn.execute("PRAGMA user_version = 1")
def store(self, packet_hash, raw, interface_reference):
self._pending[packet_hash] = (time.time(), raw, interface_reference)
def pending_count(self):
return len(self._pending)
def fetch(self, packet_hash):
pending = self._pending.get(packet_hash)
if pending != None: return pending[1], pending[2]
with self._flush_lock:
row = self._conn.execute("SELECT raw, interface_reference FROM cached_announces WHERE packet_hash=?", (packet_hash,)).fetchone()
if row != None: return row[0], row[1]
return None
def flush(self):
with self._flush_lock: return self.__apply_pending()
def __apply_pending(self):
pending = self._pending
self._pending = {}
if len(pending) == 0: return 0
try:
self._conn.execute("BEGIN")
items = list(pending.items())
for i in range(0, len(items), 4096):
self._conn.executemany("INSERT INTO cached_announces VALUES (?,?,?,?) ON CONFLICT(packet_hash) DO UPDATE SET received=excluded.received, raw=excluded.raw, interface_reference=excluded.interface_reference",
[(packet_hash, *entry) for packet_hash, entry in items[i:i+4096]])
self._conn.commit()
except Exception:
# Re-mark everything as pending, so the changes
# are retried on the next flush, then propagate.
self._pending.update(pending)
self._conn.rollback()
raise
return len(pending)
def clean(self, active_hashes):
with self._flush_lock:
self.__apply_pending()
try:
self._conn.execute("BEGIN")
self._conn.execute("DROP TABLE IF EXISTS temp.active_announces")
self._conn.execute("CREATE TEMP TABLE active_announces (packet_hash BLOB PRIMARY KEY)")
active_list = list(active_hashes)
for i in range(0, len(active_list), 4096):
self._conn.executemany("INSERT OR IGNORE INTO active_announces VALUES (?)", [(h,) for h in active_list[i:i+4096]])
changes_before = self._conn.total_changes
self._conn.execute("DELETE FROM cached_announces WHERE packet_hash NOT IN (SELECT packet_hash FROM active_announces)")
removed = self._conn.total_changes - changes_before
self._conn.commit()
except Exception:
self._conn.rollback()
raise
if removed > 0: self._conn.execute("PRAGMA incremental_vacuum")
return removed
def import_legacy(self, dir_path):
from .vendor import umsgpack
imported = 0
for filename in os.listdir(dir_path):
file_path = os.path.join(dir_path, filename)
try:
if not os.path.isfile(file_path): continue
with open(file_path, "rb") as file:
cached_data = umsgpack.unpackb(file.read())
packet_hash = bytes.fromhex(filename)
interface_reference = cached_data[1] if len(cached_data) > 1 else None
self.store(packet_hash, cached_data[0], interface_reference)
imported += 1
os.unlink(file_path)
except Exception:
try: os.unlink(file_path)
except Exception: pass
self.flush()
return imported
def close(self):
if self._conn != None:
self._conn.close()
self._conn = None

File diff suppressed because it is too large Load diff

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -33,7 +33,5 @@ import glob
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py") py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc") pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
so_modules = glob.glob(os.path.dirname(__file__)+"/*.so") modules = py_modules+pyc_modules
pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd") __all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
modules = py_modules+pyc_modules+so_modules+pyd_modules
__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))

View file

@ -2,7 +2,7 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -387,7 +387,6 @@ def fetch(configdir, identitypath = None, verbosity = 0, quietness = 0, destinat
RNS.exit(1) RNS.exit(1)
reticulum = RNS.Reticulum(configdir=configdir, loglevel=targetloglevel) reticulum = RNS.Reticulum(configdir=configdir, loglevel=targetloglevel)
reticulum.wants_destination_type(f"{APP_NAME}.receive")
if identity == None: if identity == None:
prepare_identity(identitypath) prepare_identity(identitypath)
@ -402,7 +401,7 @@ def fetch(configdir, identitypath = None, verbosity = 0, quietness = 0, destinat
i = 0 i = 0
syms = "⢄⢂⢁⡁⡈⡐⡠" syms = "⢄⢂⢁⡁⡈⡐⡠"
estab_timeout = time.time()+max(timeout, reticulum.get_medium_path_timeout()) estab_timeout = time.time()+timeout
while not RNS.Transport.has_path(destination_hash) and time.time() < estab_timeout: while not RNS.Transport.has_path(destination_hash) and time.time() < estab_timeout:
if not silent: if not silent:
time.sleep(0.1) time.sleep(0.1)
@ -432,7 +431,6 @@ def fetch(configdir, identitypath = None, verbosity = 0, quietness = 0, destinat
) )
link = RNS.Link(listener_destination) link = RNS.Link(listener_destination)
estab_timeout = max(estab_timeout, time.time()+link.establishment_timeout)
while link.status != RNS.Link.ACTIVE and time.time() < estab_timeout: while link.status != RNS.Link.ACTIVE and time.time() < estab_timeout:
if not silent: if not silent:
time.sleep(0.1) time.sleep(0.1)
@ -644,7 +642,6 @@ def send(configdir, identitypath = None, verbosity = 0, quietness = 0, destinati
print(f"{erase_str}", end="") print(f"{erase_str}", end="")
reticulum = RNS.Reticulum(configdir=configdir, loglevel=targetloglevel) reticulum = RNS.Reticulum(configdir=configdir, loglevel=targetloglevel)
reticulum.wants_destination_type(f"{APP_NAME}.receive")
if identity == None: if identity == None:
prepare_identity(identitypath) prepare_identity(identitypath)
@ -659,7 +656,7 @@ def send(configdir, identitypath = None, verbosity = 0, quietness = 0, destinati
i = 0 i = 0
syms = "⢄⢂⢁⡁⡈⡐⡠" syms = "⢄⢂⢁⡁⡈⡐⡠"
estab_timeout = time.time()+max(timeout, reticulum.get_medium_path_timeout()) estab_timeout = time.time()+timeout
while not RNS.Transport.has_path(destination_hash) and time.time() < estab_timeout: while not RNS.Transport.has_path(destination_hash) and time.time() < estab_timeout:
if not silent: if not silent:
time.sleep(0.1) time.sleep(0.1)
@ -689,7 +686,6 @@ def send(configdir, identitypath = None, verbosity = 0, quietness = 0, destinati
) )
link = RNS.Link(receiver_destination) link = RNS.Link(receiver_destination)
estab_timeout = max(estab_timeout, time.time()+link.establishment_timeout)
while link.status != RNS.Link.ACTIVE and time.time() < estab_timeout: while link.status != RNS.Link.ACTIVE and time.time() < estab_timeout:
if not silent: if not silent:
time.sleep(0.1) time.sleep(0.1)
@ -882,8 +878,8 @@ def main():
except KeyboardInterrupt: except KeyboardInterrupt:
print("") print("")
if current_resource != None: if resource != None:
current_resource.cancel() resource.cancel()
if link != None: if link != None:
link.teardown() link.teardown()
RNS.exit() RNS.exit()

View file

@ -35,7 +35,5 @@ import glob
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py") py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc") pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
so_modules = glob.glob(os.path.dirname(__file__)+"/*.so") modules = py_modules+pyc_modules
pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd") __all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
modules = py_modules+pyc_modules+so_modules+pyd_modules
__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))

View file

@ -40,7 +40,6 @@ import subprocess
from RNS._version import __version__ from RNS._version import __version__
from RNS.Utilities.rngit import APP_NAME from RNS.Utilities.rngit import APP_NAME
from RNS.Utilities.rngit.util import medium_path_timeout
from RNS.vendor.configobj import ConfigObj from RNS.vendor.configobj import ConfigObj
from tempfile import TemporaryDirectory from tempfile import TemporaryDirectory
@ -154,8 +153,6 @@ class ReticulumGitClient():
print(f"Failed to initialize Reticulum: {e}", file=sys.stderr) print(f"Failed to initialize Reticulum: {e}", file=sys.stderr)
return return
self.reticulum.wants_destination_type(f"{APP_NAME}.repositories")
self.__apply_config() self.__apply_config()
self.ready = True self.ready = True
@ -226,7 +223,7 @@ class ReticulumGitClient():
RNS.log(f"Requesting path to {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) RNS.log(f"Requesting path to {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG)
sys.stderr.write(f"Requesting path..."); sys.stderr.flush() sys.stderr.write(f"Requesting path..."); sys.stderr.flush()
if not RNS.Transport.await_path(destination_hash, timeout=medium_path_timeout(self.path_timeout)): if not RNS.Transport.await_path(destination_hash, timeout=self.path_timeout):
sys.stderr.write(f"\n"); sys.stderr.flush() sys.stderr.write(f"\n"); sys.stderr.flush()
self.abort(f"Could not resolve path to {RNS.prettyhexrep(destination_hash)}") self.abort(f"Could not resolve path to {RNS.prettyhexrep(destination_hash)}")
@ -294,8 +291,6 @@ class ReticulumGitClient():
if hasattr(self.request_response, "read") and callable(self.request_response.read): if hasattr(self.request_response, "read") and callable(self.request_response.read):
response_path = self.request_response.name response_path = self.request_response.name
try: self.request_response.close()
except Exception as e: RNS.log(f"Could not close resource file descriptor before retain: {e}", RNS.LOG_ERROR)
base_name = os.path.basename(response_path) base_name = os.path.basename(response_path)
retained_path = os.path.join(self.tmp_dir.name, base_name) retained_path = os.path.join(self.tmp_dir.name, base_name)
shutil.move(response_path, retained_path) shutil.move(response_path, retained_path)
@ -338,10 +333,10 @@ class ReticulumGitClient():
try: self.connect_server() try: self.connect_server()
except Exception as e: self.abort(str(e)) except Exception as e: self.abort(str(e))
timeout = max(self.link_timeout, self.link.establishment_timeout) timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0: while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.5) time.sleep(0.5)
timeout -= 0.5 timeout -= 1
if not self.link_ready: self.abort("Failed to establish link") if not self.link_ready: self.abort("Failed to establish link")

View file

@ -1,324 +0,0 @@
#!/usr/bin/env python3
# Reticulum License
#
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# - The Software shall not be used in any kind of system which includes amongst
# its functions the ability to purposefully do harm to human beings.
#
# - The Software shall not be used, directly or indirectly, in the creation of
# an artificial intelligence, machine learning or language model training
# dataset, including but not limited to any use that contributes to the
# training or development of such a model or algorithm.
#
# - The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import os
import sys
import RNS
import struct
import base64
import argparse
from RNS.Utilities.rnid import validate_rsg, create_rsg, extract_signed_rsg_data
SSHSIG_MAGIC = b"SSHSIG"
SSHSIG_VERSION = 1
NAMESPACE_GIT = b"git"
RESERVED_EMPTY = b""
HASH_ALGORITHM = b"sha256"
def ssh_string(data): return struct.pack(">I", len(data)) + data
def read_ssh_string(data, offset):
if offset + 4 > len(data): raise ValueError("Not enough data for string length")
length = struct.unpack(">I", data[offset:offset+4])[0]
if offset + 4 + length > len(data): raise ValueError("Not enough data for string content")
return data[offset+4:offset+4+length], offset + 4 + length
def create_ssh_signature(public_key_wire, namespace, reserved, hash_algorithm, signature_data):
# SSHSIG (6 bytes) || version (uint32) || pubkey (ssh-string) || namespace (ssh-string) ||
# reserved (ssh-string) || hash_algorithm (ssh-string) || signature (ssh-string)
sig_blob = SSHSIG_MAGIC
sig_blob += struct.pack(">I", SSHSIG_VERSION)
sig_blob += ssh_string(public_key_wire)
sig_blob += ssh_string(namespace)
sig_blob += ssh_string(reserved)
sig_blob += ssh_string(hash_algorithm)
sig_blob += ssh_string(signature_data)
return sig_blob
def parse_ssh_signature(sig_data):
offset = 0
if not sig_data.startswith(SSHSIG_MAGIC): raise ValueError("Invalid SSH signature: missing SSHSIG magic")
offset += len(SSHSIG_MAGIC)
if offset + 4 > len(sig_data): raise ValueError("Invalid SSH signature: truncated")
version = struct.unpack(">I", sig_data[offset:offset+4])[0]
if version != SSHSIG_VERSION: raise ValueError(f"Unsupported SSH signature version: {version}")
offset += 4
public_key, offset = read_ssh_string(sig_data, offset)
namespace, offset = read_ssh_string(sig_data, offset)
reserved, offset = read_ssh_string(sig_data, offset)
hash_algorithm, offset = read_ssh_string(sig_data, offset)
signature_data, offset = read_ssh_string(sig_data, offset)
return { "version": version,
"public_key": public_key,
"namespace": namespace,
"reserved": reserved,
"hash_algorithm": hash_algorithm,
"signature_data": signature_data }
def armor_ssh_signature(sig_blob):
b64_data = base64.b64encode(sig_blob).decode('ascii')
lines = [b64_data[i:i+70] for i in range(0, len(b64_data), 70)]
result = "-----BEGIN SSH SIGNATURE-----\n"
result += "\n".join(lines) + "\n"
result += "-----END SSH SIGNATURE-----\n"
return result
def unarmor_ssh_signature(armored_data):
lines = armored_data.strip().split('\n')
b64_data = ""
in_sig = False
for line in lines:
if 'BEGIN SSH SIGNATURE' in line: in_sig = True; continue
if 'END SSH SIGNATURE' in line: break
if in_sig: b64_data += line.strip()
if not b64_data: raise ValueError("No signature data found in armored input")
return base64.b64decode(b64_data)
def get_pubkey_wire_format(identity):
return ssh_string(b"ssh-ed25519")+ssh_string(identity.sig_pub_bytes)
def sign(args):
keyfile = args.keyfile
if not keyfile or not os.path.isfile(keyfile):
print(f"Identity file not found: {keyfile}", file=sys.stderr)
return 1
try:
identity = RNS.Identity.from_file(keyfile)
if not identity or not identity.get_private_key():
print("Error: Could not load identity or identity has no private key", file=sys.stderr)
return 1
except Exception as e:
print(f"Error loading identity: {e}", file=sys.stderr)
return 1
if args.file and os.path.isfile(args.file):
with open(args.file, 'rb') as f: message = f.read()
sig_file = args.file + ".sig"
else:
message = sys.stdin.buffer.read()
sig_file = None
try: rsg = create_rsg(identity, message)
except Exception as e:
print(f"Error creating signature: {e}", file=sys.stderr)
return 1
try: ssh_pubkey = get_pubkey_wire_format(identity)
except Exception as e:
print(f"Error converting public key: {e}", file=sys.stderr)
return 1
try:
ssh_sig = create_ssh_signature(public_key_wire=ssh_pubkey, namespace=NAMESPACE_GIT, reserved=RESERVED_EMPTY,
hash_algorithm=HASH_ALGORITHM, signature_data=rsg)
except Exception as e:
print(f"Error creating SSH signature: {e}", file=sys.stderr)
return 1
try: armored = armor_ssh_signature(ssh_sig)
except Exception as e:
print(f"Error armoring signature: {e}", file=sys.stderr)
return 1
if sig_file:
try:
with open(sig_file, 'w') as f: f.write(armored)
except Exception as e:
print(f"Error writing signature file: {e}", file=sys.stderr)
return 1
else: print(armored, end="")
return 0
def find_principals(args):
sigfile = args.sigfile
if not sigfile or not os.path.isfile(sigfile): print("Error: Signature file not found", file=sys.stderr); return 1
try:
with open(sigfile, 'r') as f: armored_sig = f.read()
except Exception as e: print(f"Error reading signature file: {e}", file=sys.stderr); return 1
try: ssh_sig = parse_ssh_signature(unarmor_ssh_signature(armored_sig))
except Exception as e: print(f"Error parsing SSH signature: {e}", file=sys.stderr); return 1
if ssh_sig["namespace"] != NAMESPACE_GIT:
print(f"Error: Namespace mismatch: {ssh_sig['namespace']}", file=sys.stderr)
return 1
rsg = ssh_sig["signature_data"]
try: identity_hash = extract_signed_rsg_data(rsg)["meta"]["signer"]
except Exception as e: print(f"Could not determine signer identity: {e}", file=sys.stderr); return 1
print(RNS.hexrep(identity_hash, delimit=False))
return 0
def check_novalidate(args):
sigfile = args.sigfile
if not sigfile or not os.path.isfile(sigfile): return 1
try:
with open(sigfile, 'r') as f: armored_sig = f.read()
ssh_sig = parse_ssh_signature(unarmor_ssh_signature(armored_sig))
if ssh_sig["namespace"] != NAMESPACE_GIT: return 1
rsg = ssh_sig["signature_data"]
signed_data = extract_signed_rsg_data(rsg)
if not signed_data: return 1
else: return 0
except Exception: return 1
def extract_commit_author(message):
message_lines = message.splitlines()
author = ""
AUTHOR_TARGET = b"author "
for line in message_lines:
if not line.strip(b""): break
elif line.startswith(AUTHOR_TARGET):
try:
spos = line.find(b"<"); epos = line.find(b">")
if spos > len(AUTHOR_TARGET) and epos > spos and epos < len(line)-1:
author = line[spos+1:epos].decode("utf-8")
break
except Exception as e: print(f"Error while determining author from signed commit"); return 1
return author
def extract_commit_committer(message):
message_lines = message.splitlines()
committer = ""
COMMITTER_TARGET = b"committer "
for line in message_lines:
if not line.strip(b""): break
elif line.startswith(COMMITTER_TARGET):
try:
spos = line.find(b"<"); epos = line.find(b">")
if spos > len(COMMITTER_TARGET) and epos > spos and epos < len(line)-1:
committer = line[spos+1:epos].decode("utf-8")
break
except Exception as e: print(f"Error while determining committer from signed commit"); return 1
return committer
def extract_commit_tagger(message):
message_lines = message.splitlines()
tagger = ""
is_tag = False
for line in message_lines:
TAG_TARGET = b"tag "
TAGGER_TARGET = b"tagger "
if not line.strip(b""): break
elif line.startswith(TAG_TARGET): is_tag = True
elif line.startswith(TAGGER_TARGET) and is_tag:
try:
spos = line.find(b"<"); epos = line.find(b">")
if spos > len(TAGGER_TARGET) and epos > spos and epos < len(line)-1:
tagger = line[spos+1:epos].decode("utf-8")
break
except Exception as e: print(f"Error while determining tagger from signed commit"); return 1
return tagger, is_tag
def verify(args):
sigfile = args.sigfile
principal = args.principal
if not sigfile or not os.path.isfile(sigfile): print("Error: Signature file not found", file=sys.stderr); return 1
message = sys.stdin.buffer.read()
try:
with open(sigfile, 'r') as f: armored_sig = f.read()
raw_sig = unarmor_ssh_signature(armored_sig)
ssh_sig = parse_ssh_signature(raw_sig)
except Exception as e: print(f"Error parsing signature: {e}", file=sys.stderr); return 1
author = extract_commit_author(message)
committer = extract_commit_committer(message)
tagger, is_tag = extract_commit_tagger(message)
if ssh_sig["namespace"] != NAMESPACE_GIT: print(f"Invalid commit signature namespace", file=sys.stderr); return 1
rsg = ssh_sig["signature_data"]
valid, signed_data, signing_identity = validate_rsg(rsg, message)
if not valid: print(f"Invalid signature", file=sys.stderr); return 1
if is_tag: author = tagger
signer_hash = RNS.hexrep(signing_identity.hash, delimit=False)
if not author == signer_hash: print(f"Commit not signed by author <{author}>"); return 1
if principal:
if principal != signer_hash: print(f"Principal mismatch", file=sys.stderr); return 1
print(f"Good \"git\" signature for commit, signed with Reticulum Identity key <{signer_hash}>")
return 0
def main():
parser = argparse.ArgumentParser(description="Git commit signer and validator")
parser.add_argument("-Y", dest="op", required=True, choices=["sign", "find-principals", "check-novalidate", "verify"], help="Operation to perform")
parser.add_argument("-n", dest="namespace", default="git", help="Namespace")
parser.add_argument("-f", dest="keyfile", help="Key file (for signing) or allowed signers file (for verification)")
parser.add_argument("-I", dest="principal", help="Principal identity (for verification)")
parser.add_argument("-s", dest="sigfile", help="Signature file")
parser.add_argument("file", nargs="?", help="File to sign (for signing)")
parser.add_argument("-O", dest="ssh_options", action="append", default=[], help="SSH options (for git compatibility, ignored)")
args, unknown = parser.parse_known_args()
for arg in unknown:
if arg.startswith('-O'): continue # TODO: Add options for time validation
else:
print(f"Error: Unknown argument: {arg}", file=sys.stderr)
sys.exit(1)
if args.op == "sign": return sign(args)
elif args.op == "find-principals": return find_principals(args)
elif args.op == "check-novalidate": return check_novalidate(args)
elif args.op == "verify": return verify(args)
else:
print(f"Error: Unknown operation: {args.op}", file=sys.stderr)
return 1
if __name__ == "__main__": sys.exit(main())

View file

@ -1,377 +0,0 @@
# Reticulum License
#
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# - The Software shall not be used in any kind of system which includes amongst
# its functions the ability to purposefully do harm to human beings.
#
# - The Software shall not be used, directly or indirectly, in the creation of
# an artificial intelligence, machine learning or language model training
# dataset, including but not limited to any use that contributes to the
# training or development of such a model or algorithm.
#
# - The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import os
import time
import shutil
import selectors
import subprocess
import tempfile
import RNS
CONVERSION_TIMEOUT = 8
# WebP encoding backends, in preference order. All supported
# encoder families handle the common still formats (PNG, JPEG,
# TIFF, BMP, GIF, WebP).
#
# Environment overrides:
# RNGIT_MEDIA_BACKEND force a specific backend
BACKENDS = ( ("magick", ["magick", "-", "webp:-"]),
("convert", ["convert", "-", "webp:-"]),
("gm", ["gm", "convert", "-", "webp:-"]),
("ffmpeg", ["ffmpeg", "-y", "-loglevel", "error", "-i", "-", "-f", "webp", "pipe:1"]),
("avconv", ["avconv", "-y", "-loglevel", "error", "-i", "-", "-f", "webp", "pipe:1"]) )
SUPPORTED_IMAGE_EXTS = [".webp", ".png", ".jpg", ".jpeg", ".gif", ".tiff", ".tif", ".bmp"]
CONV_SIZE_MAX = 32*1024*1024
SNIFF_BYTES = 64
RELAY_CHUNK = 256*1024
_ENV_BACKEND = os.environ.get("RNGIT_MEDIA_BACKEND")
_winner = None
_no_backend_logged = False
def available_backends():
out = []
for name, argv in BACKENDS: out.append((name, shutil.which(argv[0]) is not None))
return out
def _selected_backend():
global _winner
if _ENV_BACKEND:
for name, argv in BACKENDS:
if name == _ENV_BACKEND and shutil.which(argv[0]) is not None:
return (name, argv)
return None
order = list(BACKENDS)
if _winner is not None: order.sort(key=lambda backend: backend[0] != _winner)
for name, argv in order:
if shutil.which(argv[0]) is not None:
_winner = name
return (name, argv)
return None
def _terminate(proc):
try: proc.kill()
except Exception: pass
try: proc.wait()
except Exception: pass
def _stderr_tail(proc, limit=1024):
try:
data = proc.stderr.read(limit)
return data.decode("utf-8", "replace").strip()
except Exception: return ""
def _webp_info(data):
if len(data) < 30 or data[:4] != b"RIFF" or data[8:12] != b"WEBP": return None
fourcc = data[12:16]
if fourcc == b"VP8X":
width = int.from_bytes(data[24:27], "little") + 1
height = int.from_bytes(data[27:30], "little") + 1
elif fourcc == b"VP8 ":
width = int.from_bytes(data[26:28], "little") & 0x3FFF
height = int.from_bytes(data[28:30], "little") & 0x3FFF
elif fourcc == b"VP8L":
bits = int.from_bytes(data[21:25], "little")
width = (bits & 0x3FFF) + 1
height = ((bits >> 14) & 0x3FFF) + 1
else: return None
if width > 0 and height > 0: return (width, height)
return None
def _is_bmp_header(data):
if len(data) < 18 or data[:2] != b"BM": return False
try: dib_size = int.from_bytes(data[14:18], "little")
except Exception: return False
return dib_size in (12, 40, 52, 56, 64, 108, 124)
_IMAGE_SIGNATURES = ( ("png", lambda data: len(data) >= 8 and data[:8] == b"\x89PNG\r\n\x1a\n"),
("jpeg", lambda data: len(data) >= 3 and data[:3] == b"\xff\xd8\xff"),
("gif", lambda data: len(data) >= 6 and data[:6] in (b"GIF87a", b"GIF89a")),
("tiff", lambda data: len(data) >= 4 and data[:4] in (b"II*\x00", b"MM\x00*")),
("webp", lambda data: _webp_info(data) is not None),
("bmp", lambda data: _is_bmp_header(data)) )
def _classify_image(data):
for format_name, signature_matches in _IMAGE_SIGNATURES:
if signature_matches(data): return format_name
return None
def _valid_webp(path):
try:
with open(path, "rb") as fh: return _webp_info(fh.read(30)) is not None
except Exception: return False
def _unlink_output(path):
try: os.unlink(path)
except Exception: pass
def _configured_backend(quality=None, max_dimension=None, input_spec=None):
global _no_backend_logged
backend = _selected_backend()
if backend is None:
if not _no_backend_logged:
_no_backend_logged = True
RNS.log("No WebP encoding backend available for media conversion. You can Install ImageMagick or ffmpeg to enable image conversion.", RNS.LOG_WARNING)
return None
backend_name, encoder_argv = backend
_no_backend_logged = False
quality_arg = None
dimension_arg = None
if quality is not None:
try: quality_arg = max(1, min(100, int(quality)))
except (ValueError, TypeError): pass
if max_dimension is not None:
try: dimension_arg = int(max_dimension)
except (ValueError, TypeError): pass
if dimension_arg is not None and dimension_arg < 1: dimension_arg = None
encoder_argv = list(encoder_argv)
if backend_name in ("magick", "convert", "gm"):
if input_spec is not None:
try: input_index = encoder_argv.index("-")
except ValueError: input_index = None
if input_index is not None: encoder_argv[input_index] = input_spec
options = []
if quality_arg is not None: options += ["-quality", str(quality_arg)]
if dimension_arg is not None: options += ["-resize", f"{dimension_arg}x{dimension_arg}>"]
encoder_argv = encoder_argv[:-1] + options + encoder_argv[-1:]
elif backend_name in ("ffmpeg", "avconv"):
options = []
if quality_arg is not None: options += ["-quality", str(quality_arg)]
if dimension_arg is not None: options += ["-vf", f"scale='min(iw,{dimension_arg})':'min(ih,{dimension_arg})':force_original_aspect_ratio=decrease"]
try: format_index = encoder_argv.index("-f")
except ValueError: format_index = len(encoder_argv)
encoder_argv = encoder_argv[:format_index] + options + encoder_argv[format_index:]
return backend_name, encoder_argv
def _await(backend_name, encoder_proc, input_proc=None, timeout=None):
if timeout is None: timeout = CONVERSION_TIMEOUT
deadline = time.time() + timeout
try:
encoder_proc.wait(timeout=max(0.0, deadline - time.time()))
if input_proc is not None: input_proc.wait(timeout=max(0.0, deadline - time.time()))
except subprocess.TimeoutExpired:
_terminate(input_proc)
_terminate(encoder_proc)
RNS.log(f"Media conversion via {backend_name} timed out after {RNS.prettytime(timeout)}", RNS.LOG_WARNING)
return False
if encoder_proc.returncode != 0:
detail = ""
encoder_tail = _stderr_tail(encoder_proc)
input_tail = _stderr_tail(input_proc) if input_proc is not None else ""
if encoder_tail: detail += f" Encoder: {encoder_tail}"
if input_tail: detail += f" Input: {input_tail}"
RNS.log(f"Media conversion via {backend_name} failed.{detail}", RNS.LOG_WARNING)
return False
return True
def _await_readable(fd, timeout, selector):
try: return bool(selector.select(max(0.0, timeout)))
except Exception: return False
def _read_prefix(fd, amount, deadline):
data = bytearray()
selector = selectors.DefaultSelector()
try:
selector.register(fd, selectors.EVENT_READ)
while len(data) < amount:
if not _await_readable(fd, deadline - time.time(), selector): return None
chunk = os.read(fd, amount - len(data))
if not chunk: break
data += chunk
return bytes(data)
finally: selector.close()
def _pump(fd, encoder_stdin, prefix, deadline):
written = len(prefix)
if written > CONV_SIZE_MAX: return None
encoder_stdin.write(prefix)
selector = selectors.DefaultSelector()
try:
selector.register(fd, selectors.EVENT_READ)
while True:
if not _await_readable(fd, deadline - time.time(), selector):
raise subprocess.TimeoutExpired("input relay", max(0.0, deadline - time.time()))
chunk = os.read(fd, RELAY_CHUNK)
if not chunk: break
written += len(chunk)
if written > CONV_SIZE_MAX: return None
encoder_stdin.write(chunk)
return written
finally: selector.close()
def detect_image_format(data):
return _classify_image(data)
def convert_to_webp(input_argv, output_path, cwd=None, timeout=None, quality=None, max_dimension=None):
configured = _configured_backend(quality=quality, max_dimension=max_dimension)
if configured is None: return False
backend_name = configured[0]
if timeout is None: timeout = CONVERSION_TIMEOUT
deadline = time.time() + timeout
try:
with open(output_path, "wb") as output_fh:
input_proc = subprocess.Popen(input_argv, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
input_fd = input_proc.stdout.fileno()
prefix = _read_prefix(input_fd, SNIFF_BYTES, deadline)
if prefix is None:
RNS.log(f"Media conversion via {backend_name} timed out after {RNS.prettytime(timeout)}", RNS.LOG_WARNING)
_terminate(input_proc)
return False
detected = _classify_image(prefix)
if detected is None:
RNS.log("Media conversion failed: Input format is not supported", RNS.LOG_DEBUG)
_terminate(input_proc)
return False
configured = _configured_backend(quality=quality, max_dimension=max_dimension, input_spec=detected+":-")
if configured is None:
_terminate(input_proc)
return False
backend_name, encoder_argv = configured
encoder_proc = subprocess.Popen(encoder_argv, stdin=subprocess.PIPE, stdout=output_fh, stderr=subprocess.PIPE)
aborted = False
try:
if _pump(input_fd, encoder_proc.stdin, prefix, deadline) is None:
RNS.log(f"Media conversion failed: Streaming input exceeded maximum size of {RNS.prettysize(CONV_SIZE_MAX)}", RNS.LOG_DEBUG)
aborted = True
except subprocess.TimeoutExpired:
RNS.log(f"Media conversion via {backend_name} timed out after {RNS.prettytime(timeout)}", RNS.LOG_WARNING)
aborted = True
except (BrokenPipeError, OSError):
RNS.log(f"Media conversion via {backend_name} failed: Encoder exited before input was consumed", RNS.LOG_DEBUG)
aborted = True
if aborted:
_terminate(input_proc)
_terminate(encoder_proc)
return False
try: encoder_proc.stdin.close()
except (BrokenPipeError, OSError): pass
try: input_proc.stdout.close()
except Exception: pass
if not _await(backend_name, encoder_proc, input_proc=input_proc, timeout=max(0.0, deadline - time.time())): return False
if not _valid_webp(output_path):
RNS.log(f"Media conversion via {backend_name} produced invalid WebP output", RNS.LOG_WARNING)
return False
RNS.log(f"Media converted to WebP with {backend_name}", RNS.LOG_DEBUG)
return True
except Exception as e:
RNS.log(f"Error during media conversion: {e}", RNS.LOG_WARNING)
return False
def convert_file_to_webp(source_path, quality=85, max_dimension=None, timeout=None):
configured = _configured_backend(quality=quality, max_dimension=max_dimension)
if configured is None: return False
backend_name = configured[0]
tmp_path = None
success = False
try:
if not os.path.isfile(source_path):
RNS.log(f"Cannot convert media: Source file does not exist: {source_path}", RNS.LOG_WARNING)
return False
try: source_size = os.path.getsize(source_path)
except Exception: source_size = None
if source_size is not None and source_size > CONV_SIZE_MAX:
RNS.log(f"Cannot convert media: Source file exceeds maximum size of {RNS.prettysize(CONV_SIZE_MAX)}", RNS.LOG_DEBUG)
return False
fd, tmp_path = tempfile.mkstemp(prefix="rns_media_", suffix=".webp")
os.close(fd)
with open(source_path, "rb", buffering=0) as input_fh, open(tmp_path, "wb") as output_fh:
prefix = b""
while len(prefix) < SNIFF_BYTES:
chunk = input_fh.read(SNIFF_BYTES - len(prefix))
if not chunk: break
prefix += chunk
detected = _classify_image(prefix)
if detected is None:
RNS.log("Cannot convert media: Source file is not a supported media format", RNS.LOG_DEBUG)
return False
configured = _configured_backend(quality=quality, max_dimension=max_dimension, input_spec=detected+":-")
if configured is None: return False
backend_name, encoder_argv = configured
input_fh.seek(0)
encoder_proc = subprocess.Popen(encoder_argv, stdin=input_fh, stdout=output_fh, stderr=subprocess.PIPE)
if not _await(backend_name, encoder_proc, timeout=timeout): return False
if not _valid_webp(tmp_path):
RNS.log(f"Invalid media conversion output from {backend_name}", RNS.LOG_WARNING)
return False
success = True
RNS.log(f"Media converted to WebP with {backend_name}", RNS.LOG_DEBUG)
return tmp_path
except Exception as e:
RNS.log(f"Error during media conversion: {e}", RNS.LOG_WARNING)
return False
finally:
if not success and tmp_path is not None: _unlink_output(tmp_path)

View file

@ -32,27 +32,20 @@ import os
import time import time
import threading import threading
import subprocess import subprocess
import tempfile
import urllib.parse import urllib.parse
import RNS import RNS
import struct
import base64
from collections import deque from collections import deque
from datetime import datetime from datetime import datetime
from RNS.Utilities.rngit import APP_NAME, media from RNS.Utilities.rngit import APP_NAME
from RNS.Utilities.rngit.util import MarkdownToMicron, san_sha from RNS.Utilities.rngit.util import MarkdownToMicron, san_sha
from RNS.Utilities.rngit.highlight import SyntaxHighlighter from RNS.Utilities.rngit.highlight import SyntaxHighlighter
from RNS.vendor.configobj import ConfigObj from RNS.vendor.configobj import ConfigObj
from RNS.vendor import umsgpack as mp from RNS.vendor import umsgpack as mp
from RNS._version import __version__ from RNS._version import __version__
from RNS.Utilities.rnid import validate_rsg, extract_signed_rsg_data
from RNS.Utilities.rngit.media import SUPPORTED_IMAGE_EXTS
from RNS.Utilities.rngit.commitsigs import unarmor_ssh_signature, parse_ssh_signature, extract_commit_author
class NomadNetworkNode(): class NomadNetworkNode():
APP_NAME = "nomadnetwork" APP_NAME = "nomadnetwork"
JOBS_INTERVAL = 5 JOBS_INTERVAL = 5
LINK_CLEAN_INTERVAL = 60
PATH_INDEX = "/page/index.mu" PATH_INDEX = "/page/index.mu"
PATH_GROUP = "/page/group.mu" PATH_GROUP = "/page/group.mu"
@ -67,12 +60,10 @@ class NomadNetworkNode():
PATH_RELEASE = "/page/release.mu" PATH_RELEASE = "/page/release.mu"
PATH_WORK = "/page/work.mu" PATH_WORK = "/page/work.mu"
PATH_WORK_DOC = "/page/work_doc.mu" PATH_WORK_DOC = "/page/work_doc.mu"
PATH_MEDIA = "/media"
FILE_ARTIFACT = "/file/artifact" FILE_ARTIFACT = "/file/artifact"
FILE_DOWNLOAD = "/file/download" FILE_DOWNLOAD = "/file/download"
FILE_WORKDOC = "/file/workdoc" FILE_WORKDOC = "/file/workdoc"
DEFAULT_LINK_COST = 18.0
BLOB_SIZE_LIMIT = 256 * 1024 BLOB_SIZE_LIMIT = 256 * 1024
TREE_ENTRIES_PER_PAGE = 1000 TREE_ENTRIES_PER_PAGE = 1000
COMMITS_PER_PAGE = 100 COMMITS_PER_PAGE = 100
@ -125,10 +116,8 @@ class NomadNetworkNode():
# want to use tabs, three spaces is all you get. # want to use tabs, three spaces is all you get.
TAB_WIDTH = " " TAB_WIDTH = " "
CONVERTABLE_EXTS = [".md"]
RENDERABLE_EXTS = [".md", ".mu"] RENDERABLE_EXTS = [".md", ".mu"]
RENDER_DEFAULT = [".md", ".mu"] RENDER_DEFAULT = [".md", ".mu"]
IMAGE_EXTS = SUPPORTED_IMAGE_EXTS
def __init__(self, owner=None): def __init__(self, owner=None):
if not owner: raise TypeError(f"Invalid owner {owner} for {self}") if not owner: raise TypeError(f"Invalid owner {owner} for {self}")
@ -140,8 +129,6 @@ class NomadNetworkNode():
self.node_name = owner.node_name self.node_name = owner.node_name
self.announce_interval = owner.announce_interval self.announce_interval = owner.announce_interval
self.last_announce = 0 self.last_announce = 0
self.stamp_value = None
self.link_cost = self.DEFAULT_LINK_COST
self.null_ident = RNS.Identity.from_bytes(bytes(64)) self.null_ident = RNS.Identity.from_bytes(bytes(64))
self.templates = {} self.templates = {}
@ -159,16 +146,12 @@ class NomadNetworkNode():
self.templates["stats"] = DEFAULT_STATS_TEMPLATE self.templates["stats"] = DEFAULT_STATS_TEMPLATE
self.templates["work"] = DEFAULT_WORK_TEMPLATE self.templates["work"] = DEFAULT_WORK_TEMPLATE
self.templates["work_doc"] = DEFAULT_WORK_DOC_TEMPLATE self.templates["work_doc"] = DEFAULT_WORK_DOC_TEMPLATE
self.templates["no_ident"] = DEFAULT_NO_IDENT_TEMPLATE
self.templatesdir = self.owner.configdir+"/templates" self.templatesdir = self.owner.configdir+"/templates"
self.use_nerdfonts = self.USE_NERDFONTS self.use_nerdfonts = self.USE_NERDFONTS
self.media_conversion = True
self.highlight_syntax = True self.highlight_syntax = True
self.highlighter = SyntaxHighlighter() self.highlighter = SyntaxHighlighter()
self.mdc = MarkdownToMicron(max_width=self.MAX_RENDER_WIDTH, syntax_highlighter=self.highlighter) self.mdc = MarkdownToMicron(max_width=self.MAX_RENDER_WIDTH, syntax_highlighter=self.highlighter)
self.thanks_deque = deque(maxlen=256) self.thanks_deque = deque(maxlen=256)
self.active_links = {}
self.last_link_clean = 0
if not os.path.isdir(self.templatesdir): if not os.path.isdir(self.templatesdir):
try: os.makedirs(self.templatesdir) try: os.makedirs(self.templatesdir)
@ -177,16 +160,8 @@ class NomadNetworkNode():
if "pages" in self.owner.config: if "pages" in self.owner.config:
if "unicode_icons" in self.owner.config["pages"]: if "unicode_icons" in self.owner.config["pages"]:
if self.owner.config["pages"].as_bool("unicode_icons"): self.use_nerdfonts = False if self.owner.config["pages"].as_bool("unicode_icons"): self.use_nerdfonts = False
if "media_conversion" in self.owner.config["pages"]:
self.media_conversion = self.owner.config["pages"].as_bool("media_conversion")
if "link_cost" in self.owner.config["pages"]:
self.link_cost = RNS.Stamps.quantize_cost(self.owner.config["pages"].as_float("link_cost"))
if "stamp_value" in self.owner.config["pages"]:
self.stamp_value = RNS.Stamps.quantize_cost(self.owner.config["pages"].as_float("stamp_value"))
self.destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, self.APP_NAME, "node") self.destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, self.APP_NAME, "node")
if self.stamp_value: self.destination.set_value(self.stamp_value)
self.destination.set_link_cost(self.link_cost)
self.destination.set_link_established_callback(self.remote_connected) self.destination.set_link_established_callback(self.remote_connected)
self.destination.set_default_app_data(self.get_announce_app_data) self.destination.set_default_app_data(self.get_announce_app_data)
self.register_request_handlers() self.register_request_handlers()
@ -231,10 +206,6 @@ class NomadNetworkNode():
try: try:
if self.announce_interval and time.time() > self.last_announce + self.announce_interval: self.announce() if self.announce_interval and time.time() > self.last_announce + self.announce_interval: self.announce()
if time.time() > self.last_link_clean + self.LINK_CLEAN_INTERVAL:
self.clean_links()
self.last_link_clean = time.time()
except Exception as e: RNS.log(f"Error while running periodic jobs: {e}", RNS.LOG_ERROR) except Exception as e: RNS.log(f"Error while running periodic jobs: {e}", RNS.LOG_ERROR)
def get_announce_app_data(self): return self.node_name.encode("utf-8") def get_announce_app_data(self): return self.node_name.encode("utf-8")
@ -271,7 +242,6 @@ class NomadNetworkNode():
self.destination.register_request_handler(self.PATH_RELEASE, response_generator=self.serve_release_page, allow=RNS.Destination.ALLOW_ALL) self.destination.register_request_handler(self.PATH_RELEASE, response_generator=self.serve_release_page, allow=RNS.Destination.ALLOW_ALL)
self.destination.register_request_handler(self.PATH_WORK, response_generator=self.serve_work_page, allow=RNS.Destination.ALLOW_ALL) self.destination.register_request_handler(self.PATH_WORK, response_generator=self.serve_work_page, allow=RNS.Destination.ALLOW_ALL)
self.destination.register_request_handler(self.PATH_WORK_DOC, response_generator=self.serve_work_doc_page, allow=RNS.Destination.ALLOW_ALL) self.destination.register_request_handler(self.PATH_WORK_DOC, response_generator=self.serve_work_doc_page, allow=RNS.Destination.ALLOW_ALL)
self.destination.register_request_handler(self.PATH_MEDIA, response_generator=self.serve_media, allow=RNS.Destination.ALLOW_ALL, auto_compress=False)
self.destination.register_request_handler(self.FILE_ARTIFACT, response_generator=self.serve_artifact, allow=RNS.Destination.ALLOW_ALL) self.destination.register_request_handler(self.FILE_ARTIFACT, response_generator=self.serve_artifact, allow=RNS.Destination.ALLOW_ALL)
self.destination.register_request_handler(self.FILE_DOWNLOAD, response_generator=self.serve_download, allow=RNS.Destination.ALLOW_ALL) self.destination.register_request_handler(self.FILE_DOWNLOAD, response_generator=self.serve_download, allow=RNS.Destination.ALLOW_ALL)
self.destination.register_request_handler(self.FILE_WORKDOC, response_generator=self.serve_wd_download, allow=RNS.Destination.ALLOW_ALL) self.destination.register_request_handler(self.FILE_WORKDOC, response_generator=self.serve_wd_download, allow=RNS.Destination.ALLOW_ALL)
@ -398,10 +368,6 @@ class NomadNetworkNode():
self.owner.view_succeeded(None, None, remote_identity) self.owner.view_succeeded(None, None, remote_identity)
page_content = "".join(content_parts) page_content = "".join(content_parts)
nav_content = "".join(nav_parts) nav_content = "".join(nav_parts)
if not remote_identity and self.null_ident.hash in self.owner.blocked_identities:
return self.render_template("", template="no_ident", nav_content=nav_content, st=st)
return self.render_template(page_content, nav_content=nav_content, template="front", st=st) return self.render_template(page_content, nav_content=nav_content, template="front", st=st)
def serve_group_page(self, path, data, request_id, link_id, remote_identity, requested_at): def serve_group_page(self, path, data, request_id, link_id, remote_identity, requested_at):
@ -422,9 +388,6 @@ class NomadNetworkNode():
nav_parts.append(breadcrumb + "\n") nav_parts.append(breadcrumb + "\n")
nav_content = "".join(nav_parts) nav_content = "".join(nav_parts)
if not remote_identity and self.null_ident.hash in self.owner.blocked_identities:
return self.render_template("", template="no_ident", nav_content=nav_content, st=st)
accessible_repos = self.get_accessible_repositories(remote_identity, group_name) accessible_repos = self.get_accessible_repositories(remote_identity, group_name)
if not group_name or group_name not in self.owner.groups or not accessible_repos: if not group_name or group_name not in self.owner.groups or not accessible_repos:
@ -474,9 +437,6 @@ class NomadNetworkNode():
repo = self.get_accessible_repository(remote_identity, group_name, repo_name) repo = self.get_accessible_repository(remote_identity, group_name, repo_name)
if not remote_identity and self.null_ident.hash in self.owner.blocked_identities:
return self.render_template("", template="no_ident", nav_content="".join(nav_parts), st=st)
if not repo: if not repo:
content = self.m_heading("Not Found", 1) + "\nThe requested repository was not found.\n" content = self.m_heading("Not Found", 1) + "\nThe requested repository was not found.\n"
return self.render_template(content, nav_content="".join(nav_parts), st=st) return self.render_template(content, nav_content="".join(nav_parts), st=st)
@ -590,9 +550,6 @@ class NomadNetworkNode():
except (ValueError, TypeError): page_num = 0 except (ValueError, TypeError): page_num = 0
if not remote_identity and self.null_ident.hash in self.owner.blocked_identities:
return self.render_template("", template="no_ident", st=st)
repo = self.get_accessible_repository(remote_identity, group_name, repo_name) repo = self.get_accessible_repository(remote_identity, group_name, repo_name)
if not repo: if not repo:
content = self.m_heading("Not Found", 1) + "\n\nThe requested repository does not exist or you do not have access to it.\n" content = self.m_heading("Not Found", 1) + "\n\nThe requested repository does not exist or you do not have access to it.\n"
@ -730,9 +687,6 @@ class NomadNetworkNode():
render = True if render else False render = True if render else False
raw = True if raw else False raw = True if raw else False
if not remote_identity and self.null_ident.hash in self.owner.blocked_identities:
return self.render_template("", template="no_ident", st=st)
repo = self.get_accessible_repository(remote_identity, group_name, repo_name) repo = self.get_accessible_repository(remote_identity, group_name, repo_name)
if not repo: if not repo:
content = self.m_heading("Not Found", 1) + "\n\nThe requested repository does not exist or you do not have access to it.\n" content = self.m_heading("Not Found", 1) + "\n\nThe requested repository does not exist or you do not have access to it.\n"
@ -751,10 +705,9 @@ class NomadNetworkNode():
content = self.m_heading("Invalid Path", 1) + "\n\nNo file path specified.\n" content = self.m_heading("Invalid Path", 1) + "\n\nNo file path specified.\n"
return self.render_template(content, st=st) return self.render_template(content, st=st)
file_path = file_path.removeprefix("./").replace("/./", "/") file_path = file_path.lstrip("./").replace("/./", "/")
file_ext = os.path.splitext(file_path)[1].lower() file_ext = os.path.splitext(file_path)[1].lower()
renderable = file_ext in self.RENDERABLE_EXTS renderable = file_ext in self.RENDERABLE_EXTS
convertable = file_ext in self.CONVERTABLE_EXTS
if not renderable: raw = True; render = False if not renderable: raw = True; render = False
else: else:
if raw: render = False if raw: render = False
@ -786,11 +739,9 @@ class NomadNetworkNode():
else: else:
rnd_link = self.m_link("View rendered", self.PATH_BLOB, g=group_name, r=repo_name, ref=ref, path=file_path, render="y") rnd_link = self.m_link("View rendered", self.PATH_BLOB, g=group_name, r=repo_name, ref=ref, path=file_path, render="y")
raw_link = self.m_link("View raw", self.PATH_BLOB, g=group_name, r=repo_name, ref=ref, path=file_path, raw="y") raw_link = self.m_link("View raw", self.PATH_BLOB, g=group_name, r=repo_name, ref=ref, path=file_path, raw="y")
mu_link = self.m_link("as micron", self.FILE_DOWNLOAD, g=group_name, r=repo_name, ref=ref, path=file_path, fmt="mu")
if render: render_controls = f"Displaying Rendered {sep} {raw_link}" if render: render_controls = f"Displaying Rendered {sep} {raw_link}"
else: render_controls = f"Displaying Raw {sep} {rnd_link}" else: render_controls = f"Displaying Raw {sep} {rnd_link}"
if convertable: nav_parts.append(f"\n{render_controls} {sep} {dl_link} {self.CLR_DIM}{mu_link}`f\n") nav_parts.append(f"\n{render_controls} {sep} {dl_link}\n")
else: nav_parts.append(f"\n{render_controls} {sep} {dl_link}\n")
# Get blob info # Get blob info
blob_info = self.get_blob_info(repo_path, resolved_ref, file_path) blob_info = self.get_blob_info(repo_path, resolved_ref, file_path)
@ -816,9 +767,8 @@ class NomadNetworkNode():
content_parts.append(f"`*{self.m_escape(symlink_target or 'unknown')}`*\n") content_parts.append(f"`*{self.m_escape(symlink_target or 'unknown')}`*\n")
elif is_binary: elif is_binary:
if file_ext in self.IMAGE_EXTS: content_parts.append("This file appears to be binary and cannot be displayed as text.\n")
content_parts.append(f"`(Image file`w=n`a=c`:/media/{group_name}/{repo_name}/{ref}/{urllib.parse.quote_plus(file_path)})\n") # TODO: Implement raw file downloads
else: content_parts.append("This file appears to be binary and cannot be displayed as text.\n")
elif size > self.BLOB_SIZE_LIMIT: elif size > self.BLOB_SIZE_LIMIT:
content_parts.append(f"This file is {RNS.prettysize(size)}, which exceeds the display limit of {RNS.prettysize(self.BLOB_SIZE_LIMIT)}.\n") content_parts.append(f"This file is {RNS.prettysize(size)}, which exceeds the display limit of {RNS.prettysize(self.BLOB_SIZE_LIMIT)}.\n")
@ -869,9 +819,6 @@ class NomadNetworkNode():
except (ValueError, TypeError): page_num = 0 except (ValueError, TypeError): page_num = 0
if not remote_identity and self.null_ident.hash in self.owner.blocked_identities:
return self.render_template("", template="no_ident", st=st)
repo = self.get_accessible_repository(remote_identity, group_name, repo_name) repo = self.get_accessible_repository(remote_identity, group_name, repo_name)
if not repo: if not repo:
content = self.m_heading("Not Found", 1) + "\n\nThe requested repository does not exist or you do not have access to it.\n" content = self.m_heading("Not Found", 1) + "\n\nThe requested repository does not exist or you do not have access to it.\n"
@ -917,7 +864,7 @@ class NomadNetworkNode():
author = commit["author"] author = commit["author"]
date = self.format_absolute_time(commit["timestamp"])+" - "+self.format_relative_time(commit["timestamp"]) date = self.format_absolute_time(commit["timestamp"])+" - "+self.format_relative_time(commit["timestamp"])
hash_link = self.m_link(short_hash, self.PATH_COMMIT, g=group_name, r=repo_name, ref=ref, h=commit["hash"]) hash_link = self.m_link(short_hash, self.PATH_COMMIT, g=group_name, r=repo_name, h=commit["hash"])
content_parts.append(f"`F66d{hash_link}`f {self.m_escape(author)} {self.CLR_DIM}{date}`f\n") content_parts.append(f"`F66d{hash_link}`f {self.m_escape(author)} {self.CLR_DIM}{date}`f\n")
content_parts.append(f"{self.m_escape(subject)}\n\n") content_parts.append(f"{self.m_escape(subject)}\n\n")
@ -944,16 +891,12 @@ class NomadNetworkNode():
if not data or not type(data) == dict: data = {} if not data or not type(data) == dict: data = {}
group_name = data.get("var_g", "") if data else "" group_name = data.get("var_g", "") if data else ""
repo_name = data.get("var_r", "") if data else "" repo_name = data.get("var_r", "") if data else ""
ref = data.get("var_ref", "HEAD") if data else "HEAD"
commit_hash = data.get("var_h", "") if data else "" commit_hash = data.get("var_h", "") if data else ""
if not group_name or not repo_name: if not group_name or not repo_name:
content = self.m_heading("Error", 2) + "\nInvalid request\n" content = self.m_heading("Error", 2) + "\nInvalid request\n"
return self.render_template(content, st=st) return self.render_template(content, st=st)
if not remote_identity and self.null_ident.hash in self.owner.blocked_identities:
return self.render_template("", template="no_ident", st=st)
repo = self.get_accessible_repository(remote_identity, group_name, repo_name) repo = self.get_accessible_repository(remote_identity, group_name, repo_name)
if not repo: if not repo:
content = self.m_heading("Error", 2) + "\nThe requested repository was not found.\n" content = self.m_heading("Error", 2) + "\nThe requested repository was not found.\n"
@ -961,12 +904,6 @@ class NomadNetworkNode():
repo_path = repo["path"] repo_path = repo["path"]
# Validate ref exists
resolved_ref = self.resolve_ref(repo_path, ref)
if not resolved_ref:
content = self.m_heading("Ref Not Found", 1) + f"\n\nThe ref '{ref}' does not exist in this repository.\n"
return self.render_template(content, st=st)
# Validate commit hash # Validate commit hash
if not commit_hash or len(commit_hash) < 7: if not commit_hash or len(commit_hash) < 7:
content = self.m_heading("Error", 2) + "\nNo valid commit hash specified.\n" content = self.m_heading("Error", 2) + "\nNo valid commit hash specified.\n"
@ -980,7 +917,7 @@ class NomadNetworkNode():
# Breadcrumb navigation # Breadcrumb navigation
nav_parts = [] nav_parts = []
breadcrumb = f"{self.m_link('Node', self.PATH_INDEX)} / {self.m_link(group_name, self.PATH_GROUP, g=group_name)} / {self.m_link(repo_name, self.PATH_REPO, g=group_name, r=repo_name)} / {self.m_link('commits', self.PATH_COMMITS, g=group_name, r=repo_name, ref=ref)} / {resolved_hash[:7]}" breadcrumb = f"{self.m_link('Node', self.PATH_INDEX)} / {self.m_link(group_name, self.PATH_GROUP, g=group_name)} / {self.m_link(repo_name, self.PATH_REPO, g=group_name, r=repo_name)} / {resolved_hash[:7]}"
nav_parts.append(">>\n" + breadcrumb + "\n") nav_parts.append(">>\n" + breadcrumb + "\n")
nav_content = "".join(nav_parts) nav_content = "".join(nav_parts)
@ -1014,37 +951,27 @@ class NomadNetworkNode():
i_folder = self.icon("folder") i_folder = self.icon("folder")
content_parts.append(f"{self.m_link(f'{i_folder} Browse tree at this commit', self.PATH_TREE, g=group_name, r=repo_name, ref=resolved_hash)}\n\n") content_parts.append(f"{self.m_link(f'{i_folder} Browse tree at this commit', self.PATH_TREE, g=group_name, r=repo_name, ref=resolved_hash)}\n\n")
# Validate and display commit signature status
show_sig = False
sig_status = self.get_commit_signature(repo_path, resolved_hash)
if sig_status["signed"]:
if sig_status["valid"] and sig_status["author_match"]: sig_text = f"`FT66BB85Valid, signed by author`f"; show_sig = True
elif sig_status["valid"]: sig_text = f"`Faa0{self.m_escape(sig_status['message'])}`f"; show_sig = True
else: sig_text = f"`F900{self.m_escape(sig_status['message'])}`f"; show_sig = True
else: sig_text = "Not signed"
# Commit metadata # Commit metadata
if commit_info.get("parents"): if commit_info.get("parents"):
parent_links = [] parent_links = []
for parent_hash in commit_info["parents"]: for parent_hash in commit_info["parents"]:
parent_link = self.m_link(parent_hash[:7], self.PATH_COMMIT, g=group_name, r=repo_name, ref=ref, h=parent_hash) parent_link = self.m_link(parent_hash[:7], self.PATH_COMMIT, g=group_name, r=repo_name, h=parent_hash)
parent_links.append(parent_link) parent_links.append(parent_link)
content_parts.append(f"Parents: {' '.join(parent_links)}\n") content_parts.append(f"Parents: {' '.join(parent_links)}\n")
content_parts.append(f"Author: {self.m_escape(commit_info['author_name'])} <{self.m_escape(commit_info['author_email'])}>\n") content_parts.append(f"Author: {self.m_escape(commit_info['author_name'])} <{self.m_escape(commit_info['author_email'])}>\n")
content_parts.append(f"Signature : {sig_text}\n") if show_sig else None
content_parts.append(f"Date: {commit_info['author_date']}\n") content_parts.append(f"Date: {commit_info['author_date']}\n")
if commit_info.get("committer_name") != commit_info.get("author_name"): if commit_info.get("committer_name") != commit_info.get("author_name"):
content_parts.append(f"Committer : {self.m_escape(commit_info['committer_name'])} <{self.m_escape(commit_info['committer_email'])}>\n") content_parts.append(f"Commit: {self.m_escape(commit_info['committer_name'])} <{self.m_escape(commit_info['committer_email'])}>\n")
content_parts.append(f"Date: {commit_info['committer_date']}\n") content_parts.append(f"Date: {commit_info['committer_date']}\n")
content_parts.append("\n") content_parts.append("\n")
# Commit message # Commit message
if commit_info.get("message"): if commit_info.get("message"):
content_parts.append(self.format_commit(commit_info["message"]) + "\n") content_parts.append(self.m_escape(commit_info["message"]) + "\n")
content_parts.append("\n") content_parts.append("\n")
# Changed files # Changed files
@ -1109,9 +1036,6 @@ class NomadNetworkNode():
nav_parts.append(">>\n" + breadcrumb + "\n") nav_parts.append(">>\n" + breadcrumb + "\n")
nav_content = "".join(nav_parts) nav_content = "".join(nav_parts)
if not remote_identity and self.null_ident.hash in self.owner.blocked_identities:
return self.render_template("", template="no_ident", st=st)
if not group_name or not repo_name: if not group_name or not repo_name:
content = self.m_heading("Error", 2) + "\nInvalid request\n" content = self.m_heading("Error", 2) + "\nInvalid request\n"
return self.render_template(content, st=st) return self.render_template(content, st=st)
@ -1211,15 +1135,12 @@ class NomadNetworkNode():
content = self.m_heading("Error", 2) + "\nInvalid request\n" content = self.m_heading("Error", 2) + "\nInvalid request\n"
return self.render_template(content, st=st) return self.render_template(content, st=st)
if not remote_identity and self.null_ident.hash in self.owner.blocked_identities:
return self.render_template("", template="no_ident", st=st)
content_parts = [] content_parts = []
nav_parts = [] nav_parts = []
# Breadcrumb navigation # Breadcrumb navigation
repo_link = self.m_link(repo_name, self.PATH_REPO, g=group_name, r=repo_name) repo_link = self.m_link(repo_name, self.PATH_REPO, g=group_name, r=repo_name)
breadcrumb = f">>\n{self.m_link('Node', self.PATH_INDEX)} / {self.m_link(group_name, self.PATH_GROUP, g=group_name)} / {repo_link} / stats" breadcrumb = f">>\n{self.m_link('Node', self.PATH_INDEX)} / {self.m_link(group_name, self.PATH_GROUP, g=group_name)} / {repo_link}"
nav_parts.append(breadcrumb + "\n") nav_parts.append(breadcrumb + "\n")
repo = self.get_accessible_repository(remote_identity, group_name, repo_name) repo = self.get_accessible_repository(remote_identity, group_name, repo_name)
@ -1314,9 +1235,6 @@ class NomadNetworkNode():
content = self.m_heading("Error", 2) + "\nInvalid request\n" content = self.m_heading("Error", 2) + "\nInvalid request\n"
return self.render_template(content, st=st) return self.render_template(content, st=st)
if not remote_identity and self.null_ident.hash in self.owner.blocked_identities:
return self.render_template("", template="no_ident", st=st)
repo = self.get_accessible_repository(remote_identity, group_name, repo_name) repo = self.get_accessible_repository(remote_identity, group_name, repo_name)
if not repo: if not repo:
content = self.m_heading("Error", 2) + "\nThe requested repository was not found.\n" content = self.m_heading("Error", 2) + "\nThe requested repository was not found.\n"
@ -1381,22 +1299,11 @@ class NomadNetworkNode():
content = self.m_heading("Error", 2) + "\nInvalid request\n" content = self.m_heading("Error", 2) + "\nInvalid request\n"
return self.render_template(content, st=st) return self.render_template(content, st=st)
if not remote_identity and self.null_ident.hash in self.owner.blocked_identities:
return self.render_template("", template="no_ident", st=st)
repo = self.get_accessible_repository(remote_identity, group_name, repo_name) repo = self.get_accessible_repository(remote_identity, group_name, repo_name)
if not repo: if not repo:
content = self.m_heading("Error", 2) + "\nThe requested repository was not found.\n" content = self.m_heading("Error", 2) + "\nThe requested repository was not found.\n"
return self.render_template(content, st=st) return self.render_template(content, st=st)
content_parts = []
nav_parts = []
# Breadcrumb navigation
breadcrumb = f">>\n{self.m_link('Node', self.PATH_INDEX)} / {self.m_link(group_name, self.PATH_GROUP, g=group_name)} / {self.m_link(repo_name, self.PATH_REPO, g=group_name, r=repo_name)} / {self.m_link('releases', self.PATH_RELEASES, g=group_name, r=repo_name)} / {tag}"
nav_parts.append(breadcrumb + "\n")
nav_content = "".join(nav_parts)
releases_path = f"{repo['path']}.releases" releases_path = f"{repo['path']}.releases"
if tag == "latest": if tag == "latest":
releases, latest_release = self.owner.releases_list_data(releases_path) releases, latest_release = self.owner.releases_list_data(releases_path)
@ -1410,6 +1317,14 @@ class NomadNetworkNode():
else: tag = latest_release else: tag = latest_release
content_parts = []
nav_parts = []
# Breadcrumb navigation
breadcrumb = f">>\n{self.m_link('Node', self.PATH_INDEX)} / {self.m_link(group_name, self.PATH_GROUP, g=group_name)} / {self.m_link(repo_name, self.PATH_REPO, g=group_name, r=repo_name)} / {self.m_link('releases', self.PATH_RELEASES, g=group_name, r=repo_name)} / {tag}"
nav_parts.append(breadcrumb + "\n")
nav_content = "".join(nav_parts)
release_dir = os.path.join(releases_path, tag) release_dir = os.path.join(releases_path, tag)
if not os.path.isdir(release_dir): if not os.path.isdir(release_dir):
@ -1486,17 +1401,11 @@ class NomadNetworkNode():
content = self.m_heading("Error", 2) + "\nInvalid request\n" content = self.m_heading("Error", 2) + "\nInvalid request\n"
return self.render_template(content, st=st) return self.render_template(content, st=st)
if not remote_identity and self.null_ident.hash in self.owner.blocked_identities:
return self.render_template("", template="no_ident", st=st)
repo = self.get_accessible_repository(remote_identity, group_name, repo_name) repo = self.get_accessible_repository(remote_identity, group_name, repo_name)
if not repo: if not repo:
content = self.m_heading("Error", 2) + "\nThe requested repository was not found.\n" content = self.m_heading("Error", 2) + "\nThe requested repository was not found.\n"
return self.render_template(content, st=st) return self.render_template(content, st=st)
work_path = f"{repo['path']}.work"
scopes_to_show = ["active", "completed", "proposed"] if scope == "all" else [scope]
content_parts = [] content_parts = []
nav_parts = [] nav_parts = []
@ -1511,46 +1420,17 @@ class NomadNetworkNode():
cmplt_s = "`_" if scope == "completed" else "" cmplt_s = "`_" if scope == "completed" else ""
prpsd_s = "`_" if scope == "proposed" else "" prpsd_s = "`_" if scope == "proposed" else ""
all_s = "`_" if scope == "all" else "" all_s = "`_" if scope == "all" else ""
# Count work documents
scope_counts = {"active": 0, "completed": 0, "proposed": 0}
for s in ["active", "completed", "proposed"]:
try:
folder_path = os.path.join(work_path, s)
if os.path.isdir(folder_path):
for entry in os.listdir(folder_path):
try:
doc_dir = os.path.join(folder_path, entry)
if not os.path.isdir(doc_dir): continue
doc_id = int(entry)
read_access = self.resolve_doc_permission(remote_identity, group_name, repo_name, doc_id, self.owner.PERM_READ)
if not read_access: continue
root_path = os.path.join(doc_dir, "root")
if not os.path.isfile(root_path): continue
scope_counts[s] += 1
except Exception as e: RNS.trace_exception(e)
except Exception as e: RNS.trace_exception(e)
force_counts = True
adc = scope_counts['active']
cdc = scope_counts['completed']
pdc = scope_counts['proposed']
tdc = adc+cdc+pdc
adc_str = f" ({adc})" if force_counts or (not scope == "active" and adc) else ""
cdc_str = f" ({cdc})" if force_counts or (not scope == "completed" and cdc) else ""
pdc_str = f" ({pdc})" if force_counts or (not scope == "proposed" and pdc) else ""
tdc_str = f" ({tdc})" if force_counts or (not scope == "all" and tdc) else ""
filter_links = [] filter_links = []
filter_links.append(active_s+self.m_link("Active", self.PATH_WORK, g=group_name, r=repo_name, scope="active")+active_s+adc_str) filter_links.append(active_s+self.m_link("Active", self.PATH_WORK, g=group_name, r=repo_name, scope="active")+active_s)
filter_links.append(cmplt_s+self.m_link("Completed", self.PATH_WORK, g=group_name, r=repo_name, scope="completed")+cmplt_s+cdc_str) filter_links.append(cmplt_s+self.m_link("Completed", self.PATH_WORK, g=group_name, r=repo_name, scope="completed")+cmplt_s)
filter_links.append(prpsd_s+self.m_link("Proposed", self.PATH_WORK, g=group_name, r=repo_name, scope="proposed")+prpsd_s+pdc_str) filter_links.append(prpsd_s+self.m_link("Proposed", self.PATH_WORK, g=group_name, r=repo_name, scope="proposed")+prpsd_s)
filter_links.append(all_s+self.m_link("All", self.PATH_WORK, g=group_name, r=repo_name, scope="all")+all_s+tdc_str) filter_links.append(all_s+self.m_link("All", self.PATH_WORK, g=group_name, r=repo_name, scope="all")+all_s)
content_parts.append(f" {sep} ".join(filter_links) + "\n\n") content_parts.append(f" {sep} ".join(filter_links) + "\n\n")
# Load work documents # Load work documents
work_path = f"{repo['path']}.work"
scopes_to_show = ["active", "completed", "proposed"] if scope == "all" else [scope]
for s in scopes_to_show: for s in scopes_to_show:
folder_path = os.path.join(work_path, s) folder_path = os.path.join(work_path, s)
@ -1582,11 +1462,11 @@ class NomadNetworkNode():
docs.sort(key=lambda x: max(x["created"], x["edited"]), reverse=True) docs.sort(key=lambda x: max(x["created"], x["edited"]), reverse=True)
if not docs: if not docs:
content_parts.append(self.m_heading(f"{s.capitalize()}", 2)+f"\n`*No {s} work documents`*\n") content_parts.append(self.m_heading(f"{s.capitalize()} ({len(docs)})", 2)+f"\n`*No {s} work documents`*\n")
content_parts.append("\n") content_parts.append("\n")
else: else:
content_parts.append(self.m_heading(f"{s.capitalize()}", 2)) content_parts.append(self.m_heading(f"{s.capitalize()} ({len(docs)})", 2))
content_parts.append("\n") content_parts.append("\n")
for doc in docs: for doc in docs:
@ -1621,9 +1501,6 @@ class NomadNetworkNode():
content = self.m_heading("Error", 2) + "\nInvalid request\n" content = self.m_heading("Error", 2) + "\nInvalid request\n"
return self.render_template(content, st=st) return self.render_template(content, st=st)
if not remote_identity and self.null_ident.hash in self.owner.blocked_identities:
return self.render_template("", template="no_ident", st=st)
try: doc_id = int(doc_id) try: doc_id = int(doc_id)
except: except:
content = self.m_heading("Error", 2) + "\nInvalid document ID\n" content = self.m_heading("Error", 2) + "\nInvalid document ID\n"
@ -1773,7 +1650,7 @@ class NomadNetworkNode():
repo = self.get_accessible_repository(remote_identity, group_name, repo_name) repo = self.get_accessible_repository(remote_identity, group_name, repo_name)
if not repo: if not repo:
RNS.log(f"Repository not found or no access for artifact request {group_name}/{repo_name}/{tag}/{artifact}", RNS.LOG_DEBUG) RNS.log(f"Repository not found or no access for artifact request {group_name}/{repo_name}/{tag}/{artifact}", RNS.LOG_WARNING)
return None return None
releases_path = f"{repo['path']}.releases" releases_path = f"{repo['path']}.releases"
@ -1817,69 +1694,6 @@ class NomadNetworkNode():
self.owner.release_download_succeeded(group_name, repo_name, remote_identity) self.owner.release_download_succeeded(group_name, repo_name, remote_identity)
return [open(artifact_path, "rb"), {"name": artifact.encode("utf-8")}] return [open(artifact_path, "rb"), {"name": artifact.encode("utf-8")}]
def serve_media(self, path, data, request_id, link_id, remote_identity, requested_at):
st = time.time()
RNS.log(f"Media request from {remote_identity}", RNS.LOG_DEBUG)
if not data or not type(data) == dict: data = {}
if not "key" in data:
RNS.log(f"Missing request key in media request", RNS.LOG_DEBUG)
return False
media_path = data.get("path", None)
if not media_path:
RNS.log(f"Missing path in media request", RNS.LOG_DEBUG)
return False
comps = media_path.removeprefix("/media").lstrip("/").split("/")
if len(comps) < 4:
RNS.log(f"Insufficient path components in media request", RNS.LOG_DEBUG)
return False
group_name = comps[0]
repo_name = comps[1]
ref = comps[2]
file_path = "/".join(comps[3:])
file_path = urllib.parse.unquote_plus(file_path)
file_name = os.path.basename(file_path)
repo = self.get_accessible_repository(remote_identity, group_name, repo_name)
if not repo:
RNS.log(f"Repository not found or no access for media request {group_name}/{repo_name}/{ref}/{file_path}", RNS.LOG_DEBUG)
return False
repo_path = repo["path"]
resolved_ref = self.resolve_ref(repo_path, ref)
if not resolved_ref:
RNS.log(f"Ref not found for media request {group_name}/{repo_name}/{ref}/{file_path}", RNS.LOG_WARNING)
return False
if not file_path:
RNS.log(f"No file path for media request {group_name}/{repo_name}/{ref}/{file_path}", RNS.LOG_WARNING)
return False
blob_info = self.get_blob_info(repo_path, resolved_ref, file_path)
if blob_info is None:
RNS.log(f"File not found at ref for media request {group_name}/{repo_name}/{ref}/{file_path}", RNS.LOG_WARNING)
return False
else:
stream = None
response_name = file_name
file_ext = os.path.splitext(file_path)[1].lower()
if self.media_conversion and file_ext in self.IMAGE_EXTS and file_ext != ".webp":
converted = self.get_webp_stream(repo_path, resolved_ref, file_path, link_id)
if converted: stream, response_name = converted
if not stream: stream = self.get_blob_stream(repo_path, resolved_ref, file_path)
if stream: return [stream, {"name": response_name.encode("utf-8")}]
else:
RNS.log(f"Could not resolve blob stream for media request {group_name}/{repo_name}/{ref}/{file_path}", RNS.LOG_WARNING)
return None
return None
def serve_download(self, path, data, request_id, link_id, remote_identity, requested_at): def serve_download(self, path, data, request_id, link_id, remote_identity, requested_at):
st = time.time() st = time.time()
RNS.log(f"File download request from {remote_identity}", RNS.LOG_DEBUG) RNS.log(f"File download request from {remote_identity}", RNS.LOG_DEBUG)
@ -1888,19 +1702,13 @@ class NomadNetworkNode():
group_name = data.get("var_g", "") if data else "" group_name = data.get("var_g", "") if data else ""
repo_name = data.get("var_r", "") if data else "" repo_name = data.get("var_r", "") if data else ""
ref = data.get("var_ref", "HEAD") if data else "HEAD" ref = data.get("var_ref", "HEAD") if data else "HEAD"
file_fmt = data.get("var_fmt", "") if data else ""
file_path = data.get("var_path", "") if data else "" file_path = data.get("var_path", "") if data else ""
file_path = urllib.parse.unquote_plus(file_path) file_path = urllib.parse.unquote_plus(file_path)
file_name = os.path.basename(file_path) file_name = os.path.basename(file_path)
file_ext = os.path.splitext(file_path)[1].lower()
convertable = file_ext in self.CONVERTABLE_EXTS
if file_fmt and not convertable:
RNS.log(f"Download conversion request for non-convertable file", RNS.LOG_DEBUG)
return None
repo = self.get_accessible_repository(remote_identity, group_name, repo_name) repo = self.get_accessible_repository(remote_identity, group_name, repo_name)
if not repo: if not repo:
RNS.log(f"Repository not found or no access for download request {group_name}/{repo_name}/{ref}/{file_path}", RNS.LOG_DEBUG) RNS.log(f"Repository not found or no access for download request {group_name}/{repo_name}/{ref}/{file_path}", RNS.LOG_WARNING)
return None return None
repo_path = repo["path"] repo_path = repo["path"]
@ -1923,47 +1731,7 @@ class NomadNetworkNode():
stream = self.get_blob_stream(repo_path, resolved_ref, file_path) stream = self.get_blob_stream(repo_path, resolved_ref, file_path)
if stream is not None: if stream is not None:
self.owner.download_succeeded(group_name, repo_name, remote_identity) self.owner.download_succeeded(group_name, repo_name, remote_identity)
if not file_fmt: return [stream, {"name": file_name.encode("utf-8")}] return [stream, {"name": file_name.encode("utf-8")}]
elif file_fmt == "mu":
link = self.active_links.get(link_id)
if not link:
RNS.log(f"Could not resolve link for file conversion of {file_path}", RNS.LOG_WARNING)
return None
if not hasattr(link, "temporary_directories"): link.temporary_directories = []
tmpdir = tempfile.TemporaryDirectory()
link.temporary_directories.append(tmpdir)
stem = os.path.splitext(os.path.basename(file_path))[0]
response_name = stem + ".mu"
try:
path_components = file_path.strip("/").split("/")
path = "/".join(path_components[:-1])+"/" if len(path_components) > 1 else ""
url_scope = f":/page/blob.mu`g={group_name}|r={repo_name}|ref={ref}|path={path}"
mdc = MarkdownToMicron(max_width=self.MAX_RENDER_WIDTH, syntax_highlighter=self.highlighter, url_scope=url_scope)
mu = mdc.format_block(stream.read().decode("utf-8")).rstrip().encode("utf-8")
if not mu:
if tmpdir in link.temporary_directories: link.temporary_directories.remove(tmpdir)
tmpdir.cleanup()
return None
fd, spool_path = tempfile.mkstemp(prefix=stem+".", suffix=".mu", dir=tmpdir.name)
os.close(fd)
with open(spool_path, "wb") as f: f.write(mu)
spool = open(spool_path, "rb")
return [spool, {"name": f"{os.path.splitext(file_name)[0]}.mu".encode("utf-8")}]
except Exception as e:
RNS.log(f"Error during file conversion of {file_path} for {link}: {e}", RNS.LOG_WARNING)
if tmpdir in link.temporary_directories: link.temporary_directories.remove(tmpdir)
try: tmpdir.cleanup()
except Exception: pass
return None
else: return None
else: else:
RNS.log(f"Could not resolve blob stream for download request {group_name}/{repo_name}/{ref}/{file_path}", RNS.LOG_WARNING) RNS.log(f"Could not resolve blob stream for download request {group_name}/{repo_name}/{ref}/{file_path}", RNS.LOG_WARNING)
@ -1994,7 +1762,7 @@ class NomadNetworkNode():
repo = self.get_accessible_repository(remote_identity, group_name, repo_name) repo = self.get_accessible_repository(remote_identity, group_name, repo_name)
if not repo: if not repo:
RNS.log(f"Repository not found or no access for workdoc download request {group_name[:128]}/{repo_name[:128]}/{doc_id}", RNS.LOG_DEBUG) RNS.log(f"Repository not found or no access for workdoc download request {group_name[:128]}/{repo_name[:128]}/{doc_id}", RNS.LOG_WARNING)
return None return None
doc_access = self.resolve_doc_permission(remote_identity, group_name, repo_name, doc_id, self.owner.PERM_READ) doc_access = self.resolve_doc_permission(remote_identity, group_name, repo_name, doc_id, self.owner.PERM_READ)
@ -2295,40 +2063,6 @@ class NomadNetworkNode():
return None return None
def get_webp_stream(self, repo_path, ref, file_path, link_id):
file_path = file_path.strip("/")
link = self.active_links.get(link_id)
if not link:
RNS.log(f"Could not resolve link for media conversion of {file_path}", RNS.LOG_WARNING)
return None
if not hasattr(link, "temporary_directories"): link.temporary_directories = []
tmpdir = tempfile.TemporaryDirectory()
link.temporary_directories.append(tmpdir)
stem = os.path.splitext(os.path.basename(file_path))[0]
response_name = stem + ".webp"
try:
fd, spool_path = tempfile.mkstemp(prefix=stem+".", suffix=".webp", dir=tmpdir.name)
os.close(fd)
converted = media.convert_to_webp(["git", "show", f"{ref}:{file_path}"], spool_path, cwd=repo_path)
if not converted:
if tmpdir in link.temporary_directories: link.temporary_directories.remove(tmpdir)
tmpdir.cleanup()
return None
spool = open(spool_path, "rb")
return spool, response_name
except Exception as e:
RNS.log(f"Error during media conversion of {file_path} for {link}: {e}", RNS.LOG_WARNING)
if tmpdir in link.temporary_directories: link.temporary_directories.remove(tmpdir)
try: tmpdir.cleanup()
except Exception: pass
return None
def get_refs_info(self, repo_path, default_branch=None): def get_refs_info(self, repo_path, default_branch=None):
refs = {"heads": [], "tags": []} refs = {"heads": [], "tags": []}
@ -2456,8 +2190,7 @@ class NomadNetworkNode():
"committer_date": lines[6], "committer_date": lines[6],
"message": "\n".join(lines[7:]).strip(), "message": "\n".join(lines[7:]).strip(),
"files": [], "files": [],
"diff": None, "diff": None }
"signature_status": None }
# Get file change statistics # Get file change statistics
stats_result = subprocess.run(["git", "diff-tree", "--numstat", "-r", commit_hash], stats_result = subprocess.run(["git", "diff-tree", "--numstat", "-r", commit_hash],
@ -2527,84 +2260,6 @@ class NomadNetworkNode():
RNS.log(f"Error getting commit info: {e}", RNS.LOG_WARNING) RNS.log(f"Error getting commit info: {e}", RNS.LOG_WARNING)
return None return None
def get_commit_signature(self, repo_path, commit_hash):
try:
result = subprocess.run(["git", "cat-file", "-p", commit_hash],
cwd=repo_path, capture_output=True, text=True,
timeout=self.GIT_COMMAND_TIMEOUT, check=False)
if result.returncode != 0:
return {"signed": False, "valid": False, "signer_hash": None,
"author_match": False, "message": "Could not read commit object"}
commit_content = result.stdout
lines = commit_content.split("\n")
sig_lines = []
in_signature = False
signed_content_lines = []
for line in lines:
if line.startswith("gpgsig ") or line.startswith("gpgsig-sha256 "):
in_signature = True
sig_start = line.find(" ") + 1
sig_lines.append(line[sig_start:])
elif in_signature:
if line.startswith(" "): sig_lines.append(line[1:])
else:
in_signature = False
signed_content_lines.append(line)
else: signed_content_lines.append(line)
if not sig_lines:
return {"signed": False, "valid": False, "signer_hash": None,
"author_match": False, "message": "Not signed"}
armored_sig = "\n".join(sig_lines)
signed_content = "\n".join(signed_content_lines).encode("utf-8")
try:
sig_data = unarmor_ssh_signature(armored_sig)
try: rsg = parse_ssh_signature(sig_data)["signature_data"]
except ValueError as e:
return {"signed": True, "valid": False, "signer_hash": None,
"author_match": False, "message": "Malformed SSH wrapping for RSG data"}
valid, signed_rsg_data, signing_identity = validate_rsg(rsg, signed_content)
if not valid:
return {"signed": True, "valid": False, "signer_hash": None,
"author_match": False, "message": "Invalid signature"}
signer_hash = RNS.hexrep(signing_identity.hash, delimit=False)
author = extract_commit_author(signed_content)
if not author:
return {"signed": True, "valid": True, "signer_hash": signer_hash,
"author_match": False, "message": "Could not verify author"}
if author == signer_hash:
return {"signed": True, "valid": True, "signer_hash": signer_hash,
"author_match": True, "message": f"Valid, signed by <{signer_hash}>"}
else:
return {"signed": True, "valid": True, "signer_hash": signer_hash,
"author_match": False, "message": f"Invalid signer <{signer_hash}>, author is <{author}>"}
except Exception as e:
RNS.log(f"Error validating commit signature: {e}", RNS.LOG_DEBUG)
return {"signed": True, "valid": False, "signer_hash": None,
"author_match": False, "message": "Signature validation error"}
except subprocess.TimeoutExpired:
RNS.log(f"Timeout checking commit signature", RNS.LOG_WARNING)
return {"signed": False, "valid": False, "signer_hash": None,
"author_match": False, "message": "Timeout"}
except Exception as e:
RNS.log(f"Error checking commit signature: {e}", RNS.LOG_DEBUG)
return {"signed": False, "valid": False, "signer_hash": None,
"author_match": False, "message": "Error"}
def get_readme_content(self, repo_path): def get_readme_content(self, repo_path):
readme_names = [ ("README.mu", False), ("Readme.mu", False), ("readme.mu", False), ("README", False), readme_names = [ ("README.mu", False), ("Readme.mu", False), ("readme.mu", False), ("README", False),
("readme", False), ("README.md", True), ("readme.md", True), ("README.rst", False), ("readme", False), ("README.md", True), ("readme.md", True), ("README.rst", False),
@ -2683,15 +2338,6 @@ class NomadNetworkNode():
return "\n".join(formatted_lines) return "\n".join(formatted_lines)
def format_commit(self, diff_text):
lines = diff_text.replace("\\", "\\\\").split("\n")
formatted_lines = []
for line in lines:
if line.startswith("-"): formatted_lines.append(self.m_escape(f"\\{line}"))
else: formatted_lines.append(self.m_escape(line))
return "\n".join(formatted_lines)
def repository_thanks(self, repo_path, add=False, link_id=None): def repository_thanks(self, repo_path, add=False, link_id=None):
if add: if add:
@ -2819,7 +2465,7 @@ class NomadNetworkNode():
line = "│" line = "│"
for val in data: for val in data:
upper_filled = val >= row_top upper_filled = val >= row_top
lower_filled = val >= row_mid or row == 1 and val > 0 lower_filled = val >= row_mid
if not upper_filled and not lower_filled: line += " " if not upper_filled and not lower_filled: line += " "
elif upper_filled: line += f"`FT{gradient_color(grad_top)}`BT{gradient_color(grad_mid)}▀`f`b" elif upper_filled: line += f"`FT{gradient_color(grad_top)}`BT{gradient_color(grad_mid)}▀`f`b"
@ -2940,38 +2586,11 @@ class NomadNetworkNode():
def remote_connected(self, link): def remote_connected(self, link):
RNS.log(f"Peer connected to {self.destination}", RNS.LOG_DEBUG) RNS.log(f"Peer connected to {self.destination}", RNS.LOG_DEBUG)
self.active_links[link.link_id] = link
link.set_remote_identified_callback(self.remote_identified) link.set_remote_identified_callback(self.remote_identified)
link.set_link_closed_callback(self.remote_disconnected) link.set_link_closed_callback(self.remote_disconnected)
def remote_disconnected(self, link): def remote_disconnected(self, link):
RNS.log(f"Peer disconnected from {self.destination}", RNS.LOG_DEBUG) RNS.log(f"Peer disconnected from {self.destination}", RNS.LOG_DEBUG)
if link.link_id in self.active_links: self.active_links.pop(link.link_id)
self.cleanup_link_temporary_resources(link)
def clean_links(self):
stale_links = []
for link_id, link in self.active_links.items():
if not link.status == RNS.Link.ACTIVE: stale_links.append(link_id)
cleaned_links = 0
for link_id in stale_links:
link = self.active_links.pop(link_id, None)
if link:
self.cleanup_link_temporary_resources(link)
cleaned_links += 1
if cleaned_links > 0: RNS.log(f"Cleaned {cleaned_links} stale link{'s' if cleaned_links != 1 else ''}", RNS.LOG_DEBUG)
def cleanup_link_temporary_resources(self, link):
if hasattr(link, "temporary_directories"):
for tmpdir in link.temporary_directories:
try:
tmpdir.cleanup()
RNS.log(f"Cleaned up {tmpdir.name}", RNS.LOG_DEBUG)
except Exception as e: RNS.log(f"Error while cleaning temporary directory: {e}", RNS.LOG_ERROR)
link.temporary_directories = []
def remote_identified(self, link, identity): def remote_identified(self, link, identity):
RNS.log(f"Peer identified as {link.get_remote_identity()} on {link}", RNS.LOG_DEBUG) RNS.log(f"Peer identified as {link.get_remote_identity()} on {link}", RNS.LOG_DEBUG)
@ -3067,8 +2686,5 @@ DEFAULT_WORK_TEMPLATE = """{PAGE_CONTENT}"""
DEFAULT_WORK_DOC_TEMPLATE = """{PAGE_CONTENT}""" DEFAULT_WORK_DOC_TEMPLATE = """{PAGE_CONTENT}"""
# Error state templates
DEFAULT_NO_IDENT_TEMPLATE = """>>No Identity\n\nThis page requires identification, and none was received.\n"""
# Fallback template # Fallback template
FALLBACK_TEMPLATE = """{PAGE_CONTENT}""" FALLBACK_TEMPLATE = """{PAGE_CONTENT}"""

File diff suppressed because it is too large Load diff

View file

@ -814,6 +814,3 @@ class MarkdownToMicron:
def convert_markdown_to_micron(text): def convert_markdown_to_micron(text):
converter = MarkdownToMicron() converter = MarkdownToMicron()
return converter.format_block(text) return converter.format_block(text)
def medium_path_timeout(default_timeout):
return max(default_timeout, RNS.Reticulum.get_instance().get_medium_path_timeout())

View file

@ -2,7 +2,7 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -492,7 +492,8 @@ def create_rsg(signer_identity, message, embed=False, meta=None, output="bin"):
signed_data = { "hashtype": "sha256", "hash": get_rsg_hash(message), signed_data = { "hashtype": "sha256", "hash": get_rsg_hash(message),
"meta": { "signer": signer_identity.hash, "meta": { "signer": signer_identity.hash,
"pubkey": signer_identity.get_public_key() } } "pubkey": signer_identity.get_public_key(),
"note" : None } } # TODO: Remove default note field in 1.2.9
if embed: if embed:
if type(message) == str: message = message.encode("utf-8") if type(message) == str: message = message.encode("utf-8")
@ -585,20 +586,6 @@ def rsg_meta_from_str(meta, spec=None):
return parsed.dict() return parsed.dict()
def check_release_rsm_structure(signed_data):
release_meta = signed_data.get("meta", None)
if not release_meta: return "No release metadata in manifest"
release_name = release_meta.get("name", None)
release_version = release_meta.get("version", None)
release_origin = release_meta.get("origin", None)
release_origin_path = release_meta.get("path", None)
if not release_name or not release_version: return "Incomplete package data in manifest"
if not release_origin or not release_origin_path: return "Incomplete release origin data in manifest"
if "/" in release_name or "/" in release_version: return "Invalid data in release manifest"
if len(release_origin) != RNS.Identity.TRUNCATED_HASHLENGTH//8: return "Invalid origin hash length in manifest"
if not type(release_origin) == bytes: return "Invalid origin hash in manifest"
return True
################################### ###################################
# Signing & Validation Operations # # Signing & Validation Operations #
################################### ###################################
@ -714,7 +701,7 @@ def validate_message(args, identity, __recursive=False):
elif type(entry) == int: etype = "i" elif type(entry) == int: etype = "i"
elif type(entry) == float: etype = "f" elif type(entry) == float: etype = "f"
elif entry == None: etype = "N" elif entry == None: etype = "N"
if key == "note" and entry == None: return # TODO: Remove this check in 1.3.3 if key == "note" and entry == None: return # TODO: Remove this check in 1.2.9
if type(entry) == bytes: entry = RNS.hexrep(entry, delimit=False) if type(entry) == bytes: entry = RNS.hexrep(entry, delimit=False)
leadin = f"{etype}{indent}{key}="; leadln = len(leadin) leadin = f"{etype}{indent}{key}="; leadln = len(leadin)
entry = f"{entry}"; chunk = entry[:maxwidth]; entry = entry[maxwidth:] entry = f"{entry}"; chunk = entry[:maxwidth]; entry = entry[maxwidth:]

View file

@ -2,7 +2,7 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -55,12 +55,19 @@ def main():
parser.add_argument("--config", action="store", default=None, help="path to alternative Reticulum config directory", type=str) parser.add_argument("--config", action="store", default=None, help="path to alternative Reticulum config directory", type=str)
parser.add_argument('-v', '--verbose', action='count', default=0) parser.add_argument('-v', '--verbose', action='count', default=0)
parser.add_argument('-q', '--quiet', action='count', default=0) parser.add_argument('-q', '--quiet', action='count', default=0)
parser.add_argument("--exampleconfig", action='store_true', default=False, help="print verbose configuration example to stdout and exit")
parser.add_argument("--version", action="version", version="rnir {version}".format(version=__version__)) parser.add_argument("--version", action="version", version="rnir {version}".format(version=__version__))
args = parser.parse_args() args = parser.parse_args()
if args.config: configarg = args.config if args.exampleconfig:
else: configarg = None print(__example_rns_config__)
exit()
if args.config:
configarg = args.config
else:
configarg = None
program_setup(configdir = configarg, verbosity=args.verbose, quietness=args.quiet) program_setup(configdir = configarg, verbosity=args.verbose, quietness=args.quiet)

View file

@ -2,7 +2,7 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -3767,7 +3767,7 @@ def main():
if ec_bt == 0x73: print(f" Bluetooth : Enabled") if ec_bt == 0x73: print(f" Bluetooth : Enabled")
else: print(f" Bluetooth : Disabled") else: print(f" Bluetooth : Disabled")
if ec_wifi == 0x01: print(f" WiFi : Enabled (Station)") if ec_wifi == 0x01: print(f" WiFi : Enabled (Station)")
elif ec_wifi == 0x02: print(f" WiFi : Enabled (AP)") if ec_wifi == 0x02: print(f" WiFi : Enabled (AP)")
else: print(f" WiFi : Disabled") else: print(f" WiFi : Disabled")
if ec_wifi == 0x01 or ec_wifi == 0x02: if ec_wifi == 0x01 or ec_wifi == 0x02:
if not ec_wchn: print(f" Channel : Unknown") if not ec_wchn: print(f" Channel : Unknown")

View file

@ -2,7 +2,7 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -42,7 +42,6 @@ remote_link = None
output_rst_str = "\r \r" output_rst_str = "\r \r"
def connect_remote(destination_hash, auth_identity, timeout, no_output = False, purpose="management"): def connect_remote(destination_hash, auth_identity, timeout, no_output = False, purpose="management"):
global remote_link, reticulum global remote_link, reticulum
timeout = max(timeout, reticulum.get_medium_path_timeout())
if not RNS.Transport.has_path(destination_hash): if not RNS.Transport.has_path(destination_hash):
if not no_output: if not no_output:
print("Path to "+RNS.prettyhexrep(destination_hash)+" requested", end=" ") print("Path to "+RNS.prettyhexrep(destination_hash)+" requested", end=" ")
@ -101,13 +100,11 @@ def parse_hash(input_str):
def program_setup(configdir, table, rates, drop, destination_hexhash, verbosity, timeout, drop_queues, def program_setup(configdir, table, rates, drop, destination_hexhash, verbosity, timeout, drop_queues,
drop_via, max_hops, remote=None, management_identity=None, remote_timeout=RNS.Transport.PATH_REQUEST_TIMEOUT, drop_via, max_hops, remote=None, management_identity=None, remote_timeout=RNS.Transport.PATH_REQUEST_TIMEOUT,
blackholed=False, blackhole=False, unblackhole=False, blackhole_duration=None, blackhole_reason=None, remote_blackhole_list=False, remote_blackhole_list_filter=None, blackholed=False, blackhole=False, unblackhole=False, blackhole_duration=None, blackhole_reason=None,
active=False, usage=False, vias=False, expinfo=False, min_value=None, no_output=False, analyze=False, capture=None, json=False): remote_blackhole_list=False, remote_blackhole_list_filter=None, no_output=False, json=False):
global remote_link, reticulum global remote_link, reticulum
reticulum = RNS.Reticulum(configdir = configdir, loglevel = 3+verbosity) reticulum = RNS.Reticulum(configdir = configdir, loglevel = 3+verbosity)
reticulum.wants_destination_type("rnstransport.remote.management")
reticulum.wants_destination_type("rnstransport.info.blackhole")
if remote: if remote:
try: try:
dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2 dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2
@ -242,10 +239,6 @@ def program_setup(configdir, table, rates, drop, destination_hexhash, verbosity,
print(f"Could not unblackhole identity: {e}") print(f"Could not unblackhole identity: {e}")
exit(20) exit(20)
elif analyze:
analyze_tables(path_table=reticulum.get_path_table(), rate_table=reticulum.get_rate_table(),
link_distribution=reticulum.get_link_distribution(), capture=capture)
elif table: elif table:
destination_hash = None destination_hash = None
if destination_hexhash != None: if destination_hexhash != None:
@ -287,35 +280,13 @@ def program_setup(configdir, table, rates, drop, destination_hexhash, verbosity,
exit() exit()
else: else:
now = time.time()
if usage:
try: table = sorted(table, key=lambda e: e["used"], reverse=True)
except Exception as e: pass
now = time.time()
for path in table: for path in table:
if destination_hash == None or destination_hash == path["hash"]: if destination_hash == None or destination_hash == path["hash"]:
displayed += 1 displayed += 1
if min_value:
if not path["value"]: continue
if path["value"] < min_value: continue
if path["state"] == RNS.Transport.STATE_RESPONSIVE: state_str = " (R)"
elif path["state"] == RNS.Transport.STATE_UNRESPONSIVE: state_str = " (u)"
elif path["state"] == RNS.Transport.STATE_UNKNOWN: state_str = " (?)"
else: state_str = " (?)"
exp_str = RNS.timestamp_str(path["expires"]) exp_str = RNS.timestamp_str(path["expires"])
if path["hops"] == 1: m_str = " " if path["hops"] == 1: m_str = " "
else: m_str = "s" else: m_str = "s"
if not "used" in path: u_str = ", no usage info" print(RNS.prettyhexrep(path["hash"])+" is "+str(path["hops"])+" hop"+m_str+" away via "+RNS.prettyhexrep(path["via"])+" on "+path["interface"]+" expires "+RNS.timestamp_str(path["expires"]))
else:
if active and not path["used"]: continue
if usage: u_str = f", last used {RNS.prettytime(now-path['used'])} ago" if path["used"] else ", never used"
else: u_str = ""
via_str = f" via {RNS.prettyhexrep(path['via'])}" if vias else ""
exp_str = f" expires in {RNS.prettytime(path['expires']-now, compact=True)}" if expinfo else ""
print(f"{RNS.prettyhexrep(path['hash'])} {path['hops']:>3}h{state_str}{via_str} on {path['interface']}{exp_str} (dv:{RNS.prettycost(path['value'])}/lc:{RNS.prettycost(path['lr_cost'])}){u_str}")
if destination_hash != None and displayed == 0: if destination_hash != None and displayed == 0:
print("No path known") print("No path known")
@ -480,7 +451,7 @@ def program_setup(configdir, table, rates, drop, destination_hexhash, verbosity,
i = 0 i = 0
syms = "⢄⢂⢁⡁⡈⡐⡠" syms = "⢄⢂⢁⡁⡈⡐⡠"
limit = time.time()+max(timeout, reticulum.get_medium_path_timeout()) limit = time.time()+timeout
while not RNS.Transport.has_path(destination_hash) and time.time()<limit: while not RNS.Transport.has_path(destination_hash) and time.time()<limit:
time.sleep(0.1) time.sleep(0.1)
print(("\b\b"+syms[i]+" "), end="") print(("\b\b"+syms[i]+" "), end="")
@ -512,12 +483,7 @@ def main():
parser.add_argument("--config", action="store", default=None, help="path to alternative Reticulum config directory", type=str) parser.add_argument("--config", action="store", default=None, help="path to alternative Reticulum config directory", type=str)
parser.add_argument("--version", action="version", version="rnpath {version}".format(version=__version__)) parser.add_argument("--version", action="version", version="rnpath {version}".format(version=__version__))
parser.add_argument("-t", "--table", action="store_true", help="show all known paths", default=False) parser.add_argument("-t", "--table", action="store_true", help="show all known paths", default=False)
parser.add_argument("-a", "--active", action="store_true", help="show in-use paths only", default=False)
parser.add_argument("-u", "--usage", action="store_true", help="show path usage time", default=False)
parser.add_argument("-V", "--vias", action="store_true", help="show via transport IDs", default=False)
parser.add_argument("-E", "--expiry", action="store_true", help="show path expiry times", default=False)
parser.add_argument("-m", "--max", action="store", metavar="hops", type=int, help="maximum hops to filter path table by", default=None) parser.add_argument("-m", "--max", action="store", metavar="hops", type=int, help="maximum hops to filter path table by", default=None)
parser.add_argument("-M", "--min-value", action="store", metavar="value", type=float, help="minimum value to filter path table by", default=None)
parser.add_argument("-r", "--rates", action="store_true", help="show announce rate info", default=False) parser.add_argument("-r", "--rates", action="store_true", help="show announce rate info", default=False)
parser.add_argument("-d", "--drop", action="store_true", help="remove the path to a destination", default=False) parser.add_argument("-d", "--drop", action="store_true", help="remove the path to a destination", default=False)
parser.add_argument("-D", "--drop-announces", action="store_true", help="drop all queued announces", default=False) parser.add_argument("-D", "--drop-announces", action="store_true", help="drop all queued announces", default=False)
@ -529,12 +495,10 @@ def main():
parser.add_argument("-b", "--blackholed", action="store_true", help="list blackholed identities", default=False) parser.add_argument("-b", "--blackholed", action="store_true", help="list blackholed identities", default=False)
parser.add_argument("-B", "--blackhole", action="store_true", help="blackhole identity", default=False) parser.add_argument("-B", "--blackhole", action="store_true", help="blackhole identity", default=False)
parser.add_argument("-U", "--unblackhole", action="store_true", help="unblackhole identity", default=False) parser.add_argument("-U", "--unblackhole", action="store_true", help="unblackhole identity", default=False)
parser.add_argument( "--duration", action="store", metavar="hours", type=float, help="duration of blackhole enforcement in hours", default=None) parser.add_argument( "--duration", action="store", type=float, help="duration of blackhole enforcement in hours", default=None)
parser.add_argument( "--reason", action="store", metavar="reason", type=str, help="reason for blackholing identity", default=None) parser.add_argument( "--reason", action="store", type=str, help="reason for blackholing identity", default=None)
parser.add_argument("-p", "--blackholed-list", action="store_true", help="view published blackhole list for remote transport instance", default=False) parser.add_argument("-p", "--blackholed-list", action="store_true", help="view published blackhole list for remote transport instance", default=False)
parser.add_argument("-j", "--json", action="store_true", help="output in JSON format", default=False) parser.add_argument("-j", "--json", action="store_true", help="output in JSON format", default=False)
parser.add_argument("-A", "--analyze", action="store_true", help="Analyze tables", default=False)
parser.add_argument("--capture", action="store", metavar="path", type=str, help="path to capture file for analyze mode", default=None)
parser.add_argument("destination", nargs="?", default=None, help="hexadecimal hash of the destination", type=str) parser.add_argument("destination", nargs="?", default=None, help="hexadecimal hash of the destination", type=str)
parser.add_argument("list_filter", nargs="?", default=None, help="filter for remote blackhole list view", type=str) parser.add_argument("list_filter", nargs="?", default=None, help="filter for remote blackhole list view", type=str)
parser.add_argument('-v', '--verbose', action='count', default=0) parser.add_argument('-v', '--verbose', action='count', default=0)
@ -544,17 +508,16 @@ def main():
if args.config: configarg = args.config if args.config: configarg = args.config
else: configarg = None else: configarg = None
if not args.drop_announces and not args.table and not args.rates and not args.destination and not args.drop_via and not args.blackholed and not args.analyze: if not args.drop_announces and not args.table and not args.rates and not args.destination and not args.drop_via and not args.blackholed:
print("") print("")
parser.print_help() parser.print_help()
print("") print("")
else: else:
program_setup(configdir = configarg, table = args.table, rates = args.rates, drop = args.drop, destination_hexhash = args.destination, program_setup(configdir = configarg, table = args.table, rates = args.rates, drop = args.drop, destination_hexhash = args.destination,
verbosity=args.verbose, timeout=args.w, drop_queues=args.drop_announces, drop_via=args.drop_via, max_hops=args.max, min_value=args.min_value, verbosity = args.verbose, timeout = args.w, drop_queues = args.drop_announces, drop_via = args.drop_via, max_hops = args.max,
remote=args.R, management_identity=args.i, remote_timeout=args.W, blackholed=args.blackholed, blackhole=args.blackhole, remote=args.R, management_identity=args.i, remote_timeout=args.W, blackholed=args.blackholed, blackhole=args.blackhole,
unblackhole=args.unblackhole, blackhole_duration=args.duration, blackhole_reason=args.reason, remote_blackhole_list=args.blackholed_list, unblackhole=args.unblackhole, blackhole_duration=args.duration, blackhole_reason=args.reason, remote_blackhole_list=args.blackholed_list,
remote_blackhole_list_filter=args.list_filter, usage=args.usage, vias=args.vias, active=args.active, expinfo=args.expiry, remote_blackhole_list_filter=args.list_filter, json=args.json)
analyze=args.analyze, capture=args.capture, json=args.json)
sys.exit(0) sys.exit(0)
@ -562,94 +525,6 @@ def main():
print("") print("")
exit() exit()
def analyze_tables(path_table=None, rate_table=None, link_distribution=None, capture=None):
print("Not yet implemented")
exit()
# if not path_table:
# print("No path table to analyze")
# exit()
# else:
# print(f"Analyzing tables...")
# # TODO: If a `capture` path is specified, we save and (if existing) load from
# # this file, so that `rnpath --analyze` can be run at intervals, and continuously
# # collect data, instead of only providing an at-the-moment snapshot view.
# # Locally initialize announce cache to be able to
# # fetch full announce packet data
# if RNS.Transport.owner.is_connected_to_shared_instance: RNS.Transport.init_announce_cache()
# tm = RNS.Transport.destination_type_map
# total = len(path_table)
# destinations = {}
# unresolvable = 0
# unknown_types = set()
# known_types = set()
# unknown_type_dsts = 0
# known_type_dsts = 0
# for path in path_table:
# dst_hash = path["hash"]
# dst_hops = path["hops"]
# last_used = path["used"]
# name_hash = None
# kd_entry = RNS.Identity.known_destinations.get(dst_hash, None)
# # New destinations may have arrived after our initial known
# # destinations snapshot, which is fine, we just count the few
# # that miss here for now.
# if not kd_entry: unresolvable += 1
# # Enrich data with information from cached announces
# else:
# dst_value = kd_entry[6]
# dst_lr_cost = kd_entry[7]
# announce_hash = kd_entry[1]
# announce_packet = RNS.Transport.get_cached_packet(announce_hash, packet_type="announce")
# if not announce_packet: unresolvable += 1; continue
# try: announce_packet.unpack()
# except: unresolvable +=1; continue
# keysize = RNS.Identity.KEYSIZE//8
# name_hash_len = RNS.Identity.NAME_HASH_LENGTH//8
# name_hash = announce_packet.data[keysize:keysize+name_hash_len]
# app_data = kd_entry[3] or RNS.Identity.recall_app_data(dst_hash, _no_use=True)
# app_data_len = len(app_data) if app_data else None # No len counting if no app data
# if name_hash in tm: known_type_dsts += 1; known_types.add(name_hash)
# else: unknown_type_dsts += 1; unknown_types.add(name_hash)
# # TODO: Create actual data structure for use in analysis
# entry = {}
# destinations[dst_hash] = entry
# if rate_table:
# # TODO: Incorporate and enrich with data from rate table
# pass
# if link_distribution:
# # TODO: Incorporate and enrich with link distribution data
# pass
# # TODO: Actually analyze tables
# pass
# # TODO: Actually print a comprehensive, beautifully formatted
# # overview, charts and data. This is a stub.
# print(f"Known types : {len(known_types)}")
# print(f"Unknown types : {len(unknown_types)}")
# print(f"Total types : {len(known_types)+len(unknown_types)}")
# print(f"Known type dests : {known_type_dsts}")
# print(f"Unknown type dests : {unknown_type_dsts}")
# print(f"Total type dests : {known_type_dsts+unknown_type_dsts}")
# # TODO: Ensure we get the terminal width if possible, and fall back to
# # a sensible default if not, so everything is rendered beautifully.
# exit()
def pretty_date(time=False): def pretty_date(time=False):
from datetime import datetime from datetime import datetime
now = datetime.now() now = datetime.now()

View file

@ -2,7 +2,7 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal

View file

@ -2,7 +2,7 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -81,7 +81,7 @@ def program_setup(configdir, destination_hexhash, size=None, full_name = None, v
print("Path to "+RNS.prettyhexrep(destination_hash)+" requested ", end=" ") print("Path to "+RNS.prettyhexrep(destination_hash)+" requested ", end=" ")
sys.stdout.flush() sys.stdout.flush()
_timeout = time.time() + (timeout or max(DEFAULT_TIMEOUT+reticulum.get_first_hop_timeout(destination_hash), reticulum.get_medium_path_timeout())) _timeout = time.time() + (timeout or DEFAULT_TIMEOUT+reticulum.get_first_hop_timeout(destination_hash))
i = 0 i = 0
syms = "⢄⢂⢁⡁⡈⡐⡠" syms = "⢄⢂⢁⡁⡈⡐⡠"
while not RNS.Transport.has_path(destination_hash) and not time.time() > _timeout: while not RNS.Transport.has_path(destination_hash) and not time.time() > _timeout:
@ -131,7 +131,7 @@ def program_setup(configdir, destination_hexhash, size=None, full_name = None, v
print("\rSent probe "+str(sent)+" ("+str(size)+" bytes) to "+RNS.prettyhexrep(destination_hash)+more+" ", end=" ") print("\rSent probe "+str(sent)+" ("+str(size)+" bytes) to "+RNS.prettyhexrep(destination_hash)+more+" ", end=" ")
_timeout = time.time() + (timeout or max(DEFAULT_TIMEOUT+reticulum.get_first_hop_timeout(destination_hash), reticulum.get_medium_path_timeout())) _timeout = time.time() + (timeout or DEFAULT_TIMEOUT+reticulum.get_first_hop_timeout(destination_hash))
i = 0 i = 0
while receipt.status == RNS.PacketReceipt.SENT and not time.time() > _timeout: while receipt.status == RNS.PacketReceipt.SENT and not time.time() > _timeout:
time.sleep(0.1) time.sleep(0.1)

View file

@ -2,7 +2,7 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -100,7 +100,7 @@ __example_rns_config__ = '''# This is an example Reticulum config file.
# always-on. This directive is optional and can be removed # always-on. This directive is optional and can be removed
# for brevity. # for brevity.
enable_transport = no enable_transport = No
# By default, the first program to launch the Reticulum # By default, the first program to launch the Reticulum
@ -112,7 +112,7 @@ enable_transport = no
# user, and should generally be turned on. This directive # user, and should generally be turned on. This directive
# is optional and can be removed for brevity. # is optional and can be removed for brevity.
share_instance = yes share_instance = Yes
# If you want to run multiple *different* shared instances # If you want to run multiple *different* shared instances
@ -122,7 +122,6 @@ share_instance = yes
instance_name = default instance_name = default
# Some platforms don't support domain sockets, and if that # Some platforms don't support domain sockets, and if that
# is the case, you can isolate different instances by # is the case, you can isolate different instances by
# specifying a unique set of ports for each: # specifying a unique set of ports for each:
@ -174,18 +173,12 @@ instance_name = default
# network_identity = ~/.reticulum/storage/identity/network # network_identity = ~/.reticulum/storage/identity/network
# It is possible to configure the global default interface
# gravity. This value will be used if an interface does
# not have a specific gravity value in its configuration.
# default_gravity = 0
# You can configure whether Reticulum should discover # You can configure whether Reticulum should discover
# available interfaces from other Transport Instances over # available interfaces from other Transport Instances over
# the network. If this option is enabled, Reticulum will # the network. If this option is enabled, Reticulum will
# collect interface information discovered from the network. # collect interface information discovered from the network.
# discover_interfaces = no # discover_interfaces = No
# If you only want to discover interfaces from specific # If you only want to discover interfaces from specific
@ -216,40 +209,13 @@ instance_name = default
# required_discovery_value = 14 # required_discovery_value = 14
# You can disable allowing interfaces to be attached, de-
# tached or reloaded via rnstatus, for example if running
# on a shared system where users with normal RPC access
# should not be allowed to control interface states.
# enable_interface_management = no
# You can configure Reticulum to panic and forcibly close # You can configure Reticulum to panic and forcibly close
# if an unrecoverable interface error occurs, such as the # if an unrecoverable interface error occurs, such as the
# hardware device for an interface disappearing. This is # hardware device for an interface disappearing. This is
# an optional directive, and can be left out for brevity. # an optional directive, and can be left out for brevity.
# This behaviour is disabled by default. # This behaviour is disabled by default.
# panic_on_interface_error = no # panic_on_interface_error = No
# You can specify which mode discovered interfaces should
# be created with when auto-connecting.
# autoconnect_interface_mode = gw
# It is possible to allow announces from auto-connected
# interfaces to propagate announces to internal-mode
# interfaces, even if the auto-connected interface's mode
# would normally not allow for this.
# autoconnect_announces_to_internal = yes
# You can configure the gravity for discovered interfaces
# that are auto-connected.
# autoconnect_interface_gravity = 0
# When Transport is enabled, it is possible to allow the # When Transport is enabled, it is possible to allow the
@ -260,15 +226,14 @@ instance_name = default
# Transport Instance, and printed to the log at startup. # Transport Instance, and printed to the log at startup.
# Optional, and disabled by default. # Optional, and disabled by default.
# respond_to_probes = no # respond_to_probes = No
# You can publish your local list of blackholed identities # You can publish your local list of blackholed identities
# for other transport instances to use for automatic, # for other transport instances to use for automatic,
# network-wide blackhole management. # network-wide blackhole management.
# publish_blackhole = no # publish_blackhole = No
# List of remote transport identities from which to auto- # List of remote transport identities from which to auto-
# matically source lists of blackholed identities. # matically source lists of blackholed identities.
@ -284,97 +249,8 @@ instance_name = default
# blackhole_sources = 521c87a83afb8f29e4455e77930b973b # blackhole_sources = 521c87a83afb8f29e4455e77930b973b
# You can set the interval in minutes at which remote
# blackhole sources are updated. Defaults to one hour.
# blackhole_update_interval = 60
# You can enable a hop limiter, that will drop all packets
# exceeding the specified maximum hop count.
# hop_limiter = 16
# When not running as a transport node, it is possible to
# force the use of the same, static transport identity
# for every instance start. Defaults to using a new identity
# at every instance start, if transport is disabled.
# static_transport_identity = no
[stamps]
# This section configures stamping behaviour and defaults.
# In most cases, you do not need to change this, but on
# constrained systems especially, tuning these parameters
# can be very beneficial
# During the 1.6.x phased transition to network stamp usage,
# it is possible to enable early opt-in on stamp generation
# and enforcement. RNS 1.6.0 includes all the required stamp
# generation and verification functionality, and actively
# validates stamps, while leaving generation, emission and
# enforcement disabled by default for destinations and PRs.
# Link requests will be stamped already from 1.6.0 if a cost
# has been announced by the destination. Version 1.6.1 will
# enable generation and emission by default, and version
# 1.6.2 will finally activate enforcement. Any of these
# options can be enabled early by configuration:
# stamp_destinations = yes
# stamp_prs = yes
# enforce_stamps = yes
# The default required stamp cost for an announce, and its
# destination to be considered valid. This applies both to
# announces received directly, and transported by this node,
# if transport is enabled. Any announced destination not
# meeting the cost will be ignored.
# destination_cost = 23.5
# The default value this instance will target when generating
# new destination stamps. An application may set a higher
# target, but applications requesting a lower value than the
# default will always generate at least the default value.
# Destination stamps are cached and reused, so a one-time
# high-value generation is usually a good idea.
# destination_value = 25.0
# The default required path request stamp cost. Path requests
# that do not meet the requirement are ignored.
# pr_cost = 15.8
# The default target value of path request stamps generated
# by this instance.
# pr_value = 15.8
# The default link request stamp cost to use, if one has not
# been announced by the destination the link is established
# to, or if it is unknown. In practice, all destinations will
# include their required link cost in each announce, and this
# option is rarely necessary.
# lr_cost = 12.5
# The maximum amount of time to spend attempting to generate
# a link request and path request stamps, in seconds.
# lr_stamp_timeout = 15
# pr_stamp_timeout = 15
[logging] [logging]
# Valid log levels are 0 through 8: # Valid log levels are 0 through 7:
# 0: Log only critical information # 0: Log only critical information
# 1: Log errors and lower log levels # 1: Log errors and lower log levels
# 2: Log warnings and lower log levels # 2: Log warnings and lower log levels
@ -382,17 +258,10 @@ instance_name = default
# 4: Log info and lower (this is the default) # 4: Log info and lower (this is the default)
# 5: Verbose logging # 5: Verbose logging
# 6: Debug logging # 6: Debug logging
# 7: Path logging # 7: Extreme logging
# 8: Extreme logging
loglevel = 4 loglevel = 4
# You can disable timestamp inclusion in logs. Useful if
# you want to use an external logging tool that provides
# its own timestamps or custom formatting.
# logtimestamps = no
# The interfaces section defines the physical and virtual # The interfaces section defines the physical and virtual
# interfaces Reticulum will use to communicate on. This # interfaces Reticulum will use to communicate on. This

View file

@ -35,15 +35,7 @@
from ._version import __version__ from ._version import __version__
import os import os
import glob
module_abs_filename = os.path.abspath(__file__) module_abs_filename = os.path.abspath(__file__)
module_dir = os.path.dirname(module_abs_filename) module_dir = os.path.dirname(module_abs_filename)
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
so_modules = glob.glob(os.path.dirname(__file__)+"/*.so")
pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd")
modules = py_modules+pyc_modules+so_modules+pyd_modules
__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))
def _get_version(): return __version__ def _get_version(): return __version__

View file

@ -1 +1 @@
__version__ = "0.3.0" __version__ = "0.2.0"

View file

@ -44,8 +44,7 @@ def setup_argument_parser():
parser = argparse.ArgumentParser(description="Reticulum Remote Shell Utility", epilog="When specifying a command to execute, separate rnsh\noptions from the command and its arguments with --\n\nFor example:\n rnsh -l -- /bin/bash --login\n rnsh <destination> -- ls -la /tmp", formatter_class=argparse.RawDescriptionHelpFormatter) parser = argparse.ArgumentParser(description="Reticulum Remote Shell Utility", epilog="When specifying a command to execute, separate rnsh\noptions from the command and its arguments with --\n\nFor example:\n rnsh -l -- /bin/bash --login\n rnsh <destination> -- ls -la /tmp", formatter_class=argparse.RawDescriptionHelpFormatter)
# Common options # Common options
parser.add_argument("--config", "-c", action="store", default=None, help="path to config directory", type=str) parser.add_argument("--config", "-c", action="store", default=None, help="path to alternative Reticulum config directory", type=str)
parser.add_argument("--rnsconfig", action="store", default=None, help="path to alternative Reticulum config directory", type=str)
parser.add_argument("--identity", "-i", action="store", default=None, help="path to identity file to use", type=str) parser.add_argument("--identity", "-i", action="store", default=None, help="path to identity file to use", type=str)
parser.add_argument("-v", "--verbose", action="count", default=0, help="increase verbosity") parser.add_argument("-v", "--verbose", action="count", default=0, help="increase verbosity")
parser.add_argument("-q", "--quiet", action="count", default=0, help="decrease verbosity") parser.add_argument("-q", "--quiet", action="count", default=0, help="decrease verbosity")

View file

@ -53,6 +53,8 @@ class permit(AbstractContextManager):
""" """
def __init__(self, *exceptions): self._exceptions = exceptions def __init__(self, *exceptions): self._exceptions = exceptions
def __enter__(self): pass def __enter__(self): pass
def __exit__(self, exctype, excinst, exctb): def __exit__(self, exctype, excinst, exctb):
return exctype is not None and not issubclass(exctype, self._exceptions) return exctype is not None and not issubclass(exctype, self._exceptions)

View file

@ -55,4 +55,5 @@ class SleepRate:
return sleep_for if sleep_for > 0 else 0 return sleep_for if sleep_for > 0 else 0
async def sleep_async(self): await asyncio.sleep(self.next_sleep_time()) async def sleep_async(self): await asyncio.sleep(self.next_sleep_time())
def sleep_block(self): time.sleep(self.next_sleep_time()) def sleep_block(self): time.sleep(self.next_sleep_time())

View file

@ -35,6 +35,7 @@
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
import base64
import enum import enum
import functools import functools
import os import os
@ -52,8 +53,10 @@ import RNS.Utilities.rnsh.exception as exception
import RNS.Utilities.rnsh.process as process import RNS.Utilities.rnsh.process as process
import RNS.Utilities.rnsh.retry as retry import RNS.Utilities.rnsh.retry as retry
import RNS.Utilities.rnsh.session as session import RNS.Utilities.rnsh.session as session
import re
import contextlib import contextlib
import pwd
import bz2 import bz2
import RNS.Utilities.rnsh.protocol as protocol import RNS.Utilities.rnsh.protocol as protocol
import RNS.Utilities.rnsh.helpers as helpers import RNS.Utilities.rnsh.helpers as helpers
@ -68,6 +71,7 @@ _retry_timer: retry.RetryThread | None = None
_destination: RNS.Destination | None = None _destination: RNS.Destination | None = None
_loop: asyncio.AbstractEventLoop | None = None _loop: asyncio.AbstractEventLoop | None = None
async def _check_finished(timeout: float = 0): async def _check_finished(timeout: float = 0):
return _finished is not None and await process.event_wait(_finished, timeout=timeout) return _finished is not None and await process.event_wait(_finished, timeout=timeout)
@ -88,11 +92,13 @@ async def _spin_tty(until=None, msg=None, timeout=None):
print(("\b\b"+syms[i]+" "), end="") print(("\b\b"+syms[i]+" "), end="")
sys.stdout.flush() sys.stdout.flush()
i = (i+1)%len(syms) i = (i+1)%len(syms)
print("\r"+" "*len(msg)+" \r", end="") print("\r"+" "*len(msg)+" \r", end="")
if timeout != None and time.time() > timeout: return False if timeout != None and time.time() > timeout: return False
else: return True else: return True
async def _spin_pipe(until: callable = None, msg=None, timeout: float | None = None) -> bool: async def _spin_pipe(until: callable = None, msg=None, timeout: float | None = None) -> bool:
if timeout is not None: timeout += time.time() if timeout is not None: timeout += time.time()
@ -127,15 +133,18 @@ def _client_message_handler(message: RNS.MessageBase): _pq.put(message)
def compute_target_rns_loglevel(verbosity: int, quietness: int, base_level: int = RNS.LOG_INFO) -> int: def compute_target_rns_loglevel(verbosity: int, quietness: int, base_level: int = RNS.LOG_INFO) -> int:
try: try:
target = int(base_level) + int(verbosity) - int(quietness) target = int(base_level) + int(verbosity) - int(quietness)
if target < RNS.LOG_NONE: target = RNS.LOG_NONE if target < RNS.LOG_CRITICAL: target = RNS.LOG_CRITICAL
if target > RNS.LOG_EXTREME: target = RNS.LOG_EXTREME if target > RNS.LOG_DEBUG: target = RNS.LOG_DEBUG
return target return target
except Exception: return base_level except Exception: return base_level
class RemoteExecutionError(Exception): class RemoteExecutionError(Exception):
def __init__(self, msg): self.msg = msg def __init__(self, msg): self.msg = msg
async def _initiate_link(configdir, rnsconfigdir, identitypath=None, logfile=None, verbosity=0, quietness=0, noid=False, destination=None,
async def _initiate_link(configdir, rnsconfigdir, identitypath=None, verbosity=0, quietness=0, noid=False, destination=None,
timeout=RNS.Transport.PATH_REQUEST_TIMEOUT): timeout=RNS.Transport.PATH_REQUEST_TIMEOUT):
global _identity, _reticulum, _link, _destination, _remote_exec_grace global _identity, _reticulum, _link, _destination, _remote_exec_grace
@ -144,61 +153,67 @@ async def _initiate_link(configdir, rnsconfigdir, identitypath=None, logfile=Non
raise RemoteExecutionError( raise RemoteExecutionError(
"Allowed destination length is invalid, must be {hex} hexadecimal characters ({byte} bytes).".format( "Allowed destination length is invalid, must be {hex} hexadecimal characters ({byte} bytes).".format(
hex=dest_len, byte=dest_len // 2)) hex=dest_len, byte=dest_len // 2))
try:
try: destination_hash = bytes.fromhex(destination) destination_hash = bytes.fromhex(destination)
except Exception as e: raise RemoteExecutionError("Invalid destination entered. Check your input.") except Exception as e:
raise RemoteExecutionError("Invalid destination entered. Check your input.")
if _reticulum is None: if _reticulum is None:
targetloglevel = compute_target_rns_loglevel(verbosity, quietness, RNS.LOG_INFO) targetloglevel = compute_target_rns_loglevel(verbosity, quietness, RNS.LOG_ERROR)
RNS.logfile = logfile RNS.logfile = os.path.join(configdir, "logfile")
_reticulum = RNS.Reticulum(configdir=rnsconfigdir, loglevel=targetloglevel, logdest=RNS.LOG_FILE) _reticulum = RNS.Reticulum(configdir=rnsconfigdir, loglevel=targetloglevel, logdest=RNS.LOG_FILE)
_reticulum.wants_destination_type(f"{rnsh.APP_NAME}")
if _identity is None: if _identity is None:
_identity = rnsh.prepare_identity(identity_path=identitypath, service_name=None, configdir=configdir) _identity = rnsh.prepare_identity(identitypath)
if not RNS.Transport.has_path(destination_hash): if not RNS.Transport.has_path(destination_hash):
RNS.Transport.request_path(destination_hash) RNS.Transport.request_path(destination_hash)
RNS.log(f"Requesting path to {RNS.prettyhexrep(destination_hash)}", RNS.LOG_INFO) RNS.log(f"Requesting path...", RNS.LOG_INFO)
if not await _spin(until=lambda: RNS.Transport.has_path(destination_hash), msg=f"Requesting path to {RNS.prettyhexrep(destination_hash)}", if not await _spin(until=lambda: RNS.Transport.has_path(destination_hash), msg="Requesting path...",
timeout=timeout, quiet=quietness > 0): timeout=timeout, quiet=quietness > 0):
raise RemoteExecutionError("Path not found") raise RemoteExecutionError("Path not found")
if _destination is None: if _destination is None:
listener_identity = RNS.Identity.recall(destination_hash) listener_identity = RNS.Identity.recall(destination_hash)
_destination = RNS.Destination(listener_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, rnsh.APP_NAME) _destination = RNS.Destination(
listener_identity,
RNS.Destination.OUT,
RNS.Destination.SINGLE,
rnsh.APP_NAME
)
RNS.log(f"Establishing link with {RNS.prettyhexrep(destination_hash)}", RNS.LOG_INFO)
if _link is None or _link.status == RNS.Link.PENDING: if _link is None or _link.status == RNS.Link.PENDING:
RNS.log("No link", RNS.LOG_DEBUG)
_link = RNS.Link(_destination) _link = RNS.Link(_destination)
_link.did_identify = False _link.did_identify = False
_link.set_link_closed_callback(_client_link_closed) _link.set_link_closed_callback(_client_link_closed)
if not await _spin(until=lambda: _link.status == RNS.Link.ACTIVE, msg=f"Establishing link with {RNS.prettyhexrep(destination_hash)}", RNS.log(f"Establishing link...", RNS.LOG_VERBOSE)
if not await _spin(until=lambda: _link.status == RNS.Link.ACTIVE, msg="Establishing link...",
timeout=timeout, quiet=quietness > 0): timeout=timeout, quiet=quietness > 0):
raise RemoteExecutionError("Could not establish link with " + RNS.prettyhexrep(destination_hash)) raise RemoteExecutionError("Could not establish link with " + RNS.prettyhexrep(destination_hash))
RNS.log(f"Link established with {RNS.prettyhexrep(destination_hash)}", RNS.LOG_INFO) RNS.log("Have link", RNS.LOG_DEBUG)
if not noid and not _link.did_identify: if not noid and not _link.did_identify:
# Delay a tiny bit to allow listener to fully enter WAIT_IDENT state # Delay a tiny bit to allow listener to fully enter WAIT_IDENT state
await asyncio.sleep(min(1, _link.rtt * 1.1 + 0.05)) await asyncio.sleep(min(1, _link.rtt * 1.1 + 0.05))
_link.identify(_identity) _link.identify(_identity)
_link.did_identify = True _link.did_identify = True
async def _handle_error(errmsg: RNS.MessageBase): async def _handle_error(errmsg: RNS.MessageBase):
if isinstance(errmsg, protocol.ErrorMessage): if isinstance(errmsg, protocol.ErrorMessage):
with contextlib.suppress(Exception): with contextlib.suppress(Exception):
if _link and _link.status == RNS.Link.ACTIVE: _link.teardown() if _link and _link.status == RNS.Link.ACTIVE:
_link.teardown()
await asyncio.sleep(0.1) await asyncio.sleep(0.1)
raise RemoteExecutionError(f"Remote error: {errmsg.msg}") raise RemoteExecutionError(f"Remote error: {errmsg.msg}")
async def initiate(configdir: str, rnsconfigdir:str, identitypath: str, logfile:str, verbosity: int, quietness: int, noid: bool, destination: str,
timeout: float, command: [str] | None = None):
async def initiate(configdir: str, rnsconfigdir:str, identitypath: str, verbosity: int, quietness: int, noid: bool, destination: str,
timeout: float, command: [str] | None = None):
global _finished, _link global _finished, _link
if timeout is None: timeout = RNS.Transport.PATH_REQUEST_TIMEOUT
with process.TTYRestorer(sys.stdin.fileno()) as ttyRestorer: with process.TTYRestorer(sys.stdin.fileno()) as ttyRestorer:
loop = asyncio.get_running_loop() loop = asyncio.get_running_loop()
state = InitiatorState.IS_INITIAL state = InitiatorState.IS_INITIAL
@ -208,26 +223,33 @@ async def initiate(configdir: str, rnsconfigdir:str, identitypath: str, logfile:
await _initiate_link(configdir=configdir, await _initiate_link(configdir=configdir,
rnsconfigdir=rnsconfigdir, rnsconfigdir=rnsconfigdir,
identitypath=identitypath, identitypath=identitypath,
logfile=logfile,
verbosity=verbosity, verbosity=verbosity,
quietness=quietness, quietness=quietness,
noid=noid, noid=noid,
destination=destination, destination=destination,
timeout=timeout) timeout=timeout)
if not _link or _link.status not in [RNS.Link.ACTIVE, RNS.Link.PENDING]: return 255 if not _link or _link.status not in [RNS.Link.ACTIVE, RNS.Link.PENDING]:
return 255
state = InitiatorState.IS_LINKED state = InitiatorState.IS_LINKED
outlet = session.RNSOutlet(_link) outlet = session.RNSOutlet(_link)
channel = _link.get_channel() channel = _link.get_channel()
protocol.register_message_types(channel) protocol.register_message_types(channel)
channel.add_message_handler(_client_message_handler) channel.add_message_handler(_client_message_handler)
# Next step after linking and identifying: send version
# if not await _spin(lambda: messenger.is_outlet_ready(outlet), timeout=5, quiet=quietness > 0):
# print("Error bringing up link")
# return 253
channel.send(protocol.VersionInfoMessage()) channel.send(protocol.VersionInfoMessage())
try: try:
vm = _pq.get(timeout=max(outlet.rtt * 20, 5)) vm = _pq.get(timeout=max(outlet.rtt * 20, 5))
await _handle_error(vm) await _handle_error(vm)
if not isinstance(vm, protocol.VersionInfoMessage): raise Exception("Invalid message received") if not isinstance(vm, protocol.VersionInfoMessage):
RNS.log(f"Connected server version info: sw {vm.sw_version}, proto {vm.protocol_version}", RNS.LOG_INFO) raise Exception("Invalid message received")
RNS.log(f"Server version info: sw {vm.sw_version} prot {vm.protocol_version}", RNS.LOG_DEBUG)
state = InitiatorState.IS_RUNNING state = InitiatorState.IS_RUNNING
except queue.Empty: except queue.Empty:
print("Protocol error") print("Protocol error")
@ -259,8 +281,10 @@ async def initiate(configdir: str, rnsconfigdir:str, identitypath: str, logfile:
return None return None
elif b == "L": elif b == "L":
line_mode = not line_mode line_mode = not line_mode
if line_mode: os.write(1, "\n\rLine-interactive mode enabled\n\r".encode("utf-8")) if line_mode:
else: os.write(1, "\n\rLine-interactive mode disabled\n\r".encode("utf-8")) os.write(1, "\n\rLine-interactive mode enabled\n\r".encode("utf-8"))
else:
os.write(1, "\n\rLine-interactive mode disabled\n\r".encode("utf-8"))
return None return None
return b return b
@ -303,7 +327,8 @@ async def initiate(configdir: str, rnsconfigdir:str, identitypath: str, logfile:
pre_esc = False pre_esc = False
data.append(b) data.append(b)
if not line_mode: data_buffer.extend(data) if not line_mode:
data_buffer.extend(data)
else: else:
line_buffer.extend(data) line_buffer.extend(data)
if line_flush: if line_flush:
@ -317,7 +342,8 @@ async def initiate(configdir: str, rnsconfigdir:str, identitypath: str, logfile:
blind_write_count += len(data) blind_write_count += len(data)
except EOFError: except EOFError:
if os.isatty(0): data_buffer.extend(process.CTRL_D) if os.isatty(0):
data_buffer.extend(process.CTRL_D)
stdin_eof = True stdin_eof = True
process.tty_unset_reader_callbacks(sys.stdin.fileno()) process.tty_unset_reader_callbacks(sys.stdin.fileno())
@ -370,7 +396,7 @@ async def initiate(configdir: str, rnsconfigdir:str, identitypath: str, logfile:
if message.stream_id == protocol.StreamDataMessage.STREAM_ID_STDOUT: if message.stream_id == protocol.StreamDataMessage.STREAM_ID_STDOUT:
if message.data and len(message.data) > 0: if message.data and len(message.data) > 0:
ttyRestorer.raw() ttyRestorer.raw()
# RNS.log(f"stdout: {message.data}", RNS.LOG_EXTREME) RNS.log(f"stdout: {message.data}", RNS.LOG_DEBUG)
os.write(1, message.data) os.write(1, message.data)
sys.stdout.flush() sys.stdout.flush()
if message.eof: if message.eof:
@ -378,7 +404,7 @@ async def initiate(configdir: str, rnsconfigdir:str, identitypath: str, logfile:
if message.stream_id == protocol.StreamDataMessage.STREAM_ID_STDERR: if message.stream_id == protocol.StreamDataMessage.STREAM_ID_STDERR:
if message.data and len(message.data) > 0: if message.data and len(message.data) > 0:
ttyRestorer.raw() ttyRestorer.raw()
# RNS.log(f"stdout: {message.data}", RNS.LOG_EXTREME) RNS.log(f"stdout: {message.data}", RNS.LOG_DEBUG)
os.write(2, message.data) os.write(2, message.data)
sys.stderr.flush() sys.stderr.flush()
if message.eof: if message.eof:
@ -392,7 +418,8 @@ async def initiate(configdir: str, rnsconfigdir:str, identitypath: str, logfile:
_link.teardown() _link.teardown()
return 200 return 200
except queue.Empty: processed = False except queue.Empty:
processed = False
if channel.is_ready_to_send(): if channel.is_ready_to_send():
def compress_adaptive(buf: bytes): def compress_adaptive(buf: bytes):
@ -406,7 +433,7 @@ async def initiate(configdir: str, rnsconfigdir:str, identitypath: str, logfile:
chunk_segment = None chunk_segment = None
chunk_segment = None chunk_segment = None
max_data_len = channel.mdu - protocol.StreamDataMessage.HEADER_LEN max_data_len = channel.mdu - protocol.StreamDataMessage.OVERHEAD
while chunk_len > 32 and comp_try < comp_tries: while chunk_len > 32 and comp_try < comp_tries:
chunk_segment_length = int(chunk_len/comp_try) chunk_segment_length = int(chunk_len/comp_try)
compressed_chunk = bz2.compress(buf[:chunk_segment_length]) compressed_chunk = bz2.compress(buf[:chunk_segment_length])
@ -414,7 +441,8 @@ async def initiate(configdir: str, rnsconfigdir:str, identitypath: str, logfile:
if compressed_length < max_data_len and compressed_length < chunk_segment_length: if compressed_length < max_data_len and compressed_length < chunk_segment_length:
comp_success = True comp_success = True
break break
else: comp_try += 1 else:
comp_try += 1
if comp_success: if comp_success:
diff = max_data_len - len(compressed_chunk) diff = max_data_len - len(compressed_chunk)
@ -443,7 +471,6 @@ async def initiate(configdir: str, rnsconfigdir:str, identitypath: str, logfile:
r, c, h, v = process.tty_get_winsize(0) r, c, h, v = process.tty_get_winsize(0)
channel.send(protocol.WindowSizeMessage(r, c, h, v)) channel.send(protocol.WindowSizeMessage(r, c, h, v))
processed = True processed = True
except RemoteExecutionError as e: except RemoteExecutionError as e:
print(e.msg) print(e.msg)
return 255 return 255

View file

@ -50,6 +50,7 @@ import RNS.Utilities.rnsh.exception as exception
import RNS.Utilities.rnsh.process as process import RNS.Utilities.rnsh.process as process
import RNS.Utilities.rnsh.retry as retry import RNS.Utilities.rnsh.retry as retry
import RNS.Utilities.rnsh.session as session import RNS.Utilities.rnsh.session as session
import re
import contextlib import contextlib
import pwd import pwd
@ -111,38 +112,31 @@ def _reload_allowed_file():
def compute_target_rns_loglevel(verbosity: int, quietness: int, base_level: int = RNS.LOG_INFO) -> int: def compute_target_rns_loglevel(verbosity: int, quietness: int, base_level: int = RNS.LOG_INFO) -> int:
try: try:
target = int(base_level) + int(verbosity) - int(quietness) target = int(base_level) + int(verbosity) - int(quietness)
if target < RNS.LOG_NONE: target = RNS.LOG_NONE if target < RNS.LOG_CRITICAL: target = RNS.LOG_CRITICAL
if target > RNS.LOG_EXTREME: target = RNS.LOG_EXTREME if target > RNS.LOG_DEBUG: target = RNS.LOG_DEBUG
return target return target
except Exception: return base_level except Exception: return base_level
async def listen(configdir, rnsconfigdir, command, identitypath=None, logfile=None, service_name=None, verbosity=0, quietness=0, async def listen(configdir, rnsconfigdir, command, identitypath=None, service_name=None, verbosity=0, quietness=0, allowed=None,
allowed=None, allowed_file=None, disable_auth=None, announce_period=900, no_remote_command=True, remote_cmd_as_args=False, allowed_file=None, disable_auth=None, announce_period=900, no_remote_command=True, remote_cmd_as_args=False,
loop: asyncio.AbstractEventLoop = None): loop: asyncio.AbstractEventLoop = None):
global _identity, _allow_all, _allowed_identity_hashes, _allowed_file, _allowed_file_identity_hashes global _identity, _allow_all, _allowed_identity_hashes, _allowed_file, _allowed_file_identity_hashes
global _reticulum, _cmd, _destination, _no_remote_command, _remote_cmd_as_args, _finished global _reticulum, _cmd, _destination, _no_remote_command, _remote_cmd_as_args, _finished
if not loop: loop = asyncio.get_running_loop() if not loop: loop = asyncio.get_running_loop()
if service_name is None or len(service_name) == 0: service_name = "default" if service_name is None or len(service_name) == 0:
service_name = "default"
RNS.log(f"Using service name {service_name}", RNS.LOG_INFO) RNS.log(f"Using service name {service_name}", RNS.LOG_INFO)
if logfile:
RNS.log(f"Logging to {logfile}", RNS.LOG_NOTICE)
logdest = RNS.LOG_FILE
RNS.logfile = logfile
else:
RNS.log(f"Logging to console", RNS.LOG_NOTICE)
logdest = RNS.LOG_STDOUT
# More -v should increase verbosity (higher RNS.loglevel); -q should decrease it # More -v should increase verbosity (higher RNS.loglevel); -q should decrease it
targetloglevel = compute_target_rns_loglevel(verbosity, quietness, RNS.LOG_INFO) targetloglevel = compute_target_rns_loglevel(verbosity, quietness, RNS.LOG_INFO)
_reticulum = RNS.Reticulum(configdir=rnsconfigdir, loglevel=targetloglevel) _reticulum = RNS.Reticulum(configdir=rnsconfigdir, loglevel=targetloglevel)
_identity = rnsh.prepare_identity(identity_path=identitypath, service_name=service_name, configdir=configdir) _identity = rnsh.prepare_identity(identitypath, service_name)
_destination = RNS.Destination(_identity, RNS.Destination.IN, RNS.Destination.SINGLE, rnsh.APP_NAME) _destination = RNS.Destination(_identity, RNS.Destination.IN, RNS.Destination.SINGLE, rnsh.APP_NAME)
RNS.log(f"rnsh listening for commands on {RNS.prettyhexrep(_destination.hash)}", RNS.LOG_NOTICE) RNS.log(f"rnsh listening for commands on {RNS.prettyhexrep(_destination.hash)}", RNS.LOG_NOTICE)
RNS.logdest = logdest
_cmd = command _cmd = command
if _cmd is None or len(_cmd) == 0: if _cmd is None or len(_cmd) == 0:
@ -160,8 +154,9 @@ async def listen(configdir, rnsconfigdir, command, identitypath=None, logfile=No
_no_remote_command = no_remote_command _no_remote_command = no_remote_command
session.ListenerSession.allow_remote_command = not no_remote_command session.ListenerSession.allow_remote_command = not no_remote_command
_remote_cmd_as_args = remote_cmd_as_args _remote_cmd_as_args = remote_cmd_as_args
if (_cmd is None or len(_cmd) == 0 or _cmd[0] is None or len(_cmd[0]) == 0) and (_no_remote_command or _remote_cmd_as_args): if (_cmd is None or len(_cmd) == 0 or _cmd[0] is None or len(_cmd[0]) == 0) \
raise Exception(f"Unable to look up shell for {os.getlogin()}, cannot proceed with -A or -C and no <program>.") and (_no_remote_command or _remote_cmd_as_args):
raise Exception(f"Unable to look up shell for {os.getlogin}, cannot proceed with -A or -C and no <program>.")
session.ListenerSession.default_command = _cmd session.ListenerSession.default_command = _cmd
session.ListenerSession.remote_cmd_as_args = _remote_cmd_as_args session.ListenerSession.remote_cmd_as_args = _remote_cmd_as_args

View file

@ -488,6 +488,8 @@ class CallbackSubprocess:
self._stdin_is_pipe = stdin_is_pipe self._stdin_is_pipe = stdin_is_pipe
self._stdout_is_pipe = stdout_is_pipe self._stdout_is_pipe = stdout_is_pipe
self._stderr_is_pipe = stderr_is_pipe self._stderr_is_pipe = stderr_is_pipe
self._at_line_start: bool = True
self._tty_line_buffer: bytearray = bytearray()
def _ensure_pipes_closed(self): def _ensure_pipes_closed(self):
stdin = self._child_stdin stdin = self._child_stdin
@ -622,6 +624,14 @@ class CallbackSubprocess:
""" """
RNS.log("start()", RNS.LOG_EXTREME) RNS.log("start()", RNS.LOG_EXTREME)
# # Using the parent environment seems to do some weird stuff, at least on macOS
# parentenv = os.environ.copy()
# env = {"HOME": parentenv["HOME"],
# "PATH": parentenv["PATH"],
# "TERM": self._term if self._term is not None else parentenv.get("TERM", "xterm"),
# "LANG": parentenv.get("LANG"),
# "SHELL": self._command[0]}
env = os.environ.copy() env = os.environ.copy()
for key in self._env: for key in self._env:
env[key] = self._env[key] env[key] = self._env[key]
@ -629,6 +639,12 @@ class CallbackSubprocess:
program = self._command[0] program = self._command[0]
assert isinstance(program, str) assert isinstance(program, str)
# match = re.search("^/bin/(.*sh)$", program)
# if match:
# self._command[0] = "-" + match.group(1)
# env["SHELL"] = program
# self._log.debug(f"set login shell {self._command}")
self._pid, \ self._pid, \
self._child_stdin, \ self._child_stdin, \
self._child_stdout, \ self._child_stdout, \

View file

@ -32,7 +32,6 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE. # SOFTWARE.
import RNS
import asyncio import asyncio
import threading import threading
import time import time
@ -89,7 +88,8 @@ class RetryThread(AbstractContextManager):
self._thread = threading.Thread(name=name, target=self._thread_run, daemon=True) self._thread = threading.Thread(name=name, target=self._thread_run, daemon=True)
self._thread.start() self._thread.start()
def is_alive(self): return self._thread.is_alive() def is_alive(self):
return self._thread.is_alive()
def close(self, loop: asyncio.AbstractEventLoop = None) -> asyncio.Future: def close(self, loop: asyncio.AbstractEventLoop = None) -> asyncio.Future:
RNS.log("Stopping timer thread", RNS.LOG_DEBUG) RNS.log("Stopping timer thread", RNS.LOG_DEBUG)
@ -102,12 +102,17 @@ class RetryThread(AbstractContextManager):
return self._finished return self._finished
def wait(self, timeout: float = None): def wait(self, timeout: float = None):
if timeout: timeout = timeout + time.time() if timeout:
timeout = timeout + time.time()
while timeout is None or time.time() < timeout: while timeout is None or time.time() < timeout:
with self._lock: task_count = len(self._statuses) with self._lock:
if task_count == 0: return task_count = len(self._statuses)
if task_count == 0:
return
time.sleep(0.1) time.sleep(0.1)
def _thread_run(self): def _thread_run(self):
while self._run and self._finished is None: while self._run and self._finished is None:
time.sleep(self._loop_period) time.sleep(self._loop_period)

View file

@ -37,6 +37,7 @@
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
import base64
import re import re
import os import os
@ -56,54 +57,43 @@ loop: asyncio.AbstractEventLoop | None = None
def _sanitize_service_name(service_name:str) -> str: return re.sub(r'\W+', '', service_name) def _sanitize_service_name(service_name:str) -> str: return re.sub(r'\W+', '', service_name)
def prepare_identity(identity_path=None, service_name=None, configdir=None): def prepare_identity(identity_path, service_name: str = None) -> tuple[RNS.Identity]:
service_name = _sanitize_service_name(service_name or "") service_name = _sanitize_service_name(service_name or "")
if not identity_path: if identity_path is None:
identity_path = f"{configdir}/identity" + (f".{service_name}" if service_name and len(service_name) > 0 else "") identity_path = RNS.Reticulum.identitypath + "/" + APP_NAME + \
(f".{service_name}" if service_name and len(service_name) > 0 else "")
identity = None identity = None
if os.path.isfile(identity_path): if os.path.isfile(identity_path):
RNS.log(f"Loading identity from {identity_path}", RNS.LOG_VERBOSE)
identity = RNS.Identity.from_file(identity_path) identity = RNS.Identity.from_file(identity_path)
if identity is None: if identity is None:
RNS.log(f"Creating identity {identity_path}", RNS.LOG_NOTICE) RNS.log("No valid saved identity found, creating new...", RNS.LOG_INFO)
identity = RNS.Identity() identity = RNS.Identity()
identity.to_file(identity_path) identity.to_file(identity_path)
return identity return identity
def print_identity(configdir, rnsconfigdir, identitypath, service_name, include_destination: bool): def print_identity(configdir, identitypath, service_name, include_destination: bool):
reticulum = RNS.Reticulum(configdir=rnsconfigdir, loglevel=RNS.LOG_INFO) reticulum = RNS.Reticulum(configdir=configdir, loglevel=RNS.LOG_INFO)
if service_name and len(service_name) > 0: print(f"Using service name \"{service_name}\"") if service_name and len(service_name) > 0:
identity = prepare_identity(identity_path=identitypath, service_name=service_name, configdir=configdir) print(f"Using service name \"{service_name}\"")
identity = prepare_identity(identitypath, service_name)
destination = RNS.Destination(identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME) destination = RNS.Destination(identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME)
print("Identity : " + str(identity)) print("Identity : " + str(identity))
if include_destination: print("Listening on : " + RNS.prettyhexrep(destination.hash)) if include_destination:
print("Listening on : " + RNS.prettyhexrep(destination.hash))
exit(0) exit(0)
verbose_set = False verbose_set = False
def ensure_config_directory(configdir=None): def ensure_config_directory():
if configdir: if os.path.isdir(os.path.expanduser("~/.config/rnsh")): return os.path.expanduser("~/.config/rnsh")
if os.path.isdir(os.path.expanduser(configdir)): return os.path.expanduser(configdir)
else:
try:
RNS.log(f"Creating configuration directory: {os.path.expanduser(configdir)}")
os.makedirs(os.path.expanduser(configdir))
return os.path.expanduser(configdir)
except Exception as e:
RNS.log(f"Could not get or create rnsh configuration directory, aborting", RNS.LOG_CRITICAL)
os._exit(1)
elif os.path.isdir(os.path.expanduser("~/.config/rnsh")): return os.path.expanduser("~/.config/rnsh")
elif os.path.isdir(os.path.expanduser("~/.rnsh")): return os.path.expanduser("~/.rnsh") elif os.path.isdir(os.path.expanduser("~/.rnsh")): return os.path.expanduser("~/.rnsh")
else: else:
try: try:
RNS.log(f"Creating configuration directory: {os.path.expanduser('~/.rnsh')}")
os.makedirs(os.path.expanduser("~/.rnsh")) os.makedirs(os.path.expanduser("~/.rnsh"))
return os.path.expanduser("~/.rnsh") return os.path.expanduser("~/.rnsh")
@ -116,29 +106,25 @@ async def _rnsh_cli_main():
global verbose_set global verbose_set
args, parser = parse_arguments() args, parser = parse_arguments()
verbose_set = args.verbose > 0 verbose_set = args.verbose > 0
configdir = ensure_config_directory(args.config)
if not configdir: configdir = ensure_config_directory()
RNS.log(f"Could not resolve rnsh configuration directory", RNS.LOG_CRITICAL)
os._exit(1)
if args.print_identity: if args.print_identity:
print_identity(configdir, args.rnsconfig, args.identity, args.service, args.listen) print_identity(args.config, args.identity, args.service, args.listen)
return 0 return 0
logfile = f"{configdir}/logfile"
if args.listen: if args.listen:
allowed_file = None allowed_file = None
dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2 dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2
if os.path.isfile(os.path.expanduser(f"{configdir}/allowed_identities")): if os.path.isfile(os.path.expanduser("~/.config/rnsh/allowed_identities")):
allowed_file = os.path.expanduser(f"{configdir}/allowed_identities") allowed_file = os.path.expanduser("~/.config/rnsh/allowed_identities")
elif os.path.isfile(os.path.expanduser("~/.rnsh/allowed_identities")):
allowed_file = os.path.expanduser("~/.rnsh/allowed_identities")
await listener.listen(configdir=configdir, await listener.listen(configdir=configdir,
rnsconfigdir=args.rnsconfig, rnsconfigdir=args.config,
command=args.command, command=args.command,
identitypath=args.identity, identitypath=args.identity,
logfile=logfile,
service_name=args.service, service_name=args.service,
verbosity=args.verbose, verbosity=args.verbose,
quietness=args.quiet, quietness=args.quiet,
@ -152,9 +138,8 @@ async def _rnsh_cli_main():
if args.destination is not None: if args.destination is not None:
return_code = await initiator.initiate(configdir=configdir, return_code = await initiator.initiate(configdir=configdir,
rnsconfigdir=args.rnsconfig, rnsconfigdir=args.config,
identitypath=args.identity, identitypath=args.identity,
logfile=f"{logfile}.initiator",
verbosity=args.verbose, verbosity=args.verbose,
quietness=args.quiet, quietness=args.quiet,
noid=args.no_id, noid=args.no_id,
@ -162,7 +147,6 @@ async def _rnsh_cli_main():
timeout=args.timeout, timeout=args.timeout,
command=args.command command=args.command
) )
RNS.log(f"Sesssion with <{args.destination}> ended", RNS.LOG_INFO)
return return_code if args.mirror else 0 return return_code if args.mirror else 0
else: else:
print("") print("")
@ -186,4 +170,5 @@ def main():
if verbose_set and exc: raise exc if verbose_set and exc: raise exc
sys.exit(return_code if return_code is not None else 255) sys.exit(return_code if return_code is not None else 255)
if __name__ == "__main__": main() if __name__ == "__main__": main()

View file

@ -43,6 +43,7 @@ import RNS.Utilities.rnsh.protocol as protocol
import enum import enum
from typing import TypeVar, Generic, Callable, List from typing import TypeVar, Generic, Callable, List
from abc import abstractmethod, ABC from abc import abstractmethod, ABC
from multiprocessing import Manager
import os import os
import bz2 import bz2
import RNS import RNS
@ -102,8 +103,6 @@ class ListenerSession:
self.outlet.set_link_closed_callback(self._link_closed) self.outlet.set_link_closed_callback(self._link_closed)
self.loop = loop self.loop = loop
self.state: LSState = None self.state: LSState = None
self.terminated = False
self.authenticated = False
self.remote_identity = None self.remote_identity = None
self.term: str | None = None self.term: str | None = None
self.stdin_is_pipe: bool = False self.stdin_is_pipe: bool = False
@ -123,10 +122,8 @@ class ListenerSession:
self.return_code_sent = False self.return_code_sent = False
self.process: process.CallbackSubprocess | None = None self.process: process.CallbackSubprocess | None = None
if not self.allow_all: self._set_state(LSState.LSSTATE_WAIT_IDENT) if self.allow_all: self._set_state(LSState.LSSTATE_WAIT_VERS)
else: else: self._set_state(LSState.LSSTATE_WAIT_IDENT)
self._set_state(LSState.LSSTATE_WAIT_VERS)
self.authenticated = True
self.sessions.append(self) self.sessions.append(self)
protocol.register_message_types(self.channel) protocol.register_message_types(self.channel)
@ -150,27 +147,33 @@ class ListenerSession:
self.loop.call_soon_threadsafe(call_inner) self.loop.call_soon_threadsafe(call_inner)
def send(self, message: RNS.MessageBase): self.channel.send(message) def send(self, message: RNS.MessageBase):
def _protocol_error(self, name: str): self.terminate(f"Protocol error ({name})") self.channel.send(message)
def _protocol_timeout_error(self, name: str): self.terminate(f"Protocol timeout error: {name}")
def _protocol_error(self, name: str):
self.terminate(f"Protocol error ({name})")
def _protocol_timeout_error(self, name: str):
self.terminate(f"Protocol timeout error: {name}")
def terminate(self, error: str = None): def terminate(self, error: str = None):
self.terminated = True
self.state = LSState.LSSTATE_ERROR
with contextlib.suppress(Exception): with contextlib.suppress(Exception):
RNS.log("Terminating session" + (f": {error}" if error else ""), RNS.LOG_DEBUG) RNS.log("Terminating session" + (f": {error}" if error else ""), RNS.LOG_DEBUG)
if error and self.state != LSState.LSSTATE_TEARDOWN: if error and self.state != LSState.LSSTATE_TEARDOWN:
with contextlib.suppress(Exception): with contextlib.suppress(Exception):
self.send(protocol.ErrorMessage(error, True)) self.send(protocol.ErrorMessage(error, True))
self.state = LSState.LSSTATE_ERROR
self._terminate_process() self._terminate_process()
self._call(self._prune, max(self.outlet.rtt * 3, process.CallbackSubprocess.PROCESS_PIPE_TIME+5)) self._call(self._prune, max(self.outlet.rtt * 3, process.CallbackSubprocess.PROCESS_PIPE_TIME+5))
def _prune(self): def _prune(self):
self.state = LSState.LSSTATE_TEARDOWN self.state = LSState.LSSTATE_TEARDOWN
RNS.log("Pruning session", RNS.LOG_DEBUG) RNS.log("Pruning session", RNS.LOG_DEBUG)
with contextlib.suppress(ValueError): self.sessions.remove(self) with contextlib.suppress(ValueError):
with contextlib.suppress(Exception): self.outlet.teardown() self.sessions.remove(self)
with contextlib.suppress(Exception):
self.outlet.teardown()
def _check_protocol_timeout(self, fail_condition: Callable[[], bool], name: str): def _check_protocol_timeout(self, fail_condition: Callable[[], bool], name: str):
timeout = True timeout = True
@ -180,6 +183,7 @@ class ListenerSession:
def _link_closed(self, outlet: LSOutletBase): def _link_closed(self, outlet: LSOutletBase):
outlet.unset_link_closed_callback() outlet.unset_link_closed_callback()
if outlet != self.outlet: if outlet != self.outlet:
RNS.log("Link closed received from incorrect outlet", RNS.LOG_DEBUG) RNS.log("Link closed received from incorrect outlet", RNS.LOG_DEBUG)
return return
@ -192,22 +196,16 @@ class ListenerSession:
RNS.log("Identity received from incorrect outlet", RNS.LOG_DEBUG) RNS.log("Identity received from incorrect outlet", RNS.LOG_DEBUG)
return return
RNS.log(f"Initiator identified {identity} on link {outlet}", RNS.LOG_INFO) RNS.log(f"initiator_identified {identity} on link {outlet}", RNS.LOG_INFO)
if self.state not in [LSState.LSSTATE_WAIT_IDENT, LSState.LSSTATE_WAIT_VERS]: if self.state not in [LSState.LSSTATE_WAIT_IDENT, LSState.LSSTATE_WAIT_VERS]:
self._protocol_error(LSState.LSSTATE_WAIT_IDENT.name) self._protocol_error(LSState.LSSTATE_WAIT_IDENT.name)
return
if self.allow_all or identity.hash in self.allowed_identity_hashes or identity.hash in self.allowed_file_identity_hashes: if not self.allow_all and identity.hash not in self.allowed_identity_hashes and identity.hash not in self.allowed_file_identity_hashes:
RNS.log(f"Identity {identity} authenticated") self.terminate("Identity is not allowed.")
self.authenticated = True
self.remote_identity = identity self.remote_identity = identity
self._set_state(LSState.LSSTATE_WAIT_VERS) self._set_state(LSState.LSSTATE_WAIT_VERS)
else:
self.authenticated = False
self.terminate("Identity not allowed")
RNS.log(f"Identity {identity} not allowed")
@classmethod @classmethod
async def pump_all(cls) -> True: async def pump_all(cls) -> True:
processed_any = False processed_any = False
@ -235,7 +233,7 @@ class ListenerSession:
chunk_segment = None chunk_segment = None
chunk_segment = None chunk_segment = None
max_data_len = self.channel.mdu - protocol.StreamDataMessage.HEADER_LEN max_data_len = self.channel.mdu - protocol.StreamDataMessage.OVERHEAD
while chunk_len > 32 and comp_try < comp_tries: while chunk_len > 32 and comp_try < comp_tries:
chunk_segment_length = int(chunk_len/comp_try) chunk_segment_length = int(chunk_len/comp_try)
compressed_chunk = bz2.compress(buf[:chunk_segment_length]) compressed_chunk = bz2.compress(buf[:chunk_segment_length])
@ -243,7 +241,8 @@ class ListenerSession:
if compressed_length < max_data_len and compressed_length < chunk_segment_length: if compressed_length < max_data_len and compressed_length < chunk_segment_length:
comp_success = True comp_success = True
break break
else: comp_try += 1 else:
comp_try += 1
if comp_success: if comp_success:
diff = max_data_len - len(compressed_chunk) diff = max_data_len - len(compressed_chunk)
@ -256,8 +255,10 @@ class ListenerSession:
return comp_success, processed_length, chunk return comp_success, processed_length, chunk
try: try:
if self.state != LSState.LSSTATE_RUNNING: return False if self.state != LSState.LSSTATE_RUNNING:
elif not self.channel.is_ready_to_send(): return False return False
elif not self.channel.is_ready_to_send():
return False
elif len(self.stderr_buf) > 0: elif len(self.stderr_buf) > 0:
comp_success, processed_length, data = compress_adaptive(self.stderr_buf) comp_success, processed_length, data = compress_adaptive(self.stderr_buf)
self.stderr_buf = self.stderr_buf[processed_length:] self.stderr_buf = self.stderr_buf[processed_length:]
@ -266,7 +267,8 @@ class ListenerSession:
msg = protocol.StreamDataMessage(protocol.StreamDataMessage.STREAM_ID_STDERR, msg = protocol.StreamDataMessage(protocol.StreamDataMessage.STREAM_ID_STDERR,
data, send_eof, comp_success) data, send_eof, comp_success)
self.send(msg) self.send(msg)
if send_eof: self.stderr_eof_sent = True if send_eof:
self.stderr_eof_sent = True
return True return True
elif len(self.stdout_buf) > 0: elif len(self.stdout_buf) > 0:
comp_success, processed_length, data = compress_adaptive(self.stdout_buf) comp_success, processed_length, data = compress_adaptive(self.stdout_buf)
@ -276,7 +278,8 @@ class ListenerSession:
msg = protocol.StreamDataMessage(protocol.StreamDataMessage.STREAM_ID_STDOUT, msg = protocol.StreamDataMessage(protocol.StreamDataMessage.STREAM_ID_STDOUT,
data, send_eof, comp_success) data, send_eof, comp_success)
self.send(msg) self.send(msg)
if send_eof: self.stdout_eof_sent = True if send_eof:
self.stdout_eof_sent = True
return True return True
elif self.return_code is not None and not self.return_code_sent: elif self.return_code is not None and not self.return_code_sent:
msg = protocol.CommandExitedMessage(self.return_code) msg = protocol.CommandExitedMessage(self.return_code)
@ -292,33 +295,22 @@ class ListenerSession:
def _terminate_process(self): def _terminate_process(self):
with contextlib.suppress(Exception): with contextlib.suppress(Exception):
if self.process and self.process.running: self.process.terminate() if self.process and self.process.running:
self.process.terminate()
def _start_cmd(self, cmdline: [str], pipe_stdin: bool, pipe_stdout: bool, pipe_stderr: bool, tcflags: [any], def _start_cmd(self, cmdline: [str], pipe_stdin: bool, pipe_stdout: bool, pipe_stderr: bool, tcflags: [any],
term: str | None, rows: int, cols: int, hpix: int, vpix: int): term: str | None, rows: int, cols: int, hpix: int, vpix: int):
if self.terminated: self.cmdline = self.default_command
RNS.log(f"Attempt to start command on terminated session: {cmdline}", RNS.LOG_WARNING)
return
if not self.authenticated:
RNS.log(f"Attempt to start command on unauthenticated session: {cmdline}", RNS.LOG_WARNING)
self.terminate("Invalid state")
return
if self.state != LSState.LSSTATE_RUNNING:
RNS.log(f"Attempt to start command on session not in running state: {cmdline}", RNS.LOG_WARNING)
self.terminate("Invalid state")
return
self.cmdline = []
self.cmdline.extend(self.default_command)
if not self.allow_remote_command and cmdline and len(cmdline) > 0: if not self.allow_remote_command and cmdline and len(cmdline) > 0:
self.terminate("Remote command line not allowed by listener") self.terminate("Remote command line not allowed by listener")
return return
if self.remote_cmd_as_args and cmdline and len(cmdline) > 0: self.cmdline.extend(cmdline) if self.remote_cmd_as_args and cmdline and len(cmdline) > 0:
elif cmdline and len(cmdline) > 0: self.cmdline = cmdline self.cmdline.extend(cmdline)
elif cmdline and len(cmdline) > 0:
self.cmdline = cmdline
self.stdin_is_pipe = pipe_stdin self.stdin_is_pipe = pipe_stdin
self.stdout_is_pipe = pipe_stdout self.stdout_is_pipe = pipe_stdout
@ -326,11 +318,13 @@ class ListenerSession:
self.tcflags = tcflags self.tcflags = tcflags
self.term = term self.term = term
def stdout(data: bytes): self.stdout_buf.extend(data) def stdout(data: bytes):
def stderr(data: bytes): self.stderr_buf.extend(data) self.stdout_buf.extend(data)
def stderr(data: bytes):
self.stderr_buf.extend(data)
try: try:
RNS.log(f"Remote {self.remote_identity} executing: {self.cmdline}", RNS.LOG_VERBOSE)
self.process = process.CallbackSubprocess(argv=self.cmdline, self.process = process.CallbackSubprocess(argv=self.cmdline,
env={"TERM": self.term or os.environ.get("TERM") or "xterm", env={"TERM": self.term or os.environ.get("TERM") or "xterm",
"RNS_REMOTE_IDENTITY": (RNS.prettyhexrep(self.remote_identity.hash) "RNS_REMOTE_IDENTITY": (RNS.prettyhexrep(self.remote_identity.hash)
@ -353,28 +347,22 @@ class ListenerSession:
self.cols = cols self.cols = cols
self.hpix = hpix self.hpix = hpix
self.vpix = vpix self.vpix = vpix
with contextlib.suppress(Exception): self.process.set_winsize(rows, cols, hpix, vpix) with contextlib.suppress(Exception):
self.process.set_winsize(rows, cols, hpix, vpix)
def _received_stdin(self, data: bytes, eof: bool): def _received_stdin(self, data: bytes, eof: bool):
if data and len(data) > 0: self.process.write(data) if data and len(data) > 0:
if eof: self.process.close_stdin() self.process.write(data)
if eof:
self.process.close_stdin()
def _handle_message(self, message: RNS.MessageBase): def _handle_message(self, message: RNS.MessageBase):
if self.terminated:
RNS.log(f"Received packet, but session was terminated", RNS.LOG_ERROR)
return
if self.state > LSState.LSSTATE_RUNNING:
RNS.log(f"Received packet, but in state {self.state.name}", RNS.LOG_ERROR)
return
if self.state == LSState.LSSTATE_WAIT_IDENT: if self.state == LSState.LSSTATE_WAIT_IDENT:
# Ignore any messages until the initiator has identified to avoid race conditions # Ignore any messages until the initiator has identified to avoid race conditions
# between identity announcement and early protocol messages. # between identity announcement and early protocol messages.
RNS.log("Ignoring message while waiting for identification", RNS.LOG_DEBUG) RNS.log("Ignoring message while waiting for identification", RNS.LOG_DEBUG)
return return
elif self.state == LSState.LSSTATE_WAIT_VERS: if self.state == LSState.LSSTATE_WAIT_VERS:
if not self.authenticated:
self.terminate("Invalid state")
return
if not isinstance(message, protocol.VersionInfoMessage): if not isinstance(message, protocol.VersionInfoMessage):
self._protocol_error(self.state.name) self._protocol_error(self.state.name)
return return
@ -386,9 +374,6 @@ class ListenerSession:
self._set_state(LSState.LSSTATE_WAIT_CMD) self._set_state(LSState.LSSTATE_WAIT_CMD)
return return
elif self.state == LSState.LSSTATE_WAIT_CMD: elif self.state == LSState.LSSTATE_WAIT_CMD:
if not self.authenticated:
self.terminate("Invalid state")
return
if not isinstance(message, protocol.ExecuteCommandMesssage): if not isinstance(message, protocol.ExecuteCommandMesssage):
return self._protocol_error(self.state.name) return self._protocol_error(self.state.name)
RNS.log(f"Execute command message on link {self.outlet}: {message.cmdline}", RNS.LOG_VERBOSE) RNS.log(f"Execute command message on link {self.outlet}: {message.cmdline}", RNS.LOG_VERBOSE)
@ -397,9 +382,6 @@ class ListenerSession:
message.tcflags, message.term, message.rows, message.cols, message.hpix, message.vpix) message.tcflags, message.term, message.rows, message.cols, message.hpix, message.vpix)
return return
elif self.state == LSState.LSSTATE_RUNNING: elif self.state == LSState.LSSTATE_RUNNING:
if not self.authenticated:
self.terminate("Invalid state")
return
if isinstance(message, protocol.WindowSizeMessage): if isinstance(message, protocol.WindowSizeMessage):
self._set_window_size(message.rows, message.cols, message.hpix, message.vpix) self._set_window_size(message.rows, message.cols, message.hpix, message.vpix)
elif isinstance(message, protocol.StreamDataMessage): elif isinstance(message, protocol.StreamDataMessage):
@ -412,6 +394,9 @@ class ListenerSession:
# echo noop only on listener--used for keepalive/connectivity check # echo noop only on listener--used for keepalive/connectivity check
self.send(message) self.send(message)
return return
elif self.state in [LSState.LSSTATE_ERROR, LSState.LSSTATE_TEARDOWN]:
RNS.log(f"Received packet, but in state {self.state.name}", RNS.LOG_ERROR)
return
else: else:
self._protocol_error("unexpected message") self._protocol_error("unexpected message")
return return
@ -420,20 +405,29 @@ class ListenerSession:
class RNSOutlet(LSOutletBase): class RNSOutlet(LSOutletBase):
def set_initiator_identified_callback(self, cb: Callable[[LSOutletBase, _TIdentity], None]): def set_initiator_identified_callback(self, cb: Callable[[LSOutletBase, _TIdentity], None]):
def inner_cb(link, identity: _TIdentity): cb(self, identity) def inner_cb(link, identity: _TIdentity):
cb(self, identity)
self.link.set_remote_identified_callback(inner_cb) self.link.set_remote_identified_callback(inner_cb)
def set_link_closed_callback(self, cb: Callable[[LSOutletBase], None]): def set_link_closed_callback(self, cb: Callable[[LSOutletBase], None]):
def inner_cb(link): cb(self) def inner_cb(link):
cb(self)
self.link.set_link_closed_callback(inner_cb) self.link.set_link_closed_callback(inner_cb)
def unset_link_closed_callback(self): self.link.set_link_closed_callback(None) def unset_link_closed_callback(self):
def teardown(self): self.link.teardown() self.link.set_link_closed_callback(None)
def teardown(self):
self.link.teardown()
@property @property
def rtt(self) -> float: return self.link.rtt def rtt(self) -> float:
return self.link.rtt
def __str__(self): return f"Outlet on {self.link}" def __str__(self):
return f"Outlet RNS Link {self.link}"
def __init__(self, link: RNS.Link): def __init__(self, link: RNS.Link):
self.link = link self.link = link
@ -441,5 +435,7 @@ class RNSOutlet(LSOutletBase):
@staticmethod @staticmethod
def get_outlet(link: RNS.Link): def get_outlet(link: RNS.Link):
if hasattr(link, "lsoutlet"): return link.lsoutlet if hasattr(link, "lsoutlet"):
return link.lsoutlet
return RNSOutlet(link) return RNSOutlet(link)

View file

@ -2,7 +2,7 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -63,10 +63,9 @@ request_concluded = False
first_remote_req = True first_remote_req = True
remote_destination = None remote_destination = None
remote_link = None remote_link = None
def get_remote_status(destination_hash, include_lstats, include_profiling, identity, no_output=False, timeout=RNS.Transport.PATH_REQUEST_TIMEOUT): def get_remote_status(destination_hash, include_lstats, identity, no_output=False, timeout=RNS.Transport.PATH_REQUEST_TIMEOUT):
global request_result, request_concluded, first_remote_req, remote_destination, remote_link global request_result, request_concluded, first_remote_req, remote_destination, remote_link
link_count = None link_count = None
profiling_results = None
if not RNS.Transport.has_path(destination_hash): if not RNS.Transport.has_path(destination_hash):
if not no_output: if not no_output:
@ -115,10 +114,7 @@ def get_remote_status(destination_hash, include_lstats, include_profiling, ident
if len(response) > 1: link_count = response[1] if len(response) > 1: link_count = response[1]
else: link_count = None else: link_count = None
if len(response) > 2: profiling_results = response[2] request_result = (status, link_count)
else: profiling_results = None
request_result = (status, link_count, profiling_results)
request_concluded = True request_concluded = True
@ -129,7 +125,7 @@ def get_remote_status(destination_hash, include_lstats, include_profiling, ident
print("Sending request...", end=" ") print("Sending request...", end=" ")
sys.stdout.flush() sys.stdout.flush()
link.identify(identity) link.identify(identity)
link.request("/status", data = [include_lstats, include_profiling], response_callback = got_response, failed_callback = request_failed) link.request("/status", data = [include_lstats], response_callback = got_response, failed_callback = request_failed)
first_remote_req = False first_remote_req = False
if not remote_link and not no_output: if not remote_link and not no_output:
@ -156,11 +152,9 @@ def get_remote_status(destination_hash, include_lstats, include_profiling, ident
return request_result return request_result
def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=False, astats=False, pstats=False, lstats=False, def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=False, astats=False, pstats=False, lstats=False, sorting=None,
sorting=None, sort_reverse=False, remote=None, management_identity=None, must_exit=True, rns_instance=None, sort_reverse=False, remote=None, management_identity=None, remote_timeout=RNS.Transport.PATH_REQUEST_TIMEOUT, must_exit=True,
traffic_totals=False, discovered_interfaces=False, config_entries=False, burst_filter=False, blocked_ips=False, rns_instance=None, traffic_totals=False, discovered_interfaces=False, config_entries=False, burst_filter=False):
queue_stats=False, pps=False, profiling=False, remote_timeout=RNS.Transport.PATH_REQUEST_TIMEOUT,
attach=None, detach=None, reload=None, show_stale=False, show_unknown=False, fstats=False, tstats=False):
if remote: require_shared = False if remote: require_shared = False
else: require_shared = True else: require_shared = True
@ -171,48 +165,14 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
must_exit = False must_exit = False
else: else:
reticulum = RNS.Reticulum(configdir=configdir, loglevel=3+verbosity, require_shared_instance=require_shared) reticulum = RNS.Reticulum(configdir=configdir, loglevel=3+verbosity, require_shared_instance=require_shared)
reticulum.wants_destination_type("rnstransport.remote.management")
except Exception as e: except Exception as e:
print("No shared RNS instance available to get status from") print("No shared RNS instance available to get status from")
if must_exit: exit(1) if must_exit: exit(1)
else: return else: return
if attach:
result = reticulum.attach_interface(attach)
if result == True: print(f"Interface {attach} was attached")
elif result == False: print(f"Could not attach interface {attach}")
elif result == None: print(f"The interface {attach} does not exist")
else: print(f"Unknown error while attaching interface {attach}")
if result == True: exit(0)
else: exit(1)
if detach:
result = reticulum.detach_interface(detach)
if result == True: print(f"Interface {detach} was detached")
elif result == False: print(f"Could not detach interface {detach}")
elif result == None: print(f"The interface {detach} does not exist")
else: print(f"Unknown error while detaching interface {detach}")
if result == True: exit(0)
else: exit(1)
if reload:
result = reticulum.reload_interface(reload)
if result == True: print(f"Interface {reload} was reloaded")
elif result == False: print(f"Could not reload interface {reload}")
elif result == None: print(f"The interface {reload} does not exist")
else: print(f"Unknown error while reloading interface {reload}")
if result == True: exit(0)
else: exit(1)
link_count = None link_count = None
active_link_count = None
stats = None stats = None
profiling_results = None
details = False details = False
if config_entries: if config_entries:
@ -244,17 +204,12 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
name = i["name"] name = i["name"]
if_type = i["type"] if_type = i["type"]
status = i["status"] status = i["status"]
has_impl_info = "impl_name" in i and "version" in i and i["impl_name"] and i["version"]
impl_str = f"{i['impl_name']} {i['version']}" if has_impl_info else "Unknown"
if status == "available": status_display = "Available" if status == "available": status_display = "Available"
elif status == "unknown": status_display = "Unknown" elif status == "unknown": status_display = "Unknown"
elif status == "stale": status_display = "Stale" elif status == "stale": status_display = "Stale"
else: status_display = status else: status_display = status
if status == "stale" and not show_stale: continue
if not has_impl_info and not show_unknown: continue
now = time.time() now = time.time()
dago = now-i["discovered"] dago = now-i["discovered"]
hago = now-i["last_heard"] hago = now-i["last_heard"]
@ -262,14 +217,6 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
last_heard_display = f"{RNS.prettytime(hago, compact=True)} ago" last_heard_display = f"{RNS.prettytime(hago, compact=True)} ago"
transport_str = "Enabled" if i["transport"] else "Disabled" transport_str = "Enabled" if i["transport"] else "Disabled"
has_cost_info = "dst_cost" in i and "pr_cost" in i
dst_cost = None; pr_cost = None
if not has_cost_info: cost_str = "Unknown"
else:
dst_cost = i["dst_cost"]
pr_cost = i["pr_cost"]
cost_str = f"Dst {RNS.prettycost(dst_cost)}, PR {RNS.prettycost(pr_cost)}"
if i["latitude"] is not None and i["longitude"] is not None: if i["latitude"] is not None and i["longitude"] is not None:
lat = round(i["latitude"], 4) lat = round(i["latitude"], 4)
lon = round(i["longitude"], 4) lon = round(i["longitude"], 4)
@ -278,26 +225,20 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
location = f"{lat}, {lon}{height}" location = f"{lat}, {lon}{height}"
else: location = "Unknown" else: location = "Unknown"
lxmf_addr = None
transport_id = None transport_id = None
network = None network = None
if "operator_lxmf_address" in i: lxmf_addr = i['operator_lxmf_address']
if "transport_id" in i: transport_id = i["transport_id"] if "transport_id" in i: transport_id = i["transport_id"]
if "transport_id" in i and "network_id" in i and i["transport_id"] != i["network_id"]: if "transport_id" in i and "network_id" in i and i["transport_id"] != i["network_id"]:
network = i["network_id"] network = i["network_id"]
if idx > 0: print("\n"+"="*47+"\n") if idx > 0: print("\n"+"="*32+"\n")
if network: print(f"Network ID : {network}") if network: print(f"Network ID : {network}")
if transport_id: print(f"Transport ID : {transport_id}") if transport_id: print(f"Transport ID : {transport_id}")
if lxmf_addr: print(f"LXMF address : {lxmf_addr}")
print(f"Name : {name}") print(f"Name : {name}")
print(f"Type : {if_type}") print(f"Type : {if_type}")
print(f"Stack : {impl_str}")
print(f"Status : {status_display}") print(f"Status : {status_display}")
print(f"Transport : {transport_str}") print(f"Transport : {transport_str}")
print(f"Stamp Value : {RNS.prettycost(i['value'])}")
print(f"Costs : {cost_str}") if has_cost_info else None
print(f"Distance : {i['hops']} hop{'' if i['hops'] == 1 else 's'}") print(f"Distance : {i['hops']} hop{'' if i['hops'] == 1 else 's'}")
print(f"Discovered : {discovered_display}") print(f"Discovered : {discovered_display}")
print(f"Last Heard : {last_heard_display}") print(f"Last Heard : {last_heard_display}")
@ -311,22 +252,22 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
if "reachable_on" in i: print(f"Address : {i['reachable_on']}") if "reachable_on" in i: print(f"Address : {i['reachable_on']}")
if "port" in i: print(f"Port : {i['port']}") if "port" in i: print(f"Port : {i['port']}")
print(f"Stamp Value : {i['value']}")
print(f"\nConfiguration Entry:") print(f"\nConfiguration Entry:")
config_lines = i["config_entry"].split('\n') config_lines = i["config_entry"].split('\n')
for line in config_lines: print(f" {line}") for line in config_lines: print(f" {line}")
except Exception as e: pass except Exception as e:
pass
else: else:
print(f"{'Name':<25} {'Type':<10} {'Status':<10} {'Last Heard':<12} {'Value':<7} {'Running':<16} {'Location':<15}") print(f"{'Name':<25} {'Type':<12} {'Status':<12} {'Last Heard':<12} {'Value':<8} {'Location':<15}")
print("-" * 110) print("-" * 89)
for i in filtered_ifs: for i in filtered_ifs:
try: try:
name = i["name"][:24] + "…" if len(i["name"]) > 24 else i["name"] name = i["name"][:24] + "…" if len(i["name"]) > 24 else i["name"]
has_impl_info = "impl_name" in i and "version" in i and i["impl_name"] and i["version"]
impl_str = f"{i['impl_name']} {i['version']}" if has_impl_info else "Unknown"
if len(impl_str) > 16: impl_str = impl_str[:15]+"…"
if_type = i["type"].replace("Interface", "") if_type = i["type"].replace("Interface", "")
@ -336,9 +277,6 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
elif status == "stale": status_display = "× Stale" elif status == "stale": status_display = "× Stale"
else: status_display = status else: status_display = status
if status == "stale" and not show_stale: continue
if not has_impl_info and not show_unknown: continue
now = time.time() now = time.time()
last_heard = i["last_heard"] last_heard = i["last_heard"]
diff = now - last_heard diff = now - last_heard
@ -354,7 +292,7 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
days = int(diff / 86400) days = int(diff / 86400)
last_heard_display = f"{days}d ago" last_heard_display = f"{days}d ago"
value = RNS.prettycost(i["value"]) value = str(i["value"])
if i["latitude"] is not None and i["longitude"] is not None: if i["latitude"] is not None and i["longitude"] is not None:
lat = round(i["latitude"], 4) lat = round(i["latitude"], 4)
@ -362,7 +300,7 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
location = f"{lat}, {lon}" location = f"{lat}, {lon}"
else: location = "N/A" else: location = "N/A"
print(f"{name:<25} {if_type:<10} {status_display:<12} {last_heard_display:<12} {value:<7} {impl_str:<16} {location:<15}") print(f"{name:<25} {if_type:<12} {status_display:<12} {last_heard_display:<12} {value:<8} {location:<15}")
except Exception as e: except Exception as e:
pass pass
@ -385,8 +323,8 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
if identity == None: raise ValueError("Could not load management identity from "+str(management_identity)) if identity == None: raise ValueError("Could not load management identity from "+str(management_identity))
try: try:
remote_status = get_remote_status(destination_hash, lstats, profiling, identity, no_output=json, timeout=remote_timeout) remote_status = get_remote_status(destination_hash, lstats, identity, no_output=json, timeout=remote_timeout)
if remote_status != None: stats, link_count, profiling_results = remote_status if remote_status != None: stats, link_count = remote_status
except Exception as e: raise e except Exception as e: raise e
except Exception as e: except Exception as e:
@ -398,12 +336,6 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
if lstats: if lstats:
try: link_count = reticulum.get_link_count() try: link_count = reticulum.get_link_count()
except Exception as e: pass except Exception as e: pass
try: active_link_count = reticulum.get_active_link_count()
except Exception as e: pass
if profiling:
try: profiling_results = reticulum.get_profiling_results()
except Exception as e: pass
try: stats = reticulum.get_interface_stats() try: stats = reticulum.get_interface_stats()
except Exception as e: pass except Exception as e: pass
@ -429,34 +361,30 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
interfaces = stats["interfaces"] interfaces = stats["interfaces"]
if sorting != None and isinstance(sorting, str): if sorting != None and isinstance(sorting, str):
sorting = sorting.lower() sorting = sorting.lower()
if sorting == "rate" or sorting == "bitrate": interfaces.sort(key=lambda i: i["bitrate"], reverse=not sort_reverse) if sorting == "rate" or sorting == "bitrate":
if sorting == "rx": interfaces.sort(key=lambda i: i["rxb"], reverse=not sort_reverse) interfaces.sort(key=lambda i: i["bitrate"], reverse=not sort_reverse)
if sorting == "tx": interfaces.sort(key=lambda i: i["txb"], reverse=not sort_reverse) if sorting == "rx":
if sorting == "frx": interfaces.sort(key=lambda i: i["frxb"], reverse=not sort_reverse) interfaces.sort(key=lambda i: i["rxb"], reverse=not sort_reverse)
if sorting == "ftx": interfaces.sort(key=lambda i: i["ftxb"], reverse=not sort_reverse) if sorting == "tx":
if sorting == "rxs": interfaces.sort(key=lambda i: i["rxs"], reverse=not sort_reverse) interfaces.sort(key=lambda i: i["txb"], reverse=not sort_reverse)
if sorting == "txs": interfaces.sort(key=lambda i: i["txs"], reverse=not sort_reverse) if sorting == "rxs":
if sorting == "frxs": interfaces.sort(key=lambda i: i["frxs"], reverse=not sort_reverse) interfaces.sort(key=lambda i: i["rxs"], reverse=not sort_reverse)
if sorting == "ftxs": interfaces.sort(key=lambda i: i["ftxs"], reverse=not sort_reverse) if sorting == "txs":
if sorting == "traffic": interfaces.sort(key=lambda i: i["rxb"]+i["txb"], reverse=not sort_reverse) interfaces.sort(key=lambda i: i["txs"], reverse=not sort_reverse)
if sorting == "anns" or sorting == "announces": interfaces.sort(key=lambda i: i["incoming_announce_frequency"]+i["outgoing_announce_frequency"], reverse=not sort_reverse) if sorting == "traffic":
if sorting == "arx": interfaces.sort(key=lambda i: i["incoming_announce_frequency"], reverse=not sort_reverse) interfaces.sort(key=lambda i: i["rxb"]+i["txb"], reverse=not sort_reverse)
if sorting == "atx": interfaces.sort(key=lambda i: i["outgoing_announce_frequency"], reverse=not sort_reverse) if sorting == "announces" or sorting == "announce":
if sorting == "arxc": interfaces.sort(key=lambda i: i["arxc"], reverse=not sort_reverse) interfaces.sort(key=lambda i: i["incoming_announce_frequency"]+i["outgoing_announce_frequency"], reverse=not sort_reverse)
if sorting == "atxc": interfaces.sort(key=lambda i: i["atxc"], reverse=not sort_reverse) if sorting == "arx":
if sorting == "afrxc": interfaces.sort(key=lambda i: i["afrxc"], reverse=not sort_reverse) interfaces.sort(key=lambda i: i["incoming_announce_frequency"], reverse=not sort_reverse)
if sorting == "prx": interfaces.sort(key=lambda i: i["incoming_pr_frequency"], reverse=not sort_reverse) if sorting == "atx":
if sorting == "ptx": interfaces.sort(key=lambda i: i["outgoing_pr_frequency"], reverse=not sort_reverse) interfaces.sort(key=lambda i: i["outgoing_announce_frequency"], reverse=not sort_reverse)
if sorting == "prxc": interfaces.sort(key=lambda i: i["prxc"], reverse=not sort_reverse) if sorting == "prx":
if sorting == "ptxc": interfaces.sort(key=lambda i: i["ptxc"], reverse=not sort_reverse) interfaces.sort(key=lambda i: i["incoming_pr_frequency"], reverse=not sort_reverse)
if sorting == "held": interfaces.sort(key=lambda i: i["held_announces"], reverse=not sort_reverse) if sorting == "ptx":
if sorting == "pvs": interfaces.sort(key=lambda i: i["protocol_violations"], reverse=not sort_reverse) interfaces.sort(key=lambda i: i["outgoing_pr_frequency"], reverse=not sort_reverse)
if sorting == "ivs": interfaces.sort(key=lambda i: i["ifac_violations"], reverse=not sort_reverse) if sorting == "held":
if sorting == "flt": interfaces.sort(key=lambda i: i["packet_filter_hits"], reverse=not sort_reverse) interfaces.sort(key=lambda i: i["held_announces"], reverse=not sort_reverse)
if sorting == "gravity" or sorting == "g": interfaces.sort(key=lambda i: i["gravity"], reverse=not sort_reverse)
if sorting == "txdrp": interfaces.sort(key=lambda i: i["txdrp"], reverse=not sort_reverse)
if sorting == "txdrb": interfaces.sort(key=lambda i: i["txdrb"], reverse=not sort_reverse)
if sorting == "txbuf": interfaces.sort(key=lambda i: i["txbuffered"], reverse=not sort_reverse)
for ifstat in interfaces: for ifstat in interfaces:
@ -490,16 +418,13 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
if ifstat["status"]: ss = "Up" if ifstat["status"]: ss = "Up"
else: ss = "Down" else: ss = "Down"
if "gravity" in ifstat and ifstat["gravity"]: ss += ", gravity "+str(ifstat["gravity"])
if ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_ACCESS_POINT: modestr = "Access Point" if ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_ACCESS_POINT: modestr = "Access Point"
elif ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_POINT_TO_POINT: modestr = "Point-to-Point" elif ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_POINT_TO_POINT: modestr = "Point-to-Point"
elif ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_ROAMING: modestr = "Roaming" elif ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_ROAMING: modestr = "Roaming"
elif ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_BOUNDARY: modestr = "Boundary" elif ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_BOUNDARY: modestr = "Boundary"
elif ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_GATEWAY: modestr = "Gateway" elif ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_GATEWAY: modestr = "Gateway"
elif ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_INTERNAL: modestr = "Internal"
else: modestr = "Full" else: modestr = "Full"
if "announces_to_internal" in ifstat and ifstat["announces_to_internal"]: modestr += " (a>i)"
if ifstat["clients"] != None: if ifstat["clients"] != None:
clients = ifstat["clients"] clients = ifstat["clients"]
@ -524,12 +449,6 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
clients_string = "" clients_string = ""
else: else:
clients_string = "Clients : "+str(clients) clients_string = "Clients : "+str(clients)
if "blocked_ips" in ifstat:
p = ifstat["blocked_ips"] > 0
if p: clients_string += "\n Blocked : "+str(ifstat["blocked_ips"])+" IP"+"s" if p else ""
if blocked_ips and "blocked_ip_list" in ifstat and len(ifstat["blocked_ip_list"]) > 0:
for bip in ifstat["blocked_ip_list"]: clients_string += f"\n {bip}"
else: else:
clients = None clients = None
@ -548,28 +467,10 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
print(" "+clients_string) print(" "+clients_string)
if not (name.startswith("Shared Instance[") or name.startswith("TCPInterface[Client") or name.startswith("LocalInterface[")): if not (name.startswith("Shared Instance[") or name.startswith("TCPInterface[Client") or name.startswith("LocalInterface[")):
print(f" Mode : {modestr}") print(" Mode : {mode}".format(mode=modestr))
if ifstat["txdrp"]:
print(f" TX Drops : {ifstat['txdrp']} ({RNS.prettysize(ifstat['txdrb'])})")
if "bitrate" in ifstat and ifstat["bitrate"] != None: if "bitrate" in ifstat and ifstat["bitrate"] != None:
print(f" Rate : {speed_str(ifstat['bitrate'])}, MTU {ifstat['mtu']}") print(" Rate : {ss}".format(ss=speed_str(ifstat["bitrate"])))
if tstats and not clients:
print(f" Tunnel ID : {RNS.hexrep(ifstat['tunnel_id'], delimit=False) if ifstat['tunnel_id'] else 'None'}")
if tstats and ifstat["tunnel_id"]:
print(f" Rem.Tr.ID : {RNS.prettyhexrep(ifstat['remote_transport_id']) if ifstat['remote_transport_id'] else 'Unknown'}")
trns_str = ", transport enabled" if ifstat['rem_is_tn'] else ", no transport"
impl_str = f"{ifstat['rem_impl']}" if ifstat['rem_impl'] else "Unknown implementation"
vers_str = f" {'.'.join([str(v) for v in ifstat['rem_vers']])}{trns_str}" if ifstat['rem_vers'] else ", unknown version"
print(f" Rem.Vers. : {impl_str}{vers_str}")
dst_cost_str = f"Dst {RNS.prettycost(ifstat['rem_dst_cost'])}" if ifstat["rem_dst_cost"] != None else "Dst ?"
pr_cost_str = f"PR {RNS.prettycost(ifstat['rem_pr_cost'])}" if ifstat["rem_pr_cost"] != None else "PR ?"
print(f" Rem.Csts. : {dst_cost_str}, {pr_cost_str}")
if "noise_floor" in ifstat: if "noise_floor" in ifstat:
if not "interference" in ifstat: nstr = "" if not "interference" in ifstat: nstr = ""
@ -663,17 +564,13 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
burst_str = "" burst_str = ""
if "burst_active" in ifstat and ifstat["burst_active"]: if "burst_active" in ifstat and ifstat["burst_active"]:
if "burst_count" in ifstat and ifstat["burst_count"]: bcstr = f"on {ifstat['burst_count']} "
else: bcstr = ""
for_str = RNS.prettytime(time.time()-ifstat["burst_activated"]) for_str = RNS.prettytime(time.time()-ifstat["burst_activated"])
burst_str = f" burst {bcstr}for {for_str}" burst_str = f" burst for {for_str}"
pburst_str = "" pburst_str = ""
if "pr_burst_active" in ifstat and ifstat["pr_burst_active"]: if "pr_burst_active" in ifstat and ifstat["pr_burst_active"]:
if "pr_burst_count" in ifstat and ifstat["pr_burst_count"]: prbcstr = f"on {ifstat['pr_burst_count']} "
else: prbcstr = ""
for_str = RNS.prettytime(time.time()-ifstat["pr_burst_activated"]) for_str = RNS.prettytime(time.time()-ifstat["pr_burst_activated"])
pburst_str = f"burst {prbcstr}for {for_str}" pburst_str = f"burst for {for_str}"
rxb_str = "↓"+RNS.prettysize(ifstat["rxb"]) rxb_str = "↓"+RNS.prettysize(ifstat["rxb"])
txb_str = "↑"+RNS.prettysize(ifstat["txb"]) txb_str = "↑"+RNS.prettysize(ifstat["txb"])
@ -690,14 +587,6 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
if clients == None and "peers" in ifstat and ifstat["peers"]: clients = ifstat["peers"]; cspec = "p" if clients == None and "peers" in ifstat and ifstat["peers"]: clients = ifstat["peers"]; cspec = "p"
if clients != None and clients > 0: pc_str = f"{RNS.prettyfrequency(ifstat['outgoing_announce_frequency']/clients, d=1, lpf=True)}/{cspec}" if clients != None and clients > 0: pc_str = f"{RNS.prettyfrequency(ifstat['outgoing_announce_frequency']/clients, d=1, lpf=True)}/{cspec}"
else: pc_str = "" else: pc_str = ""
if "prxs" in ifstat and "rxs" in ifstat and "ptxs" in ifstat and "txs" in ifstat:
arxspct = min(100.0, (ifstat["arxs"] / ifstat["rxs"])*100.0) if ifstat["rxs"] and ifstat["arxs"] else 0.0
atxspct = min(100.0, (ifstat["atxs"] / ifstat["txs"])*100.0) if ifstat["txs"] and ifstat["atxs"] else 0.0
apctstr = f"(↓{int(arxspct)}% / ↑{int(atxspct)}% of flow)"
if pc_str: pc_str = f"{pc_str} {apctstr}"
else: pc_str = f"{apctstr}"
asr = True asr = True
psr = False psr = False
@ -715,14 +604,6 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
if clients == None and "peers" in ifstat and ifstat["peers"]: clients = ifstat["peers"]; cspec = "p" if clients == None and "peers" in ifstat and ifstat["peers"]: clients = ifstat["peers"]; cspec = "p"
if clients != None and clients > 0: rpc_str = f"{RNS.prettyfrequency(ifstat['outgoing_pr_frequency']/clients, d=1, lpf=True)}/{cspec}" if clients != None and clients > 0: rpc_str = f"{RNS.prettyfrequency(ifstat['outgoing_pr_frequency']/clients, d=1, lpf=True)}/{cspec}"
else: rpc_str = "" else: rpc_str = ""
if "prxs" in ifstat and "rxs" in ifstat and "ptxs" in ifstat and "txs" in ifstat:
prxspct = min(100.0, (ifstat["prxs"] / ifstat["rxs"])*100.0) if ifstat["rxs"] and ifstat["prxs"] else 0.0
ptxspct = min(100.0, (ifstat["ptxs"] / ifstat["txs"])*100.0) if ifstat["txs"] and ifstat["ptxs"] else 0.0
ppctstr = f"(↓{int(prxspct)}% / ↑{int(ptxspct)}% of flow)"
if rpc_str: rpc_str = f"{rpc_str} {ppctstr}"
else: rpc_str = f"{ppctstr}"
psr = True psr = True
if not asr: iaf = ""; oaf = "" if not asr: iaf = ""; oaf = ""
@ -738,30 +619,11 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
rxb_str += (mlen-len(rxb_str))*" " rxb_str += (mlen-len(rxb_str))*" "
txb_str += (mlen-len(txb_str))*" " txb_str += (mlen-len(txb_str))*" "
if "protocol_violations" in ifstat and "ifac_violations" in ifstat:
pv = ifstat["protocol_violations"]; iv = ifstat["ifac_violations"]
if pv or iv:
pvstr = f"{pv} protocol{', ' if iv else ''}"
ivstr = f"{iv} IFAC" if iv else ""
print(f" Violatns. : {pvstr}{ivstr}")
if "packet_filter_hits" in ifstat and ifstat["packet_filter_hits"]:
print(f" Flt. Hits : {ifstat['packet_filter_hits']}")
if psr: if psr:
if ifstat["prxc"] and ifstat["ptxc"]:
print(f" Path Rqs. : {ifstat['prxc']}↓ {ifstat['ptxc']}↑ total")
print(f" {opf} {rpc_str}")
else:
print(f" Path Rqs. : {opf} {rpc_str}") print(f" Path Rqs. : {opf} {rpc_str}")
print(f" {ipf} {pburst_str}") print(f" {ipf} {pburst_str}")
if asr: if asr:
if ifstat["arxc"] and ifstat["atxc"]:
f_str = f", {ifstat['afrxc']} filtered" if ifstat["afrxc"] else ""
print(f" Announces : {ifstat['arxc']}↓ {ifstat['atxc']}↑ total{f_str}")
print(f" {oaf} {pc_str}")
else:
print(f" Announces : {oaf} {pc_str}") print(f" Announces : {oaf} {pc_str}")
print(f" {iaf} {art_str}{burst_str}") print(f" {iaf} {art_str}{burst_str}")
@ -771,21 +633,8 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
rxstat += " "+RNS.prettyspeed(ifstat["rxs"]) rxstat += " "+RNS.prettyspeed(ifstat["rxs"])
txstat += " "+RNS.prettyspeed(ifstat["txs"]) txstat += " "+RNS.prettyspeed(ifstat["txs"])
if ifstat["txbuffered"]:
if ifstat["txstalled"]: stlstr = ", stalled"
else: stlstr = ""
txstat += f" ({RNS.prettysize(ifstat['txbuffered'])} waiting{stlstr})"
print(f" Traffic : {txstat}\n {rxstat}") print(f" Traffic : {txstat}\n {rxstat}")
if fstats:
frxstat = "↓"+RNS.prettysize(ifstat["frxb"])
ftxstat = "↑"+RNS.prettysize(ifstat["ftxb"])
if "frxs" in ifstat and "ftxs" in ifstat:
frxstat += " "+RNS.prettyspeed(ifstat["frxs"])
ftxstat += " "+RNS.prettyspeed(ifstat["ftxs"])
print(f" Filtered : {ftxstat}\n {frxstat}")
lstr = "" lstr = ""
if link_count != None and lstats: if link_count != None and lstats:
ms = "y" if link_count == 1 else "ies" ms = "y" if link_count == 1 else "ies"
@ -794,131 +643,19 @@ def program_setup(configdir, dispall=False, verbosity=0, name_filter=None, json=
else: else:
lstr = f" {link_count} entr{ms} in link table" lstr = f" {link_count} entr{ms} in link table"
if active_link_count: lstr = f"{lstr} ({active_link_count} active)"
if traffic_totals: if traffic_totals:
rxb_str = "↓"+RNS.prettysize(stats["rxb"]) rxb_str = "↓"+RNS.prettysize(stats["rxb"])
txb_str = "↑"+RNS.prettysize(stats["txb"]) txb_str = "↑"+RNS.prettysize(stats["txb"])
strdiff = len(rxb_str)-len(txb_str) strdiff = len(rxb_str)-len(txb_str)
if strdiff > 0: txb_str += " "*strdiff if strdiff > 0:
elif strdiff < 0: rxb_str += " "*-strdiff txb_str += " "*strdiff
elif strdiff < 0:
rxb_str += " "*-strdiff
rxstat = rxb_str+" "+RNS.prettyspeed(stats["rxs"]) rxstat = rxb_str+" "+RNS.prettyspeed(stats["rxs"])
txstat = txb_str+" "+RNS.prettyspeed(stats["txs"]) txstat = txb_str+" "+RNS.prettyspeed(stats["txs"])
if pstats or astats:
if "prxs" in stats and "ptxs" in stats and "arxs" in stats and "atxs" in stats:
parxs = stats["prxs"]+stats["arxs"]
patxs = stats["ptxs"]+stats["atxs"]
drxpct = 100.0-min(100.0, (parxs / stats["rxs"])*100.0) if stats["rxs"] else 100.0
dtxpct = 100.0-min(100.0, (patxs / stats["txs"])*100.0) if stats["txs"] else 100.0
drxs = stats["rxs"]*(drxpct/100.0)
dtxs = stats["txs"]*(dtxpct/100.0)
if stats["rxs"] > 0: rxstat = f"{rxstat}, {int(drxpct)}% data ({RNS.prettyspeed(drxs)})"
if stats["txs"] > 0: txstat = f"{txstat}, {int(dtxpct)}% data ({RNS.prettyspeed(dtxs)})"
if pps:
rpps = RNS.prettysize(stats['rxpps'], suffix="pps")
tpps = RNS.prettysize(stats['txpps'], suffix="pps")
rxstat = f"{rxstat}, {rpps}"
txstat = f"{txstat}, {tpps}"
print(f"\n Totals : {txstat}\n {rxstat}") print(f"\n Totals : {txstat}\n {rxstat}")
if fstats:
frxstat = "↓"+RNS.prettysize(stats["frxb"])
ftxstat = "↑"+RNS.prettysize(stats["ftxb"])
strdiff = len(frxstat)-len(ftxstat)
if strdiff > 0: ftxstat += " "*strdiff
elif strdiff < 0: frxstat += " "*-strdiff
if "frxs" in stats and "ftxs" in stats:
frxstat += " "+RNS.prettyspeed(stats["frxs"])
ftxstat += " "+RNS.prettyspeed(stats["ftxs"])
print(f"\n Filtered : {ftxstat}\n {frxstat}")
if lstats:
if "lrxb" in stats and "ltxb" in stats and "lrxs" in stats and "ltxs" in stats:
lrxb_str = "↓"+RNS.prettysize(stats["lrxb"])
ltxb_str = "↑"+RNS.prettysize(stats["ltxb"])
strdiff = len(lrxb_str)-len(ltxb_str)
if strdiff > 0: ltxb_str += " "*strdiff
elif strdiff < 0: lrxb_str += " "*-strdiff
parxs = stats["prxs"]+stats["arxs"]
patxs = stats["ptxs"]+stats["atxs"]
drxpct = 100.0-min(100.0, (parxs / stats["rxs"])*100.0) if stats["rxs"] else 100.0
dtxpct = 100.0-min(100.0, (patxs / stats["txs"])*100.0) if stats["txs"] else 100.0
drxs = stats["rxs"]*(drxpct/100.0)
dtxs = stats["txs"]*(dtxpct/100.0)
lrxspct = min(100.0, (stats["lrxs"] / drxs)*100.0) if drxs and stats["lrxs"] else 0.0
ltxspct = min(100.0, (stats["ltxs"] / dtxs)*100.0) if dtxs and stats["ltxs"] else 0.0
lrxstat = lrxb_str+" "+RNS.prettyspeed(stats["lrxs"])+f", {int(lrxspct)}% of data flow"
ltxstat = ltxb_str+" "+RNS.prettyspeed(stats["ltxs"])+f", {int(ltxspct)}% of data flow"
print(f"\n Links : {ltxstat}\n {lrxstat}")
if pstats:
if "prxb" in stats and "ptxb" in stats and "prxs" in stats and "ptxs" in stats:
prxb_str = "↓"+RNS.prettysize(stats["prxb"])
ptxb_str = "↑"+RNS.prettysize(stats["ptxb"])
strdiff = len(prxb_str)-len(ptxb_str)
if strdiff > 0: ptxb_str += " "*strdiff
elif strdiff < 0: prxb_str += " "*-strdiff
prxspct = min(100.0, (stats["prxs"] / stats["rxs"])*100.0) if stats["rxs"] and stats["prxs"] else 0.0
ptxspct = min(100.0, (stats["ptxs"] / stats["txs"])*100.0) if stats["txs"] and stats["ptxs"] else 0.0
prxfstr = RNS.prettyfrequency(stats["prxf"])
ptxfstr = RNS.prettyfrequency(stats["ptxf"])
prxstat = prxb_str+" "+RNS.prettyspeed(stats["prxs"])+f", {int(prxspct)}% of flow, {prxfstr}"
ptxstat = ptxb_str+" "+RNS.prettyspeed(stats["ptxs"])+f", {int(ptxspct)}% of flow, {ptxfstr}"
print(f"\n Path Rqs. : {ptxstat}\n {prxstat}")
if astats:
if "arxb" in stats and "atxb" in stats and "arxs" in stats and "atxs" in stats:
arxb_str = "↓"+RNS.prettysize(stats["arxb"])
atxb_str = "↑"+RNS.prettysize(stats["atxb"])
strdiff = len(arxb_str)-len(atxb_str)
if strdiff > 0: atxb_str += " "*strdiff
elif strdiff < 0: arxb_str += " "*-strdiff
arxspct = min(100.0, (stats["arxs"] / stats["rxs"])*100.0) if stats["rxs"] and stats["arxs"] else 0.0
atxspct = min(100.0, (stats["atxs"] / stats["txs"])*100.0) if stats["txs"] and stats["atxs"] else 0.0
arxfstr = RNS.prettyfrequency(stats["arxf"])
atxfstr = RNS.prettyfrequency(stats["atxf"])
arxstat = arxb_str+" "+RNS.prettyspeed(stats["arxs"])+f", {int(arxspct)}% of flow, {arxfstr}"
atxstat = atxb_str+" "+RNS.prettyspeed(stats["atxs"])+f", {int(atxspct)}% of flow, {atxfstr}"
print(f"\n Announces : {atxstat}\n {arxstat}")
if queue_stats:
tqdp = f", {stats['rxqtd']} dropped" if stats['rxqtd'] else ""
dqdp = f", {stats['rxqdd']} dropped" if stats['rxqdd'] else ""
aqdp = f", {stats['rxqad']} dropped" if stats['rxqad'] else ""
pqdp = f", {stats['rxqpd']} dropped" if stats['rxqpd'] else ""
ilqdp = f", {stats['rxqild']} dropped" if stats['rxqild'] else ""
tqpress = f"{round(stats['tqpressure']*100.0, 1)}% total, {stats['rxqt']} pkts{tqdp}"
dqpress = f"{round(stats['dqpressure']*100.0, 1)}% data, {stats['rxqd']} pkts{dqdp}"
aqpress = f"{round(stats['aqpressure']*100.0, 1)}% announce, {stats['rxqa']} pkts{aqdp}"
pqpress = f"{round(stats['pqpressure']*100.0, 1)}% path request, {stats['rxqp']} pkts{pqdp}"
ilpress = f"{round(stats['ilqpressure']*100.0, 1)}% ingress limiter, {stats['rxqil']} pkts{ilqdp}"
print(f"\n Qu. Pressure : {tqpress}")
print(f" {dqpress}")
print(f" {aqpress}")
print(f" {pqpress}")
print(f" {ilpress}")
if profiling_results:
print(f"\n Profiling :\n{RNS.Profiler.format_results(profiling_results)}")
if "transport_id" in stats and stats["transport_id"] != None: if "transport_id" in stats and stats["transport_id"] != None:
print("\n Transport Instance "+RNS.prettyhexrep(stats["transport_id"])+" running") print("\n Transport Instance "+RNS.prettyhexrep(stats["transport_id"])+" running")
if "network_id" in stats and stats["network_id"] != None: if "network_id" in stats and stats["network_id"] != None:
@ -949,23 +686,13 @@ def main(must_exit=True, rns_instance=None):
parser.add_argument("--config", action="store", default=None, help="path to alternative Reticulum config directory", type=str) parser.add_argument("--config", action="store", default=None, help="path to alternative Reticulum config directory", type=str)
parser.add_argument("--version", action="version", version="rnstatus {version}".format(version=__version__)) parser.add_argument("--version", action="version", version="rnstatus {version}".format(version=__version__))
parser.add_argument("--attach", action="store", metavar="name", help="Attach interface by name", default=None, type=str)
parser.add_argument("--detach", action="store", metavar="name", help="Detach interface by name", default=None, type=str)
parser.add_argument("--reload", action="store", metavar="name", help="Reload interface by name", default=None, type=str)
parser.add_argument("-a", "--all", action="store_true", help="show all interfaces", default=False) parser.add_argument("-a", "--all", action="store_true", help="show all interfaces", default=False)
parser.add_argument("-A", "--announce-stats", action="store_true", help="show announce stats", default=False) parser.add_argument("-A", "--announce-stats", action="store_true", help="show announce stats", default=False)
parser.add_argument("-P", "--pr-stats", action="store_true", help="show path request stats", default=False) parser.add_argument("-P", "--pr-stats", action="store_true", help="show path request stats", default=False)
parser.add_argument("-F", "--filter-stats", action="store_true", help="show filtered traffic stats", default=False)
parser.add_argument("-l", "--link-stats", action="store_true", help="show link stats", default=False) parser.add_argument("-l", "--link-stats", action="store_true", help="show link stats", default=False)
parser.add_argument("-B", "--burst", action="store_true", help="only show interfaces with active bursts", default=False) parser.add_argument("-B", "--burst", action="store_true", help="only show interfaces with active bursts", default=False)
parser.add_argument("-b", "--blocked-ips", action="store_true", help="show blocked IPs per interface", default=False)
parser.add_argument("-t", "--totals", action="store_true", help="display traffic totals", default=False) parser.add_argument("-t", "--totals", action="store_true", help="display traffic totals", default=False)
parser.add_argument("-T", "--tunnels", action="store_true", help="display tunnel information", default=False) parser.add_argument("-s", "--sort", action="store", help="sort interfaces by [rate, traffic, rx, tx, rxs, txs, announces, arx, atx, prx, ptx, held]", default=None, type=str)
parser.add_argument("-p", "--pps", action="store_true", help="display packets per second in totals", default=False)
parser.add_argument("-q", "--queues", action="store_true", help="display queue stats", default=False)
parser.add_argument("-z", "--profiling", action="store_true", help="display live profiling results", default=False)
parser.add_argument("-s", "--sort", action="store", help="sort interfaces by [rate, traffic, rx, tx, rxs, txs, frx, ftx, anns, arx, atx, arxc, atxc, afrxc, held, prx, ptx, prxc, ptxc, pvs, ivs, flt, txdrp, txdrb, txbuf]", default=None, type=str)
parser.add_argument("-r", "--reverse", action="store_true", help="reverse sorting", default=False) parser.add_argument("-r", "--reverse", action="store_true", help="reverse sorting", default=False)
parser.add_argument("-j", "--json", action="store_true", help="output in JSON format", default=False) parser.add_argument("-j", "--json", action="store_true", help="output in JSON format", default=False)
parser.add_argument("-R", action="store", metavar="hash", help="transport identity hash of remote instance to get status from", default=None, type=str) parser.add_argument("-R", action="store", metavar="hash", help="transport identity hash of remote instance to get status from", default=None, type=str)
@ -973,8 +700,6 @@ def main(must_exit=True, rns_instance=None):
parser.add_argument("-w", action="store", metavar="seconds", type=float, help="timeout before giving up on remote queries", default=RNS.Transport.PATH_REQUEST_TIMEOUT) parser.add_argument("-w", action="store", metavar="seconds", type=float, help="timeout before giving up on remote queries", default=RNS.Transport.PATH_REQUEST_TIMEOUT)
parser.add_argument("-d", "--discovered", action="store_true", help="list discovered interfaces", default=False) parser.add_argument("-d", "--discovered", action="store_true", help="list discovered interfaces", default=False)
parser.add_argument("-D", action="store_true", help="show details and config entries for discovered interfaces", default=False) parser.add_argument("-D", action="store_true", help="show details and config entries for discovered interfaces", default=False)
parser.add_argument("--show-stale", action="store_true", help="show stale discovery entries", default=False)
parser.add_argument("--show-unknown", action="store_true", help="show discovery entries without version info", default=False)
parser.add_argument("-m", "--monitor", action="store_true", help="continuously monitor status", default=False) parser.add_argument("-m", "--monitor", action="store_true", help="continuously monitor status", default=False)
parser.add_argument("-I", "--monitor-interval", action="store", metavar="seconds", type=float, help="refresh interval for monitor mode (default: 1)", default=1.0) parser.add_argument("-I", "--monitor-interval", action="store", metavar="seconds", type=float, help="refresh interval for monitor mode (default: 1)", default=1.0)
parser.add_argument('-v', '--verbose', action='count', default=0) parser.add_argument('-v', '--verbose', action='count', default=0)
@ -989,9 +714,7 @@ def main(must_exit=True, rns_instance=None):
if args.R: require_shared = False if args.R: require_shared = False
else: require_shared = True else: require_shared = True
try: try: reticulum = RNS.Reticulum(configdir=configarg, loglevel=3+args.verbose, require_shared_instance=require_shared)
reticulum = RNS.Reticulum(configdir=configarg, loglevel=3+args.verbose, require_shared_instance=require_shared)
reticulum.wants_destination_type("rnstransport.remote.management")
except Exception as e: except Exception as e:
print("No shared RNS instance available to get status from") print("No shared RNS instance available to get status from")
exit(1) exit(1)
@ -1006,9 +729,7 @@ def main(must_exit=True, rns_instance=None):
program_setup(configdir = configarg, dispall = args.all, verbosity=args.verbose, name_filter=args.filter, json=args.json, program_setup(configdir = configarg, dispall = args.all, verbosity=args.verbose, name_filter=args.filter, json=args.json,
astats=args.announce_stats, pstats=args.pr_stats, lstats=args.link_stats, sorting=args.sort, sort_reverse=args.reverse, astats=args.announce_stats, pstats=args.pr_stats, lstats=args.link_stats, sorting=args.sort, sort_reverse=args.reverse,
remote=args.R, management_identity=args.i, remote_timeout=args.w, must_exit=False, rns_instance=reticulum, remote=args.R, management_identity=args.i, remote_timeout=args.w, must_exit=False, rns_instance=reticulum,
traffic_totals=args.totals, discovered_interfaces=args.discovered, config_entries=args.D, burst_filter=args.burst, traffic_totals=args.totals, discovered_interfaces=args.discovered, config_entries=args.D, burst_filter=args.burst)
blocked_ips=args.blocked_ips, queue_stats=args.queues, pps=args.pps, profiling=args.profiling,
show_stale=args.show_stale, show_unknown=args.show_unknown, fstats=args.filter_stats, tstats=args.tunnels)
finally: finally:
sys.stdout = old_stdout sys.stdout = old_stdout
@ -1025,10 +746,7 @@ def main(must_exit=True, rns_instance=None):
program_setup(configdir = configarg, dispall = args.all, verbosity=args.verbose, name_filter=args.filter, json=args.json, program_setup(configdir = configarg, dispall = args.all, verbosity=args.verbose, name_filter=args.filter, json=args.json,
astats=args.announce_stats, pstats=args.pr_stats, lstats=args.link_stats, sorting=args.sort, sort_reverse=args.reverse, astats=args.announce_stats, pstats=args.pr_stats, lstats=args.link_stats, sorting=args.sort, sort_reverse=args.reverse,
remote=args.R, management_identity=args.i, remote_timeout=args.w, must_exit=must_exit, rns_instance=rns_instance, remote=args.R, management_identity=args.i, remote_timeout=args.w, must_exit=must_exit, rns_instance=rns_instance,
traffic_totals=args.totals, discovered_interfaces=args.discovered, config_entries=args.D, burst_filter=args.burst, traffic_totals=args.totals, discovered_interfaces=args.discovered, config_entries=args.D, burst_filter=args.burst)
blocked_ips=args.blocked_ips, queue_stats=args.queues, pps=args.pps, profiling=args.profiling,
attach=args.attach, detach=args.detach, reload=args.reload, show_stale=args.show_stale, tstats=args.tunnels,
show_unknown=args.show_unknown, fstats=args.filter_stats)
except KeyboardInterrupt: except KeyboardInterrupt:
print("") print("")

View file

@ -2,7 +2,7 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -342,14 +342,12 @@ def execute(configdir, identitypath = None, verbosity = 0, quietness = 0, detail
targetloglevel = 3+verbosity-quietness targetloglevel = 3+verbosity-quietness
reticulum = RNS.Reticulum(configdir=configdir, loglevel=targetloglevel) reticulum = RNS.Reticulum(configdir=configdir, loglevel=targetloglevel)
reticulum.wants_destination_type(f"{APP_NAME}.execute")
if identity == None: if identity == None:
prepare_identity(identitypath) prepare_identity(identitypath)
if not RNS.Transport.has_path(destination_hash): if not RNS.Transport.has_path(destination_hash):
RNS.Transport.request_path(destination_hash) RNS.Transport.request_path(destination_hash)
if not spin(until=lambda: RNS.Transport.has_path(destination_hash), msg="Path to "+RNS.prettyhexrep(destination_hash)+" requested", timeout=max(timeout, reticulum.get_medium_path_timeout())): if not spin(until=lambda: RNS.Transport.has_path(destination_hash), msg="Path to "+RNS.prettyhexrep(destination_hash)+" requested", timeout=timeout):
print("Path not found") print("Path not found")
exit(242) exit(242)
@ -367,7 +365,7 @@ def execute(configdir, identitypath = None, verbosity = 0, quietness = 0, detail
link = RNS.Link(listener_destination) link = RNS.Link(listener_destination)
link.did_identify = False link.did_identify = False
if not spin(until=lambda: link.status == RNS.Link.ACTIVE, msg="Establishing link with "+RNS.prettyhexrep(destination_hash), timeout=max(timeout, link.establishment_timeout)): if not spin(until=lambda: link.status == RNS.Link.ACTIVE, msg="Establishing link with "+RNS.prettyhexrep(destination_hash), timeout=timeout):
print("Could not establish link with "+RNS.prettyhexrep(destination_hash)) print("Could not establish link with "+RNS.prettyhexrep(destination_hash))
exit(243) exit(243)

View file

@ -1,6 +1,6 @@
# Reticulum License # Reticulum License
# #
# Copyright (c) 2016-2026 Mark Qvist # Copyright (c) 2016-2025 Mark Qvist
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -35,24 +35,30 @@ import time
import datetime import datetime
import random import random
import threading import threading
import math
import bisect
from ._version import __version__ from ._version import __version__
from collections import deque
from functools import partial, wraps from .Reticulum import Reticulum
from RNS.Stamps import quantize_cost from .Identity import Identity
from .Link import Link, RequestReceipt
from .Channel import MessageBase
from .Buffer import Buffer, RawChannelReader, RawChannelWriter
from .Transport import Transport
from .Discovery import InterfaceAnnouncer
from .Destination import Destination
from .Packet import Packet
from .Packet import PacketReceipt
from .Resolver import Resolver
from .Resource import Resource, ResourceAdvertisement
from .Cryptography import HKDF
from .Cryptography import Hashes
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py") py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc") pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
so_modules = glob.glob(os.path.dirname(__file__)+"/*.so") modules = py_modules+pyc_modules
pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd") __all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
modules = py_modules+pyc_modules+so_modules+pyd_modules
__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))
import importlib.util import importlib.util
try: from ._buildinfo import compiled
except ImportError:
if importlib.util.find_spec("cython"): import cython; compiled = cython.compiled if importlib.util.find_spec("cython"): import cython; compiled = cython.compiled
else: compiled = False else: compiled = False
@ -64,8 +70,7 @@ LOG_NOTICE = 3
LOG_INFO = 4 LOG_INFO = 4
LOG_VERBOSE = 5 LOG_VERBOSE = 5
LOG_DEBUG = 6 LOG_DEBUG = 6
LOG_PATHING = 7 LOG_EXTREME = 7
LOG_EXTREME = 8
LOG_STDOUT = 0x91 LOG_STDOUT = 0x91
LOG_FILE = 0x92 LOG_FILE = 0x92
@ -77,7 +82,6 @@ loglevel = LOG_NOTICE
logfile = None logfile = None
logdest = LOG_STDOUT logdest = LOG_STDOUT
logcall = None logcall = None
logtimestamps = True
logtimefmt = "%Y-%m-%d %H:%M:%S" logtimefmt = "%Y-%m-%d %H:%M:%S"
logtimefmt_p = "%H:%M:%S.%f" logtimefmt_p = "%H:%M:%S.%f"
compact_log_fmt = False compact_log_fmt = False
@ -87,7 +91,7 @@ instance_random.seed(os.urandom(10))
_always_override_destination = False _always_override_destination = False
logging_lock = threading.RLock() logging_lock = threading.Lock()
def loglevelname(level): def loglevelname(level):
if (level == LOG_CRITICAL): return "[Critical]" if (level == LOG_CRITICAL): return "[Critical]"
@ -97,7 +101,6 @@ def loglevelname(level):
if (level == LOG_INFO): return "[Info] " if (level == LOG_INFO): return "[Info] "
if (level == LOG_VERBOSE): return "[Verbose] " if (level == LOG_VERBOSE): return "[Verbose] "
if (level == LOG_DEBUG): return "[Debug] " if (level == LOG_DEBUG): return "[Debug] "
if (level == LOG_PATHING): return "[Pathing] "
if (level == LOG_EXTREME): return "[Extra] " if (level == LOG_EXTREME): return "[Extra] "
return "Unknown" return "Unknown"
@ -124,8 +127,8 @@ def log(msg, level=3, _override_destination = False, pt=False):
if loglevel >= level: if loglevel >= level:
if pt: logstring = "["+precise_timestamp_str(time.time())+"] "+loglevelname(level)+" "+msg if pt: logstring = "["+precise_timestamp_str(time.time())+"] "+loglevelname(level)+" "+msg
else: else:
if not compact_log_fmt: logstring = ("["+timestamp_str(time.time())+"] " if logtimestamps else "")+loglevelname(level)+" "+msg if not compact_log_fmt: logstring = "["+timestamp_str(time.time())+"] "+loglevelname(level)+" "+msg
else: logstring = ("["+timestamp_str(time.time())+"] " if logtimestamps else "")+msg else: logstring = "["+timestamp_str(time.time())+"] "+msg
with logging_lock: with logging_lock:
if (logdest == LOG_STDOUT or _always_override_destination or _override_destination): if (logdest == LOG_STDOUT or _always_override_destination or _override_destination):
@ -156,6 +159,7 @@ def log(msg, level=3, _override_destination = False, pt=False):
log("Dumping future log events to console!", LOG_CRITICAL) log("Dumping future log events to console!", LOG_CRITICAL)
log(msg, level) log(msg, level)
def rand(): def rand():
result = instance_random.random() result = instance_random.random()
return result return result
@ -180,19 +184,6 @@ def prettyhexrep(data):
hexrep = "<"+delimiter.join("{:02x}".format(c) for c in data)+">" hexrep = "<"+delimiter.join("{:02x}".format(c) for c in data)+">"
return hexrep return hexrep
__namehash_resolver = None
def prettynamehash(name_hash):
global __namehash_resolver
if not __namehash_resolver:
import RNS.Transport
__namehash_resolver = RNS.Transport.destination_type_name
return f"<{__namehash_resolver(name_hash)}>"
def prettycost(value):
if value == None: return "n"
elif not type(value) in (float, int): return "n"
else: return f"{quantize_cost(value):.3f}"
def prettyspeed(num, suffix="b"): def prettyspeed(num, suffix="b"):
return prettysize(num/8, suffix=suffix)+"ps" return prettysize(num/8, suffix=suffix)+"ps"
@ -297,7 +288,7 @@ def prettytime(time, verbose=False, compact=False):
if not neg: return tstr if not neg: return tstr
else: return f"-{tstr}" else: return f"-{tstr}"
def prettyshorttime(time, verbose=False, compact=False, tight=False): def prettyshorttime(time, verbose=False, compact=False):
neg = False neg = False
time = time*1e6 time = time*1e6
if time < 0: if time < 0:
@ -333,8 +324,8 @@ def prettyshorttime(time, verbose=False, compact=False, tight=False):
for c in components: for c in components:
i += 1 i += 1
if i == 1: pass if i == 1: pass
elif i < len(components): tstr += ", " if not tight else " " elif i < len(components): tstr += ", "
elif i == len(components): tstr += " and " if not tight else " " elif i == len(components): tstr += " and "
tstr += c tstr += c
@ -371,50 +362,20 @@ class Profiler:
profilers = {} profilers = {}
tags = {} tags = {}
# Samples per tag
MAX_CAPTURES = 10000
@staticmethod @staticmethod
def get_profiler(tag=None, super_tag=None, max_captures=None): def get_profiler(tag=None, super_tag=None):
if tag in Profiler.profilers: return Profiler.profilers[tag] if tag in Profiler.profilers: return Profiler.profilers[tag]
else: else:
if max_captures is None: max_captures = Profiler.MAX_CAPTURES profiler = Profiler(tag, super_tag)
profiler = Profiler(tag, super_tag, max_captures)
Profiler.profilers[tag] = profiler Profiler.profilers[tag] = profiler
return profiler return profiler
@staticmethod def __init__(self, tag=None, super_tag=None):
def profile(func=None, *, tag=None, max_captures=None):
"""
Decorator to profile a function. With no arguments, it can be
used as `@RNS.Profiler.profile`, in which case it will use the
qualified name of the function as the profiler tag. It can
also be used as `@RNS.Profiler.profile(tag="prof_tag")` to use
a specific profiler tag.
:param tag: Either a profiler tag as passed to `Profiler.get_profiler(...)` or a `Profiler` object.
:param max_captures: Maximum samples to capture for the specified profiler tag.
"""
if func is None:
return partial(Profiler.profile, tag=tag, max_captures=max_captures)
tag = func.__qualname__ if tag is None else tag
if isinstance(tag, Profiler): profiler = tag
else: profiler = Profiler.get_profiler(tag=tag, max_captures=max_captures)
@wraps(func)
def wrapper(*args, **kwargs):
with profiler:
return func(*args, **kwargs)
return wrapper
def __init__(self, tag=None, super_tag=None, max_captures=None):
self.paused = False self.paused = False
self.pause_time = 0 self.pause_time = 0
self.pause_started = None self.pause_started = None
self.tag = tag self.tag = tag
self.super_tag = super_tag self.super_tag = super_tag
self.max_captures = max_captures if max_captures is not None else Profiler.MAX_CAPTURES
if self.super_tag in Profiler.profilers: if self.super_tag in Profiler.profilers:
self.super_profiler = Profiler.profilers[self.super_tag] self.super_profiler = Profiler.profilers[self.super_tag]
@ -432,18 +393,11 @@ class Profiler:
tag = self.tag tag = self.tag
super_tag = self.super_tag super_tag = self.super_tag
thread_ident = threading.get_ident() thread_ident = threading.get_ident()
if not tag in Profiler.tags: Profiler.tags[tag] = {"threads": {}, "super": super_tag, "captures": deque(maxlen=self.max_captures)} if not tag in Profiler.tags: Profiler.tags[tag] = {"threads": {}, "super": super_tag}
if not thread_ident in Profiler.tags[tag]["threads"]: if not thread_ident in Profiler.tags[tag]["threads"]:
Profiler.tags[tag]["threads"][thread_ident] = {"running": deque()} Profiler.tags[tag]["threads"][thread_ident] = {"current_start": None, "captures": []}
# The tag deque stores a shared reference to the capture, and the end Profiler.tags[tag]["threads"][thread_ident]["current_start"] = time.perf_counter()
# time isn't updated until the context manager exits. This leaves the
# deque sorted by start time, except for cases where the thread is
# preemted between getting the current time and appending. We also store
# a stack of start times per thread to support reentrancy.
capture = {"start": time.perf_counter(), "end": None, "thread_ident": thread_ident}
Profiler.tags[tag]["captures"].append(capture)
Profiler.tags[tag]["threads"][thread_ident]["running"].append(capture)
self.resume_super() self.resume_super()
def __exit__(self, exc_type, exc_value, traceback): def __exit__(self, exc_type, exc_value, traceback):
@ -453,14 +407,13 @@ class Profiler:
end = time.perf_counter() - self.pause_time end = time.perf_counter() - self.pause_time
self.pause_time = 0 self.pause_time = 0
thread_ident = threading.get_ident() thread_ident = threading.get_ident()
try:
if tag in Profiler.tags and thread_ident in Profiler.tags[tag]["threads"]: if tag in Profiler.tags and thread_ident in Profiler.tags[tag]["threads"]:
try: capture = Profiler.tags[tag]["threads"][thread_ident]["running"].pop() if Profiler.tags[tag]["threads"][thread_ident]["current_start"] != None:
except IndexError as e: return begin = Profiler.tags[tag]["threads"][thread_ident]["current_start"]
capture["end"] = end Profiler.tags[tag]["threads"][thread_ident]["current_start"] = None
Profiler.tags[tag]["threads"][thread_ident]["captures"].append(end-begin)
if not Profiler._ran: if not Profiler._ran:
Profiler._ran = True Profiler._ran = True
finally:
self.resume_super() self.resume_super()
def pause(self, pause_started=None): def pause(self, pause_started=None):
@ -480,153 +433,78 @@ class Profiler:
@staticmethod @staticmethod
def results(): def results():
def find_window_start(captures, start_time, hi=None): from statistics import mean, median, stdev
hi = len(captures) if hi is None else hi
idx = bisect.bisect_left(captures, start_time, hi=hi, key=lambda c: c["start"])
return idx if len(captures) - idx > 1 else None
# Fast one-pass calculation of summary statistics
def calc_stats(captures, start=0, end=None, key=lambda c: c, threads_key=None):
if end is None: end = len(captures)
while start > 0 and start < len(captures) and start < end and key(captures[start]) is None: start += 1
while end > 0 and end <= len(captures) and start < end and key(captures[end-1]) is None: end -= 1
count = end - start
if count <= 0: return None
elif count == 1:
capture = key(captures[start])
return { "count": 1,
"mean": capture,
"median": capture,
"min": capture,
"max": capture,
"stdev": None,
"sum": capture,
"threads": 1 if threads_key else None}
# Median is approximate if there are any incomplete captures in the
# middle of the range.
med_even = count % 2 == 0
med_idx = start + (count // 2 if med_even else (count - 1) // 2)
while key(captures[med_idx]) is None and med_idx < end: med_idx += 1
if med_idx == end: c_median = None
elif med_even: c_median = key(captures[med_idx])
else:
med_idx2 = med_idx + 1
while key(captures[med_idx2]) is None and med_idx2 < end: med_idx2 += 1
if med_idx2 == end: c_median = None
else:
med_cap1 = key(captures[med_idx])
med_cap2 = key(captures[med_idx+1])
if med_cap1 is None or med_cap2 is None: c_median = None
else: c_median = (med_cap1 + med_cap2) / 2
c_count = 0; c_sum = 0; c_min = key(captures[start]); c_max = key(captures[start]); ck = 0; ck2 = 0
uniq_threads = set()
for idx in range(start, end):
c = key(captures[idx])
if c is None: continue
else: c_count += 1
c_sum += c
if c < c_min: c_min = c
if c > c_max: c_max = c
ck += c - c_median
ck2 += (c - c_median) ** 2
if threads_key: uniq_threads.add(threads_key(captures[idx]))
c_mean = c_sum / c_count if c_count > 0 else None
c_std = math.sqrt((ck2 - (ck ** 2)/c_count) / (c_count - 1)) if c_count > 1 else None
return { "count": c_count,
"mean": c_mean,
"median": c_median,
"min": c_min,
"max": c_max,
"stdev": c_std,
"sum": c_sum,
"threads": len(uniq_threads) if threads_key else None }
def stats_key(capture):
end = capture["end"]
if end is None: return None
else: return end - capture["start"]
results = {} results = {}
now = time.perf_counter()
for tag in sorted(Profiler.tags): for tag in Profiler.tags:
tag_captures = []
tag_entry = Profiler.tags[tag] tag_entry = Profiler.tags[tag]
tag_captures = sorted(tag_entry["captures"], key=lambda c: c["start"])
if len(tag_captures): for thread_ident in tag_entry["threads"]:
captures_1m = None; captures_5m = None; captures_30m = None; captures_60m = None thread_entry = tag_entry["threads"][thread_ident]
stats_1m = None; stats_5m = None; stats_30m = None; stats_60m = None thread_captures = thread_entry["captures"]
sample_count = len(thread_captures)
captures_1m = find_window_start(tag_captures, now - 1*60) if sample_count > 1:
if captures_1m: captures_5m = find_window_start(tag_captures, now - 5*60, hi=captures_1m) thread_results = { "count": sample_count,
if captures_5m: captures_30m = find_window_start(tag_captures, now - 30*60, hi=captures_5m) "mean": mean(thread_captures),
if captures_30m: captures_60m = find_window_start(tag_captures, now - 60*60, hi=captures_30m) "median": median(thread_captures),
"stdev": stdev(thread_captures) }
stats_all = calc_stats(tag_captures, 0, None, key=stats_key, threads_key=lambda c: c["thread_ident"]) elif sample_count == 1:
if captures_1m: stats_1m = calc_stats(tag_captures, captures_1m, None, key=stats_key) thread_results = { "count": sample_count,
if captures_5m: stats_5m = calc_stats(tag_captures, captures_5m, captures_1m, key=stats_key) "mean": mean(thread_captures),
if captures_30m: stats_30m = calc_stats(tag_captures, captures_30m, captures_5m, key=stats_key) "median": median(thread_captures),
if captures_60m: stats_60m = calc_stats(tag_captures, captures_60m, captures_30m, key=stats_key) "stdev": None }
if stats_all["count"]: tag_captures.extend(thread_captures)
results[tag] = { "name": tag,
sample_count = len(tag_captures)
if sample_count > 1:
tag_results = { "name": tag,
"super": tag_entry["super"], "super": tag_entry["super"],
"stats_all": stats_all, "count": len(tag_captures),
"stats_1m": stats_1m, "mean": mean(tag_captures),
"stats_5m": stats_5m, "median": median(tag_captures),
"stats_30m": stats_30m, "stdev": stdev(tag_captures) }
"stats_60m": stats_60m }
# Yield to avoid bogging down the instance elif sample_count == 1:
time.sleep(0.001) tag_results = { "name": tag,
"super": tag_entry["super"],
"count": len(tag_captures),
"mean": mean(tag_captures),
"median": median(tag_captures),
"stdev": None }
return results results[tag] = tag_results
@staticmethod
def format_results(results):
def pst(time):
if time is not None: return prettyshorttime(time, tight=True)
else: return "-----"
def print_results_recursive(tag, results, level=0): def print_results_recursive(tag, results, level=0):
results_str = print_tag_results(tag, level+1) + "\n" print_tag_results(tag, level+1)
for tag_name in results: for tag_name in results:
sub_tag = results[tag_name] sub_tag = results[tag_name]
if sub_tag["super"] == tag["name"]: if sub_tag["super"] == tag["name"]:
results_str += print_results_recursive(sub_tag, results, level=level+1) print_results_recursive(sub_tag, results, level=level+1)
return results_str
def print_tag_results(tag, level): def print_tag_results(tag, level):
ind = " "*level ind = " "*level
name = tag["name"] name = tag["name"]; count = tag["count"]
stats_all = tag["stats_all"]; stats_1m = tag["stats_1m"]; stats_5m = tag["stats_5m"]; stats_30m = tag["stats_30m"]; stats_60m = tag["stats_60m"] mean = tag["mean"]; median = tag["median"]; stdev = tag["stdev"]
results_str = f" {ind}{name}\n" print( f"{ind}{name}")
results_str += f" {ind} Samples : {stats_all['count']} from {stats_all['threads']} thread{'s' if stats_all['threads'] > 1 else ''}\n" print( f"{ind} Samples : {count}")
if stats_all != None: if stdev != None:
results_str += f" {ind} {'Mean':^15} | {'Median':^15} | {'Min':^15} | {'Max':^15} | {'St. dev':^15} | {'Total':^15}\n" print(f"{ind} Mean : {prettyshorttime(mean)}")
results_str += f" {ind} Stats : ({pst(stats_all['mean']):^15} | {pst(stats_all['median']):^15} | {pst(stats_all['min']):^15} | {pst(stats_all['max']):^15} | {pst(stats_all['stdev']):^15} | {pst(stats_all['sum']):^15})\n" print(f"{ind} Median : {prettyshorttime(median)}")
if stats_1m != None: print(f"{ind} St.dev. : {prettyshorttime(stdev)}")
results_str += f" {ind} 0-1m : ({pst(stats_1m['mean']):^15} | {pst(stats_1m['median']):^15} | {pst(stats_1m['min']):^15} | {pst(stats_1m['max']):^15} | {pst(stats_1m['stdev']):^15} | {pst(stats_1m['sum']):^15})\n" print( f"{ind} Total : {prettyshorttime(mean*count)}")
if stats_5m != None: print("")
results_str += f" {ind} 1-5m : ({pst(stats_5m['mean']):^15} | {pst(stats_5m['median']):^15} | {pst(stats_5m['min']):^15} | {pst(stats_5m['max']):^15} | {pst(stats_5m['stdev']):^15} | {pst(stats_5m['sum']):^15})\n"
if stats_30m != None:
results_str += f" {ind} 5-30m : ({pst(stats_30m['mean']):^15} | {pst(stats_30m['median']):^15} | {pst(stats_30m['min']):^15} | {pst(stats_30m['max']):^15} | {pst(stats_30m['stdev']):^15} | {pst(stats_30m['sum']):^15})\n"
if stats_60m != None:
results_str += f" {ind} 30-60m : ({pst(stats_60m['mean']):^15} | {pst(stats_60m['median']):^15} | {pst(stats_60m['min']):^15} | {pst(stats_60m['max']):^15} | {pst(stats_60m['stdev']):^15} | {pst(stats_60m['sum']):^15})\n"
return results_str
results_str = "" print("\nProfiler results:\n")
for tag_name in results: for tag_name in results:
tag = results[tag_name] tag = results[tag_name]
if tag["super"] == None: if tag["super"] == None:
results_str += print_results_recursive(tag, results) print_results_recursive(tag, results)
return results_str
profile = Profiler.get_profiler profile = Profiler.get_profiler
@ -676,19 +554,3 @@ def bytes_to_b256(data):
if not type(data) == bytes: raise TypeError("Invalid input data for base256 encode") if not type(data) == bytes: raise TypeError("Invalid input data for base256 encode")
try: return [byte_to_b256(c) for c in data] try: return [byte_to_b256(c) for c in data]
except Exception as e: raise TypeError(f"Could not encode to base256: {e}") except Exception as e: raise TypeError(f"Could not encode to base256: {e}")
from .Reticulum import Reticulum
from .Identity import Identity
from .Link import Link, RequestReceipt
from .Channel import MessageBase
from .Buffer import Buffer, RawChannelReader, RawChannelWriter
from .Transport import Transport
from .Discovery import InterfaceAnnouncer
from .Destination import Destination
from .Packet import Packet
from .Packet import PacketReceipt
from .Resolver import Resolver
from .Resource import Resource, ResourceAdvertisement
from .Cryptography import HKDF
from .Cryptography import Hashes

View file

@ -1 +1 @@
__version__ = "1.6.1" __version__ = "1.2.8"

View file

@ -3,7 +3,5 @@ import glob
py_modules = glob.glob(os.path.dirname(__file__)+"/*.py") py_modules = glob.glob(os.path.dirname(__file__)+"/*.py")
pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc") pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
so_modules = glob.glob(os.path.dirname(__file__)+"/*.so") modules = py_modules+pyc_modules
pyd_modules = glob.glob(os.path.dirname(__file__)+"/*.pyd") __all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
modules = py_modules+pyc_modules+so_modules+pyd_modules
__all__ = list(set([os.path.basename(f).split(".")[0] for f in modules if not os.path.basename(f).split(".")[0] == "__init__"]))

View file

@ -6,12 +6,14 @@ from . import utils
from .log import logger from .log import logger
def parse_reply(data): def parse_reply(data):
if not data: raise ConnectionAbortedError("Empty response: SAM API went offline") if not data:
raise ConnectionAbortedError("Empty response: SAM API went offline")
try: try:
msg = sam.Message(data.decode().strip()) msg = sam.Message(data.decode().strip())
logger.debug("SAM reply: "+str(msg)) logger.debug("SAM reply: "+str(msg))
except: raise ConnectionAbortedError("Invalid SAM response") except:
raise ConnectionAbortedError("Invalid SAM response")
return msg return msg
@ -26,7 +28,8 @@ async def get_sam_socket(sam_address=sam.DEFAULT_ADDRESS, loop=None):
reader, writer = await asyncio.open_connection(*sam_address) reader, writer = await asyncio.open_connection(*sam_address)
writer.write(sam.hello("3.1", "3.1")) writer.write(sam.hello("3.1", "3.1"))
reply = parse_reply(await reader.readline()) reply = parse_reply(await reader.readline())
if reply.ok: return (reader, writer) if reply.ok:
return (reader, writer)
else: else:
writer.close() writer.close()
raise exceptions.SAM_EXCEPTIONS[reply["RESULT"]]() raise exceptions.SAM_EXCEPTIONS[reply["RESULT"]]()
@ -46,8 +49,10 @@ async def dest_lookup(domain, sam_address=sam.DEFAULT_ADDRESS,
writer.write(sam.naming_lookup(domain)) writer.write(sam.naming_lookup(domain))
reply = parse_reply(await reader.readline()) reply = parse_reply(await reader.readline())
writer.close() writer.close()
if reply.ok: return sam.Destination(reply["VALUE"]) if reply.ok:
else: raise exceptions.SAM_EXCEPTIONS[reply["RESULT"]]() return sam.Destination(reply["VALUE"])
else:
raise exceptions.SAM_EXCEPTIONS[reply["RESULT"]]()
async def new_destination(sam_address=sam.DEFAULT_ADDRESS, loop=None, async def new_destination(sam_address=sam.DEFAULT_ADDRESS, loop=None,
sig_type=sam.Destination.default_sig_type): sig_type=sam.Destination.default_sig_type):
@ -86,10 +91,15 @@ async def create_session(session_name, sam_address=sam.DEFAULT_ADDRESS,
""" """
logger.debug("Creating session {}".format(session_name)) logger.debug("Creating session {}".format(session_name))
if destination: if destination:
if type(destination) == sam.Destination: destination = destination if type(destination) == sam.Destination:
else: destination = sam.Destination(destination, has_private_key=True) destination = destination
else:
destination = sam.Destination(
destination, has_private_key=True)
dest_string = destination.private_key.base64 dest_string = destination.private_key.base64
else: dest_string = sam.TRANSIENT_DESTINATION else:
dest_string = sam.TRANSIENT_DESTINATION
options = " ".join(["{}={}".format(k, v) for k, v in options.items()]) options = " ".join(["{}={}".format(k, v) for k, v in options.items()])
@ -99,7 +109,9 @@ async def create_session(session_name, sam_address=sam.DEFAULT_ADDRESS,
reply = parse_reply(await reader.readline()) reply = parse_reply(await reader.readline())
if reply.ok: if reply.ok:
if not destination: destination = sam.Destination(reply["DESTINATION"], has_private_key=True) if not destination:
destination = sam.Destination(
reply["DESTINATION"], has_private_key=True)
logger.debug(destination.base32) logger.debug(destination.base32)
logger.debug("Session created {}".format(session_name)) logger.debug("Session created {}".format(session_name))
return (reader, writer) return (reader, writer)
@ -118,8 +130,10 @@ async def stream_connect(session_name, destination,
:return: A (reader, writer) pair :return: A (reader, writer) pair
""" """
logger.debug("Connecting stream {}".format(session_name)) logger.debug("Connecting stream {}".format(session_name))
if isinstance(destination, str) and not destination.endswith(".i2p"): destination = sam.Destination(destination) if isinstance(destination, str) and not destination.endswith(".i2p"):
elif isinstance(destination, str): destination = await dest_lookup(destination, sam_address, loop) destination = sam.Destination(destination)
elif isinstance(destination, str):
destination = await dest_lookup(destination, sam_address, loop)
reader, writer = await get_sam_socket(sam_address, loop) reader, writer = await get_sam_socket(sam_address, loop)
writer.write(sam.stream_connect(session_name, destination.base64, writer.write(sam.stream_connect(session_name, destination.base64,
@ -144,7 +158,8 @@ async def stream_accept(session_name, sam_address=sam.DEFAULT_ADDRESS,
reader, writer = await get_sam_socket(sam_address, loop) reader, writer = await get_sam_socket(sam_address, loop)
writer.write(sam.stream_accept(session_name, silent="false")) writer.write(sam.stream_accept(session_name, silent="false"))
reply = parse_reply(await reader.readline()) reply = parse_reply(await reader.readline())
if reply.ok: return (reader, writer) if reply.ok:
return (reader, writer)
else: else:
writer.close() writer.close()
raise exceptions.SAM_EXCEPTIONS[reply["RESULT"]]() raise exceptions.SAM_EXCEPTIONS[reply["RESULT"]]()
@ -168,9 +183,9 @@ class Session:
:return: :class:`Session` object :return: :class:`Session` object
""" """
def __init__(self, session_name, sam_address=sam.DEFAULT_ADDRESS, def __init__(self, session_name, sam_address=sam.DEFAULT_ADDRESS,
loop=None, style="STREAM", signature_type=sam.Destination.default_sig_type, loop=None, style="STREAM",
signature_type=sam.Destination.default_sig_type,
destination=None, options={}): destination=None, options={}):
self.session_name = session_name self.session_name = session_name
self.sam_address = sam_address self.sam_address = sam_address
self.loop = loop self.loop = loop
@ -180,8 +195,9 @@ class Session:
self.options = options self.options = options
async def __aenter__(self): async def __aenter__(self):
self.reader, self.writer = await create_session(self.session_name, sam_address=self.sam_address, loop=self.loop, self.reader, self.writer = await create_session(self.session_name,
style=self.style, signature_type=self.signature_type, sam_address=self.sam_address, loop=self.loop, style=self.style,
signature_type=self.signature_type,
destination=self.destination, options=self.options) destination=self.destination, options=self.options)
return self return self
@ -198,15 +214,16 @@ class StreamConnection:
:param loop: (optional) Event loop instance :param loop: (optional) Event loop instance
:return: :class:`StreamConnection` object :return: :class:`StreamConnection` object
""" """
def __init__(self, session_name, destination, sam_address=sam.DEFAULT_ADDRESS, loop=None): def __init__(self, session_name, destination,
sam_address=sam.DEFAULT_ADDRESS, loop=None):
self.session_name = session_name self.session_name = session_name
self.sam_address = sam_address self.sam_address = sam_address
self.loop = loop self.loop = loop
self.destination = destination self.destination = destination
async def __aenter__(self): async def __aenter__(self):
self.reader, self.writer = await stream_connect(self.session_name, self.destination, self.reader, self.writer = await stream_connect(self.session_name,
sam_address=self.sam_address, loop=self.loop) self.destination, sam_address=self.sam_address, loop=self.loop)
self.read = self.reader.read self.read = self.reader.read
self.write = self.writer.write self.write = self.writer.write
return self return self
@ -223,13 +240,15 @@ class StreamAcceptor:
:param loop: (optional) Event loop instance :param loop: (optional) Event loop instance
:return: :class:`StreamAcceptor` object :return: :class:`StreamAcceptor` object
""" """
def __init__(self, session_name, sam_address=sam.DEFAULT_ADDRESS, loop=None): def __init__(self, session_name, sam_address=sam.DEFAULT_ADDRESS,
loop=None):
self.session_name = session_name self.session_name = session_name
self.sam_address = sam_address self.sam_address = sam_address
self.loop = loop self.loop = loop
async def __aenter__(self): async def __aenter__(self):
self.reader, self.writer = await stream_accept(self.session_name, sam_address=self.sam_address, loop=self.loop) self.reader, self.writer = await stream_accept(self.session_name,
sam_address=self.sam_address, loop=self.loop)
self.read = self.reader.read self.read = self.reader.read
self.write = self.writer.write self.write = self.writer.write
return self return self

View file

@ -30,7 +30,8 @@ class PeerNotFound(SAMException):
class Timeout(SAMException): class Timeout(SAMException):
"""The peer cannot be found on the network""" """The peer cannot be found on the network"""
SAM_EXCEPTIONS = { "CANT_REACH_PEER": CantReachPeer, SAM_EXCEPTIONS = {
"CANT_REACH_PEER": CantReachPeer,
"DUPLICATED_DEST": DuplicatedDest, "DUPLICATED_DEST": DuplicatedDest,
"DUPLICATED_ID": DuplicatedId, "DUPLICATED_ID": DuplicatedId,
"I2P_ERROR": I2PError, "I2P_ERROR": I2PError,
@ -38,4 +39,6 @@ SAM_EXCEPTIONS = { "CANT_REACH_PEER": CantReachPeer,
"INVALID_KEY": InvalidKey, "INVALID_KEY": InvalidKey,
"KEY_NOT_FOUND": KeyNotFound, "KEY_NOT_FOUND": KeyNotFound,
"PEER_NOT_FOUND": PeerNotFound, "PEER_NOT_FOUND": PeerNotFound,
"TIMEOUT": Timeout } "TIMEOUT": Timeout,
}

View file

@ -3,6 +3,7 @@ from hashlib import sha256
import struct import struct
import re import re
I2P_B64_CHARS = "-~" I2P_B64_CHARS = "-~"
def i2p_b64encode(x): def i2p_b64encode(x):
@ -40,9 +41,11 @@ class Message(object):
return self.opts[key] return self.opts[key]
@property @property
def ok(self): return self["RESULT"] == "OK" def ok(self):
return self["RESULT"] == "OK"
def __repr__(self): return self._reply_string def __repr__(self):
return self._reply_string
# SAM request messages # SAM request messages

View file

@ -15,13 +15,15 @@ async def proxy_data(reader, writer):
while True: while True:
data = await reader.read(BUFFER_SIZE) data = await reader.read(BUFFER_SIZE)
if not data: if not data:
logger.debug('no more data')
break break
writer.write(data) writer.write(data)
except Exception as e: logger.debug('proxy_data_task exception {}'.format(e)) except Exception as e:
logger.debug('proxy_data_task exception {}'.format(e))
finally: finally:
try: writer.close() try:
except RuntimeError: pass writer.close()
except RuntimeError:
pass
logger.debug('close connection') logger.debug('close connection')
class I2PTunnel(object): class I2PTunnel(object):
@ -42,7 +44,6 @@ class I2PTunnel(object):
def __init__(self, local_address, destination=None, session_name=None, def __init__(self, local_address, destination=None, session_name=None,
options={}, loop=None, sam_address=sam.DEFAULT_ADDRESS): options={}, loop=None, sam_address=sam.DEFAULT_ADDRESS):
self.local_address = local_address self.local_address = local_address
self.destination = destination self.destination = destination
self.session_name = session_name or utils.generate_session_id() self.session_name = session_name or utils.generate_session_id()
@ -52,13 +53,16 @@ class I2PTunnel(object):
async def _pre_run(self): async def _pre_run(self):
if not self.destination: if not self.destination:
self.destination = await aiosam.new_destination(sam_address=self.sam_address, loop=self.loop) self.destination = await aiosam.new_destination(
sam_address=self.sam_address, loop=self.loop)
_, self.session_writer = await aiosam.create_session(self.session_name, style=self.style, options=self.options, _, self.session_writer = await aiosam.create_session(
self.session_name, style=self.style, options=self.options,
sam_address=self.sam_address, sam_address=self.sam_address,
loop=self.loop, destination=self.destination) loop=self.loop, destination=self.destination)
def stop(self): self.session_writer.close() def stop(self):
"""Stop the tunnel"""
self.session_writer.close()
class ClientTunnel(I2PTunnel): class ClientTunnel(I2PTunnel):
"""Client tunnel, a subclass of tunnel.I2PTunnel """Client tunnel, a subclass of tunnel.I2PTunnel
@ -81,25 +85,20 @@ class ClientTunnel(I2PTunnel):
"""A coroutine used to run the tunnel""" """A coroutine used to run the tunnel"""
await self._pre_run() await self._pre_run()
self.status = { "setup_ran": False, "setup_failed": False, "exception": None, self.status = { "setup_ran": False, "setup_failed": False, "exception": None, "connect_tasks": [] }
"connect_tasks": set(), "background_tasks": set() }
async def handle_client(client_reader, client_writer): async def handle_client(client_reader, client_writer):
"""Handle local client connection""" """Handle local client connection"""
try: try:
sc_task = aiosam.stream_connect(self.session_name, self.remote_destination, sc_task = aiosam.stream_connect(
self.session_name, self.remote_destination,
sam_address=self.sam_address, loop=self.loop) sam_address=self.sam_address, loop=self.loop)
self.status["connect_tasks"].append(sc_task)
self.status["connect_tasks"].add(sc_task)
remote_reader, remote_writer = await sc_task remote_reader, remote_writer = await sc_task
task = asyncio.ensure_future(proxy_data(remote_reader, client_writer), loop=self.loop) asyncio.ensure_future(proxy_data(remote_reader, client_writer),
self.status["background_tasks"].add(task) loop=self.loop)
task.add_done_callback(self.status["background_tasks"].discard) asyncio.ensure_future(proxy_data(client_reader, remote_writer),
loop=self.loop)
task = asyncio.ensure_future(proxy_data(client_reader, remote_writer), loop=self.loop)
self.status["background_tasks"].add(task)
task.add_done_callback(self.status["background_tasks"].discard)
except Exception as e: except Exception as e:
self.status["setup_ran"] = True self.status["setup_ran"] = True
@ -136,15 +135,14 @@ class ServerTunnel(I2PTunnel):
"""A coroutine used to run the tunnel""" """A coroutine used to run the tunnel"""
await self._pre_run() await self._pre_run()
self.status = { "setup_ran": False, "setup_failed": False, "exception": None, self.status = { "setup_ran": False, "setup_failed": False, "exception": None, "connect_tasks": [] }
"connect_tasks": set(), "background_tasks": set() }
async def handle_client(incoming, client_reader, client_writer): async def handle_client(incoming, client_reader, client_writer):
try: try:
# data and dest may come in one chunk # data and dest may come in one chunk
dest, data = incoming.split(b"\n", 1) dest, data = incoming.split(b"\n", 1)
remote_destination = sam.Destination(dest.decode()) remote_destination = sam.Destination(dest.decode())
logger.debug("{} client connected: {}.b32.i2p".format(self.session_name, remote_destination.base32)) logger.debug("{} client connected: {}.b32.i2p".format(
self.session_name, remote_destination.base32))
except Exception as e: except Exception as e:
self.status["exception"] = e self.status["exception"] = e
@ -152,20 +150,21 @@ class ServerTunnel(I2PTunnel):
data = None data = None
try: try:
sc_task = asyncio.wait_for(asyncio.open_connection(host=self.local_address[0], port=self.local_address[1]), timeout=5) sc_task = asyncio.wait_for(
self.status["connect_tasks"].add(sc_task) asyncio.open_connection(
host=self.local_address[0],
port=self.local_address[1]),
timeout=5)
self.status["connect_tasks"].append(sc_task)
remote_reader, remote_writer = await sc_task remote_reader, remote_writer = await sc_task
if data: remote_writer.write(data) if data: remote_writer.write(data)
task = asyncio.ensure_future(proxy_data(remote_reader, client_writer), loop=self.loop) asyncio.ensure_future(proxy_data(remote_reader, client_writer),
self.status["background_tasks"].add(task) loop=self.loop)
task.add_done_callback(self.status["background_tasks"].discard) asyncio.ensure_future(proxy_data(client_reader, remote_writer),
loop=self.loop)
task = asyncio.ensure_future(proxy_data(client_reader, remote_writer), loop=self.loop) except ConnectionRefusedError:
self.status["background_tasks"].add(task)
task.add_done_callback(self.status["background_tasks"].discard)
except ConnectionRefusedError as e:
client_writer.close() client_writer.close()
self.status["exception"] = e self.status["exception"] = e
self.status["setup_failed"] = True self.status["setup_failed"] = True
@ -173,14 +172,14 @@ class ServerTunnel(I2PTunnel):
async def server_loop(): async def server_loop():
try: try:
while True: while True:
client_reader, client_writer = await aiosam.stream_accept(self.session_name, sam_address=self.sam_address, client_reader, client_writer = await aiosam.stream_accept(
self.session_name, sam_address=self.sam_address,
loop=self.loop) loop=self.loop)
incoming = await client_reader.read(BUFFER_SIZE) incoming = await client_reader.read(BUFFER_SIZE)
task = asyncio.ensure_future(handle_client(incoming, client_reader, client_writer), loop=self.loop) asyncio.ensure_future(handle_client(
self.status["background_tasks"].add(task) incoming, client_reader, client_writer), loop=self.loop)
task.add_done_callback(self.status["background_tasks"].discard) except asyncio.CancelledError:
pass
except asyncio.CancelledError: pass
self.server_loop = asyncio.ensure_future(server_loop(), loop=self.loop) self.server_loop = asyncio.ensure_future(server_loop(), loop=self.loop)
self.status["setup_ran"] = True self.status["setup_ran"] = True
@ -192,11 +191,16 @@ class ServerTunnel(I2PTunnel):
if __name__ == '__main__': if __name__ == '__main__':
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('type', metavar="TYPE", choices=('server', 'client'), help="Tunnel type (server or client)") parser.add_argument('type', metavar="TYPE", choices=('server', 'client'),
parser.add_argument('address', metavar="ADDRESS", help="Local address (e.g. 127.0.0.1:8000)") help="Tunnel type (server or client)")
parser.add_argument('--debug', '-d', action='store_true', help='Debugging') parser.add_argument('address', metavar="ADDRESS",
parser.add_argument('--key', '-k', default='', metavar='PRIVATE_KEY', help='Path to private key file') help="Local address (e.g. 127.0.0.1:8000)")
parser.add_argument('--destination', '-D', default='', metavar='DESTINATION', help='Remote destination') parser.add_argument('--debug', '-d', action='store_true',
help='Debugging')
parser.add_argument('--key', '-k', default='', metavar='PRIVATE_KEY',
help='Path to private key file')
parser.add_argument('--destination', '-D', default='',
metavar='DESTINATION', help='Remote destination')
args = parser.parse_args() args = parser.parse_args()
SAM_ADDRESS = utils.get_sam_address() SAM_ADDRESS = utils.get_sam_address()
@ -205,21 +209,26 @@ if __name__ == '__main__':
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
loop.set_debug(args.debug) loop.set_debug(args.debug)
if args.key: destination = sam.Destination(path=args.key, has_private_key=True) if args.key:
else: destination = None destination = sam.Destination(path=args.key, has_private_key=True)
else:
destination = None
local_address = utils.address_from_string(args.address) local_address = utils.address_from_string(args.address)
if args.type == "client": if args.type == "client":
tunnel = ClientTunnel(args.destination, local_address, loop=loop, destination=destination, sam_address=SAM_ADDRESS) tunnel = ClientTunnel(args.destination, local_address, loop=loop,
destination=destination, sam_address=SAM_ADDRESS)
elif args.type == "server": elif args.type == "server":
tunnel = ServerTunnel(local_address, loop=loop, destination=destination, sam_address=SAM_ADDRESS) tunnel = ServerTunnel(local_address, loop=loop, destination=destination,
sam_address=SAM_ADDRESS)
asyncio.ensure_future(tunnel.run(), loop=loop) asyncio.ensure_future(tunnel.run(), loop=loop)
try: loop.run_forever() try:
except KeyboardInterrupt: tunnel.stop() loop.run_forever()
except KeyboardInterrupt:
tunnel.stop()
finally: finally:
loop.stop() loop.stop()
loop.close() loop.close()

213
RNS/vendor/umsgpack.py vendored
View file

@ -2,7 +2,7 @@
# https://github.com/vsergeev/u-msgpack-python # https://github.com/vsergeev/u-msgpack-python
# #
# u-msgpack-python is a lightweight MessagePack serializer and deserializer # u-msgpack-python is a lightweight MessagePack serializer and deserializer
# module, compatible with Python 3, as well CPython and PyPy # module, compatible with both Python 2 and 3, as well CPython and PyPy
# implementations of Python. u-msgpack-python is fully compliant with the # implementations of Python. u-msgpack-python is fully compliant with the
# latest MessagePack specification.com/msgpack/msgpack/blob/master/spec.md). In # latest MessagePack specification.com/msgpack/msgpack/blob/master/spec.md). In
# particular, it supports the new binary, UTF-8 string, and application ext # particular, it supports the new binary, UTF-8 string, and application ext
@ -35,7 +35,7 @@ u-msgpack-python v2.7.1 - v at sergeev.io
https://github.com/vsergeev/u-msgpack-python https://github.com/vsergeev/u-msgpack-python
u-msgpack-python is a lightweight MessagePack serializer and deserializer u-msgpack-python is a lightweight MessagePack serializer and deserializer
module, compatible with Python 3, as well CPython and PyPy module, compatible with both Python 2 and 3, as well CPython and PyPy
implementations of Python. u-msgpack-python is fully compliant with the implementations of Python. u-msgpack-python is fully compliant with the
latest MessagePack specification.com/msgpack/msgpack/blob/master/spec.md). In latest MessagePack specification.com/msgpack/msgpack/blob/master/spec.md). In
particular, it supports the new binary, UTF-8 string, and application ext particular, it supports the new binary, UTF-8 string, and application ext
@ -49,7 +49,10 @@ import datetime
import sys import sys
import io import io
if sys.version_info[0:2] >= (3, 3):
from collections.abc import Hashable from collections.abc import Hashable
else:
from collections import Hashable
__version__ = "2.7.1" __version__ = "2.7.1"
"Module version string" "Module version string"
@ -101,6 +104,8 @@ class Ext(object):
# Check data is type bytes or str # Check data is type bytes or str
elif sys.version_info[0] == 3 and not isinstance(data, bytes): elif sys.version_info[0] == 3 and not isinstance(data, bytes):
raise TypeError("ext data is not type \'bytes\'") raise TypeError("ext data is not type \'bytes\'")
elif sys.version_info[0] == 2 and not isinstance(data, str):
raise TypeError("ext data is not type \'str\'")
self.type = type self.type = type
self.data = data self.data = data
@ -124,7 +129,7 @@ class Ext(object):
""" """
s = "Ext Object (Type: {:d}, Data: ".format(self.type) s = "Ext Object (Type: {:d}, Data: ".format(self.type)
s += " ".join(["0x{:02}".format(ord(self.data[i:i + 1])) s += " ".join(["0x{:02}".format(ord(self.data[i:i + 1]))
for i in range(min(len(self.data), 8))]) for i in xrange(min(len(self.data), 8))])
if len(self.data) > 8: if len(self.data) > 8:
s += " ..." s += " ..."
s += ")" s += ")"
@ -447,6 +452,94 @@ def _pack_map(obj, fp, options):
######################################## ########################################
# Pack for Python 2, with 'unicode' type, 'str' type, and 'long' type
def _pack2(obj, fp, **options):
"""
Serialize a Python object into MessagePack bytes.
Args:
obj: a Python object
fp: a .write()-supporting file-like object
Kwargs:
ext_handlers (dict): dictionary of Ext handlers, mapping a custom type
to a callable that packs an instance of the type
into an Ext object
force_float_precision (str): "single" to force packing floats as
IEEE-754 single-precision floats,
"double" to force packing floats as
IEEE-754 double-precision floats.
Returns:
None.
Raises:
UnsupportedType(PackException):
Object type not supported for packing.
Example:
>>> f = open('test.bin', 'wb')
>>> umsgpack.pack({u"compact": True, u"schema": 0}, f)
>>>
"""
global compatibility
ext_handlers = options.get("ext_handlers")
if obj is None:
_pack_nil(obj, fp, options)
elif ext_handlers and obj.__class__ in ext_handlers:
_pack_ext(ext_handlers[obj.__class__](obj), fp, options)
elif obj.__class__ in _ext_class_to_type:
try:
_pack_ext(Ext(_ext_class_to_type[obj.__class__], obj.packb()), fp, options)
except AttributeError:
raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(obj.__class__)))
elif isinstance(obj, bool):
_pack_boolean(obj, fp, options)
elif isinstance(obj, (int, long)):
_pack_integer(obj, fp, options)
elif isinstance(obj, float):
_pack_float(obj, fp, options)
elif compatibility and isinstance(obj, unicode):
_pack_oldspec_raw(bytes(obj), fp, options)
elif compatibility and isinstance(obj, bytes):
_pack_oldspec_raw(obj, fp, options)
elif isinstance(obj, unicode):
_pack_string(obj, fp, options)
elif isinstance(obj, str):
_pack_binary(obj, fp, options)
elif isinstance(obj, (list, tuple)):
_pack_array(obj, fp, options)
elif isinstance(obj, dict):
_pack_map(obj, fp, options)
elif isinstance(obj, datetime.datetime):
_pack_ext_timestamp(obj, fp, options)
elif isinstance(obj, Ext):
_pack_ext(obj, fp, options)
elif ext_handlers:
# Linear search for superclass
t = next((t for t in ext_handlers.keys() if isinstance(obj, t)), None)
if t:
_pack_ext(ext_handlers[t](obj), fp, options)
else:
raise UnsupportedTypeException(
"unsupported type: {:s}".format(str(type(obj))))
elif _ext_class_to_type:
# Linear search for superclass
t = next((t for t in _ext_class_to_type if isinstance(obj, t)), None)
if t:
try:
_pack_ext(Ext(_ext_class_to_type[t], obj.packb()), fp, options)
except AttributeError:
raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(t)))
else:
raise UnsupportedTypeException("unsupported type: {:s}".format(str(type(obj))))
else:
raise UnsupportedTypeException("unsupported type: {:s}".format(str(type(obj))))
# Pack for Python 3, with unicode 'str' type, 'bytes' type, and no 'long' type # Pack for Python 3, with unicode 'str' type, 'bytes' type, and no 'long' type
def _pack3(obj, fp, **options): def _pack3(obj, fp, **options):
""" """
@ -534,6 +627,40 @@ def _pack3(obj, fp, **options):
raise UnsupportedTypeException( raise UnsupportedTypeException(
"unsupported type: {:s}".format(str(type(obj)))) "unsupported type: {:s}".format(str(type(obj))))
def _packb2(obj, **options):
"""
Serialize a Python object into MessagePack bytes.
Args:
obj: a Python object
Kwargs:
ext_handlers (dict): dictionary of Ext handlers, mapping a custom type
to a callable that packs an instance of the type
into an Ext object
force_float_precision (str): "single" to force packing floats as
IEEE-754 single-precision floats,
"double" to force packing floats as
IEEE-754 double-precision floats.
Returns:
A 'str' containing serialized MessagePack bytes.
Raises:
UnsupportedType(PackException):
Object type not supported for packing.
Example:
>>> umsgpack.packb({u"compact": True, u"schema": 0})
'\x82\xa7compact\xc3\xa6schema\x00'
>>>
"""
fp = io.BytesIO()
_pack2(obj, fp, **options)
return fp.getvalue()
def _packb3(obj, **options): def _packb3(obj, **options):
""" """
Serialize a Python object into MessagePack bytes. Serialize a Python object into MessagePack bytes.
@ -758,9 +885,9 @@ def _unpack_array(code, fp, options):
raise Exception("logic error, not array: 0x{:02x}".format(ord(code))) raise Exception("logic error, not array: 0x{:02x}".format(ord(code)))
if options.get('use_tuple'): if options.get('use_tuple'):
return tuple((_unpack(fp, options) for i in range(length))) return tuple((_unpack(fp, options) for i in xrange(length)))
return [_unpack(fp, options) for i in range(length)] return [_unpack(fp, options) for i in xrange(length)]
def _deep_list_to_tuple(obj): def _deep_list_to_tuple(obj):
@ -780,7 +907,7 @@ def _unpack_map(code, fp, options):
raise Exception("logic error, not map: 0x{:02x}".format(ord(code))) raise Exception("logic error, not map: 0x{:02x}".format(ord(code)))
d = {} if not options.get('use_ordered_dict') else collections.OrderedDict() d = {} if not options.get('use_ordered_dict') else collections.OrderedDict()
for _ in range(length): for _ in xrange(length):
# Unpack key # Unpack key
k = _unpack(fp, options) k = _unpack(fp, options)
@ -811,6 +938,53 @@ def _unpack(fp, options):
######################################## ########################################
def _unpack2(fp, **options):
"""
Deserialize MessagePack bytes into a Python object.
Args:
fp: a .read()-supporting file-like object
Kwargs:
ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext
type to a callable that unpacks an instance of
Ext into an object
use_ordered_dict (bool): unpack maps into OrderedDict, instead of
unordered dict (default False)
use_tuple (bool): unpacks arrays into tuples, instead of lists (default
False)
allow_invalid_utf8 (bool): unpack invalid strings into instances of
InvalidString, for access to the bytes
(default False)
Returns:
A Python object.
Raises:
InsufficientDataException(UnpackException):
Insufficient data to unpack the serialized object.
InvalidStringException(UnpackException):
Invalid UTF-8 string encountered during unpacking.
UnsupportedTimestampException(UnpackException):
Unsupported timestamp format encountered during unpacking.
ReservedCodeException(UnpackException):
Reserved code encountered during unpacking.
UnhashableKeyException(UnpackException):
Unhashable key encountered during map unpacking.
The serialized map cannot be deserialized into a Python dictionary.
DuplicateKeyException(UnpackException):
Duplicate key encountered during map unpacking.
Example:
>>> f = open('test.bin', 'rb')
>>> umsgpack.unpackb(f)
{u'compact': True, u'schema': 0}
>>>
"""
return _unpack(fp, options)
def _unpack3(fp, **options): def _unpack3(fp, **options):
""" """
Deserialize MessagePack bytes into a Python object. Deserialize MessagePack bytes into a Python object.
@ -975,11 +1149,27 @@ def __init():
global _utc_tzinfo global _utc_tzinfo
global _float_precision global _float_precision
global _unpack_dispatch_table global _unpack_dispatch_table
global xrange
# Compatibility mode for handling strings/bytes with the old specification # Compatibility mode for handling strings/bytes with the old specification
compatibility = False compatibility = False
if sys.version_info[0] == 3:
_utc_tzinfo = datetime.timezone.utc _utc_tzinfo = datetime.timezone.utc
else:
class UTC(datetime.tzinfo):
ZERO = datetime.timedelta(0)
def utcoffset(self, dt):
return UTC.ZERO
def tzname(self, dt):
return "UTC"
def dst(self, dt):
return UTC.ZERO
_utc_tzinfo = UTC()
# Calculate an aware epoch datetime # Calculate an aware epoch datetime
_epoch = datetime.datetime(1970, 1, 1, tzinfo=_utc_tzinfo) _epoch = datetime.datetime(1970, 1, 1, tzinfo=_utc_tzinfo)
@ -991,6 +1181,7 @@ def __init():
_float_precision = "single" _float_precision = "single"
# Map packb and unpackb to the appropriate version # Map packb and unpackb to the appropriate version
if sys.version_info[0] == 3:
pack = _pack3 pack = _pack3
packb = _packb3 packb = _packb3
dump = _pack3 dump = _pack3
@ -999,6 +1190,16 @@ def __init():
unpackb = _unpackb3 unpackb = _unpackb3
load = _unpack3 load = _unpack3
loads = _unpackb3 loads = _unpackb3
xrange = range
else:
pack = _pack2
packb = _packb2
dump = _pack2
dumps = _packb2
unpack = _unpack2
unpackb = _unpackb2
load = _unpack2
loads = _unpackb2
# Build a dispatch table for fast lookup of unpacking function # Build a dispatch table for fast lookup of unpacking function

94
Roadmap.md Normal file
View file

@ -0,0 +1,94 @@
# Reticulum Development Roadmap
This document outlines the currently established development roadmap for Reticulum.
1. [Currently Active Work Areas](#currently-active-work-areas)
2. [Primary Efforts](#primary-efforts)
- [Comprehensibility](#comprehensibility)
- [Universality](#universality)
- [Functionality](#functionality)
- [Usability & Utility](#usability--utility)
- [Interfaceability](#interfaceability)
3. [Auxillary Efforts](#auxillary-efforts)
4. [Release History](#release-history)
## Currently Active Work Areas
For each release cycle of Reticulum, improvements and additions from the five [Primary Efforts](#primary-efforts) are selected as active work areas, and can be expected to be included in the upcoming releases within that cycle. While not entirely set in stone for each release cycle, they serve as a pointer of what to expect in the near future.
## Primary Efforts
The development path for Reticulum is currently laid out in five distinct areas: *Comprehensibility*, *Universality*, *Functionality*, *Usability & Utility* and *Interfaceability*. Conceptualising the development of Reticulum into these areas serves to advance the implementation and work towards the Foundational Goals & Values of Reticulum.
### Comprehensibility
These efforts are aimed at improving the ease of which Reticulum is understood, and lowering the barrier to entry for people who wish to start building systems on Reticulum.
- Improving [the manual](https://markqvist.github.io/Reticulum/manual/) with tutorials specifically for beginners
- Updating the documentation to reflect recent changes and improvements
- Update descriptions of protocol mechanics
- Update announce description
- Add in-depth explanation of the IFAC system
- Software
- Update software descriptions and screenshots
- Communications hardware section
- Add information about RNode external displays.
- Possibly add other relevant types here as well.
- Setup *Best Practices For...* / *Installation Examples* section.
- Home or office (example)
- Vehicles (example)
- No-grid/solar/remote sites (example)
### Universality
These efforts seek to broaden the universality of the Reticulum software and hardware ecosystem by continously diversifying platform support, and by improving the overall availability and ease of deployment of the Reticulum stack.
- Create a standalone RNS Daemon app for Android
- A lightweight and portable C implementation for microcontrollers, µRNS
- A portable, high-performance Reticulum implementation in C/C++, see [#21](https://github.com/markqvist/Reticulum/discussions/21)
- Bindings for other programming languages
### Functionality
These efforts aim to expand and improve the core functionality and reliability of Reticulum.
- Add interface hot-plug and live up/down control to running instances
- Add automatic retries to all use cases of the `Request` API
- Network-wide path balancing
- Distributed Destination Naming System
- Globally routable multicast
- Destination proxying
- [Metric-based path selection and multiple paths](https://github.com/markqvist/Reticulum/discussions/86)
### Usability & Utility
These efforts seek to make Reticulum easier to use and operate, and to expand the utility of the stack on deployed systems.
- Easy way to share interface configurations, see [#19](https://github.com/markqvist/Reticulum/discussions/19)
- Transit traffic display in `rnstatus`
- `rnsconfig` utility
### Interfaceability
These efforts aim to expand the types of physical and virtual interfaces that Reticulum can natively use to transport data.
- Plain ESP32 devices (ESP-Now, WiFi, Bluetooth, etc.)
- More LoRa transceivers
- AT-compatible modems
- Filesystem interface
- Direct SDR Support
- Optical mediums
- IR Transceivers
- AWDL / OWL
- HF Modems
- GNU Radio
- CAN-bus
- Raw SPI
- Raw i²c
- MQTT
- XBee
- Tor
## Auxillary Efforts
The Reticulum ecosystem is enriched by several other software and hardware projects, and the support and improvement of these, in symbiosis with the core Reticulum project helps expand the reach and utility of Reticulum itself.
This section lists, in no particular order, various important efforts that would be beneficial to the goals of Reticulum.
- The [RNode](https://unsigned.io/rnode/) project
- [x] Create a WebUSB-based bootstrapping utility, and integrate this directly into the [RNode Bootstrap Console](#), both on-device, and on an Internet-reachable copy. This will make it much easier to create new RNodes for average users.
## Release History
Please see the [Changelog](./Changelog.md) for a complete release history and changelog of Reticulum.

View file

@ -1,7 +0,0 @@
# Reporting Security Issues
Security notices that affect Reticulum are published [in-repository in the docs/security directory](./docs/security).
It goes without saying: If you have discovered a security issue in Reticulum or any related project, please report it responsibly to the maintainer over LXMF at `<8dd57a738226809646089335a6b03695>`. If you wish to remain anonymous, please create a disposable identity to send from, and notify us of any such wishes.
If you are not able to use LXMF, you can send an email to the address currently posted on the maintainer's profile on this site.

View file

@ -1,68 +0,0 @@
Recently, and mostly from people who I've never seen before, the opinions about how this project should be run has started flooding in again. In a recent forum thread of such opinions, specifically about:
- The decision to no longer mirror release notes on GitHub.
- Some people feeling there were too many "barriers to entry" to joining RNS development.
- The project not really being "[open source](https://reticulum.network/manual/brandolinis.html#open-source-means-open-source)" because random strangers couldn't just "contribute".
Joakim posted some very relevant observations about how Reticulum operates, along with the following quote:
> The modern industrial system has a built-in tendency to grow; it cannot really work unless it is growing. The word “stability” has been struck from its dictionary and replaced by “stagnation”. Its continuous growth pursues no particular aims or objectives: it is growth for the sake of growing. No one even enquires after its final shape. There is none; there is no “saturation point”.
That E. F. Schumacher quote perfectly illustrates the ontological schism that makes it so tiresome to deal with stuff like this.
There is, in this day and age, between different people, widely different base conceptual integrations of what "open source" means. For many people, "open source" has become synonymous **not** with skilled people working together in a coordinated and careful way on complex engineering challenges, but a sort of growth- and attention-focused "free-for-all" *behavioral* codex that must be followed above all else; a *social* modus operandi of fake inclusivity where everyone "should have their voice heard", and adherence to that specific *process* is weighed much higher than the final results.
I do not subscribe to, and consequently do not operate the Reticulum project under *any* versions of that idea.
**Here's the statistical, boring reality:**
- Around 90% of pull requests and "recommendations" I received when people could just submit stuff via GitHub would
have *severely* broken things, introduced bugs or security issues, created roadblocks for future work, or otherwise
damaged the software. Usually just for the sake of satisfying a random newcomers "idea" or personal preference.
- Similarly, around 90% "bug reports" were actually people asking for help, because of having failed to read even the
most basic parts of the documentation.
- The people with the least amount of understanding, skill and effort invested tend to be loudest and most vocal. When
all you have is "opinions", those are iterated upon ad infinitum, apparently.
Can you imagine how much time that wasted? Can you imagine what we could have accomplished with that time instead?
The only thing that this creates is *noise* and confusion. Clogging up the mental and physical workspaces, of people who are actually investing time and effort on the project, with stuff like that is objectively just taking time that could have been used on development, and replacing it with *nothing*.
I was receiving *actual* bug reports, pull requests, proper technical investigations and patches via methods outside GitHub and "public" internet-based channels *way* before GitHub interaction and similar was closed down. That was were almost *all* of the real contributions were coming from, anyway. Apparently, and not unsurprisingly, the people who has invested the time and effort to understand Reticulum also prefer to collaborate in this way. Since leaving the GitHub madhouse behind, the signal-to-noise ratio has **significantly** increased.
Managing a public "issue" tracker with global read/write access is a futile and useless endeavor. Consider this:
- User A reports a "bug" that is really just a failure of understanding.
- User B sees this and seconds is, proposing a "fix" that in continued failure of understanding would actually break functionality X.
- User C joins the bandwagon and asks why this hasn't already been implemented like that? It's obvious!
The sensible response here from the developer is closing the issue with "No. Go RTFM". Today, though, this usually results in hurt feelings, animosity towards the developer and in some cases (as experienced and documented in the case of RNS), months of perfidious personal vendetta against the developer for being so brazen as to suggest the user was wrong and wasting people's time.
When this pattern repeats, over and over, the only sensible, measured and constructive course of action is to shrug your shoulders and say:
*"This system is fundamentally broken. It ain't working. I can give up here, or I can go build something better that has a chance of working."*
So, now it's your turn. Go look at the diffs for the last six months. What does it look like I have been doing?
But I will be damned straight with you all, and say that part of that solution is **absolutely** to erect barriers to entry. You can fucking bet your arse on that. I don't want opinionated man-babies running around in my living-room at 3am. I don't want to clean up the floor after a wannabe "dev-ops star" with LLMs and a peripheral case of influencitis has puked all over the office.
- If you want to join the fun of changing core networking code that thousands of people rely on for communication
daily, you better know what the fuck you're doing.
- I'm not here to provide validation and hugs to random strangers. I'm here to make sure the reference implementation
of Reticulum works.
- If you cannot figure out how to submit a patch or valid bug investigation over RNS, you cannot expect I will take
you seriously. At all.
If someone can't handle that, they should find their entertainment elsewhere.
I've said it before: I've provided the information and code required to make Reticulum *work*, and build networked systems, protocols and applications on top of it. That information is deep, complex, and requires you to read hundreds of pages, and put in weeks of efforts to get the *full* picture. A lot less is required to get started, but it *will* still be a steep learning experience.
This is a full networking stack, based on some pretty complex principles, for crying out loud. It's **not** a `hello_world` designed to make you feel good about yourself. It turns almost everything you know about networked systems on its head. That's **challenging** for *anyone*. Climb the mountain, and it will be satisfying in the end. Refuse to climb... Well, what do you think will happen?
As for barriers to entry of *using* RNS and related programs, utilities and clients, it's not my task to teach every single user how to do X, Y and Z. The information *is* out there. If it wasn't organized optimally for your way of learning, you can choose to "raise your concerns" about it, discuss "the fact of it" on a forum or chatroom, or: *You can choose to remedy that, and help others along*.
I sure know what *I* would have done.

BIN
docs/Reticulum Manual.epub Normal file

Binary file not shown.

BIN
docs/Reticulum Manual.pdf Normal file

Binary file not shown.

View file

@ -11,7 +11,6 @@ LINE_START_PATTERNS = [
] ]
LINE_ANY_PATTERNS = [ LINE_ANY_PATTERNS = [
# r'<br/>',
# r'<div[^>]*>', # r'<div[^>]*>',
# r'</div>', # r'</div>',
] ]
@ -37,10 +36,6 @@ def should_remove_line(line, start_patterns, any_patterns):
def clean_markdown_content(content, start_patterns, any_patterns, api_ref=False): def clean_markdown_content(content, start_patterns, any_patterns, api_ref=False):
content = content.replace("**\n : ", "**\n ")
content = content.replace("\n* **", "\n\n* **")
content = content.replace("\n\n\n", "\n\n")
lines = content.split('\n') lines = content.split('\n')
result = [] result = []
skip_next_empty = False skip_next_empty = False
@ -62,8 +57,6 @@ def clean_markdown_content(content, start_patterns, any_patterns, api_ref=False)
if line.startswith("#### "): line = line.replace("#### ", "#### `") if line.startswith("#### "): line = line.replace("#### ", "#### `")
line = f"{line}`" line = f"{line}`"
line = line.replace("<br/>", "")
result.append(line) result.append(line)
# Remove trailing empty lines from end of file # Remove trailing empty lines from end of file

View file

@ -1,341 +0,0 @@
# Yes, I am Angry
Polite, but deeply malicious bullshit is a much worse evil than the coarse variant, so you will have to excuse my bluntness here.
Unfortunately, the only way to respond, as a normal, healthy human being, to something so astronomically tasteless and disgraceful as this, is not in kind, but in honest outrage, modulated to some degree with sarcasm and humor (the dark variant, yes).
Because *this* shit, this *is* outrageous.
The man that produced the "article" in question here is either so disposessed of intelligence that his head is teetering on the brink of collapsing into a singularity, or he is deeply malicious.
If any rational explanation can produce an alternative, I am all ears.
**To the footsoldiers of the "cause" the article represents**
Let's get this show on. Let's see the barrage of new smear pieces you can prompt forth from the fuel I provide you here. It'll fit straight into your theater production, so: Be my fucking guest, and dig your own hole of delusion, mistrust and ridicule even deeper than it already is. I can then hurl it right back at you, and we can all have a crap-fight till the end of days. *Or* we could take responsibility and ownership for our words and actions, and actually build something together. Ball's in your court.
**To anyone else**
Consider this a paranthetical, sarcastic (and for me, cathartic) aside in the whole of the corpus of writings I have published, and weigh it in the context of the rest of those 200,000 words or so, together with the frustration and exhaustion that inevitably entails from dealing with this same fabricated campaign and its regurgitated nonsense for *years* on end, while trying to do constructive and helpful work; and not expecting anything else than basic decency in return.
I am no angel, not perfect, flawed in many ways, but I certainly ain't afraid to show that I am a real human being, with the capacity for anger, and that in this case, the anger is fed by the most bitter form of disappointment and sadness in what people will stoop to for influence and attention.
[This is the article in question](https://gaggl.com/blogs/2026-09-05-enclosure-by-good-intentions/)
[This is the polite, measured analysis of the *generalized* phenomenon, all 18,000 words of it](https://reticulum.network/manual/brandolinis.html)
And here is a very short, but central point from that analysis:
> In whole, this is the description of three fronts of the same operation:
>
> * The license attack and other urgency-engineered narratives are the *cover*: They supply
> the ideological justification and battering ram for treating the reference as illegitimate,
> and needing replacement.
> * The ports are the *inventory*: they supply the “replacement” material that the narrative
> claims the ecosystem needs.
> * The takeover narrative is the *objective function*: the repeated, persistent, escalating claim that
> the reference should be replaced, and that the speakers themself are the natural heirs.
---
*This*, right here, is the real, human and honest response. And the only sane one there could ever be, lest madness be allowed to descend wholly.
While this is *written* as a response to you, Leo Gaggl, you absolute degenerate piece of fecal matter, it is really a response to the phenomenon *as a whole*.
I chose your article because it is such a complete and rich example of the madness. What I attack here is *conduct*.
Leo, according to his own evident preference, is the actor on the theatre stage, and I will treat him as such, in the play that follows.
---
> So when Mark Qvist, the person behind the [Reticulum](https://gaggl.com/blogs/2026-02-25-lpwan-meshes-reticulum-deep-dive/) mesh networking stack
Color me surprised, the link to "Reticulum" doesn't even point to the repo, but to *another* one of your sloporated "articles".
Don't wanna accidentially lead users to anywhere they could discover the truth about things now, would we? Like commit histories, the *actual* Reticulum community, that kind of stuff. Nah, nah...
> ... replaced its MIT licence with a custom one built to keep the project out of corporate AI training sets and out of weapons systems
Which happened 18 months ago. *Fuuuunny* timing bringing it up again now, right where it's [mighty fucking convenient](https://reticulum.network/manual/brandolinis.html#a-persistent-but-fragile-triad) for your slop-coding license-violating friends, don't you think?
Interesting you don't bring up *any* of the extensive argumentation made against the same old points you recycle here, but act as they simply never existed. Or maybe you just chose to completely ignore them in the service of your God-given mission?
> Reticulum is now marked non-free by Debian
Which is [excellent](https://reticulum.network/manual/brandolinis.html#the-distribution-argument), since debian users will not have to use old, and potentially outdated versions of a package that updates *fast* everytime it is required.
And "now"? Weasel wording.
> kept out of the main F-Droid repository
RNS? In the F-Droid repository? How would that even work? People install the base stack as an *app* and run it... how? Are we going to bundle a command line or `systemd` with that? Do you have any idea about how software actually works?
If people want to do that, open Termux and type `pip install rns`. Done. Has been possible and documented since the dawn of the project.
All actual *apps* that use Reticulum are distributed as APKs, ready to install and download, distribute offline, fetch from their websites, over Reticulum itself, or whatever platform they prefer. Commercially or for free. That's the point.
> It cannot be combined with GPL-licensed code
[Cry me a river](https://reticulum.network/manual/brandolinis.html#on-freedom). Not my fault that people choose to use viral licenses that absolutely must subsume everything and their aunt under their own restrictive terms.
> It is not eligible for most open-source grant funding.
Exactly, and I specifically rejected that, several times, to keep the license right and avoid political takeover. That's 50.000€ I said no to, right there, while I barely had enough to make it three months into the future. Because I value integrity and *real* sustainability a lot higher.
Do *you* have anything to show for it, in that regard? Who's footing your pay-check? A varied community of people from all walks of life who use the things you created? No? Didn't think so. I wonder who, then.
> And the clause that triggered all of this is very likely unenforceable.
"Very likely" is very likely a lot of handwaving, Leo. It's certainly not an argument. [This is](https://reticulum.network/manual/brandolinis.html#what-the-reticulum-license-is).
> As one Debian developer put it in the project’s own discussion thread, people who build harmful systems are not going to be stopped by a term they were always going to ignore.
It turns out [that most do](https://reticulum.network/manual/brandolinis.html#the-prediction-and-what-happened), because of a thing called "the law".
Or are you proposing we live in a state of complete and utter anarchy already? Or maybe that your cult is simply above the law?
> The moment you add a “not for harm” or “not for AI” condition, you have left open source behind
Strike up the organ, start the hymns.
> The Open Source Definition rules out licences that discriminate against people or groups, or against fields of endeavour
[Erratum](https://reticulum.network/manual/brandolinis.html#open-source-means-open-source): *Your* "open source" religion, not "The Open Source Definition" (Capitals Of Authority, bravo).
Yes, let's not discriminate against people, **BY FUCKING MURDERING THEM WITH DRONES**. Time for a party Leo!
Beauuuutiful way to hide [the true nonsense](https://reticulum.network/manual/brandolinis.html#what-the-argument-is-actually-doing) of the "fields of endeavour" by slyly insinuating *I* am being discriminatory.
Side-note: The overall patronizing, better-than-thy and fakely knowledgable tone is almost hilarious, if it wasn't so malicious.
> The tool was already on the shelf
Oh for Christ's sake, this one [again](https://reticulum.network/manual/brandolinis.html#recycled-wine-fermenting)...
> Kaonic tactical mesh radios are sold for military command and control, drone and autonomous-systems networking, and operation in “austere, contested environments”, and which has since been selected for NATO’s DIANA defence-innovation accelerator. It is MIT-licensed, it is an independent implementation of a protocol that is itself public domain, and its repository predates Qvist’s harm clause by nine months.
Those fuckers came to me and presented themselves as wanting to create a rust port for the good of the ecosystem, the world and the community. That was a blatant lie and front for their actual goal of acquiring my goodwill and expertise so they could quickly create a shallow clone, hardfork it with their own sloppy "Post-Quantum" crypto, and get big money from the military. They deliberately lied and manipulated to extract from a whole community and someone they believed to be a naive, vulnerable developer.
Why do you think a non-harm clause is needed? Their implementation is still not *working*, by the way, and has provided **nothing** to the community or world at large.
I cut ties with those bastards when I found out what their end-game was, but not until I had spent every ounce of energy I had slowing them down as much as humanly possible, while *I was still working on the actual Reticulum implementation and providing updates to users, managing a community, and living on next to nothing in a camper*.
Holy fuck I hate people like you Leo, you willfully ignorant, destructive, spoiled piece of shit.
> His fix trades a door nobody has worked out how to close for the loss of every distribution channel
Every distribution channel that [we never used, and never needed](https://reticulum.network/manual/brandolinis.html#the-distribution-argument)? I haven't seen PyPi acting up about our apparently infuriating conduct yet.
Moot point, even if they do, since we can distribute over Reticulum just fine. With full cryptographic authentication, and no centralized infrastructure or PKI required. Also, no governance required. *That's* resilience.
> and every downstream collaborator who needs a known-good licence.
We are definitely not short of downstream collaborators currently, so no thanks, you don't have much to sell me there.
> You pay the full price in distribution and collaboration
The quality of collaboration has gone up *significantly* since that decision. Most of the wannabes stay away and sob in their Stallman handkerchiefs, while we keep working, dilligently as always. Great success.
(Sorry Stallman, I love you)
Reticulum is available everywhere. It is one of the leanest, most dependency free software packages there is, and it ships an entire networking stack, a plethora of management utilities, a device flashing utility, a full GitHub replacement running over Reticulum, remote shell utilities, and supports a whole ecosystem of application-layer protocols and user-facing applications, including messaging (LXMF), voice (LXST), chat (RRC), content hosting and browsing (nomadnet), and many others.
*In a cross-platform, portable wheel of 537 kilobytes* that installs or embeds on every platform known to man. Drop it on a floppy, baby.
The delusions run so deep here I cannot even begin to fathom it.
> and you get an AI-training restriction a determined actor routes around in an afternoon.
In reality, of the 10 sampled language models (both proprietary and open-weights) that were tested for traces of Reticulum source material, *all* stopped training on the Reticulum source after the license change.
Turns out, *it worked*, Leo.
Did *you* check? No you didn't, handwaving and inventions again.
> A written answer to “what happens when the maintainer walks away”, because for Reticulum that stopped being hypothetical in December 2025.
Did you check the [commit logs](https://github.com/markqvist/Reticulum/commits/) you dimwit? No, you just randomly rolled around in the personal puddle of phantasmogoria you constructed from wishful thinking.
Have you seen what I have produced in those ten months? I'd wager it's a lot more than you, you lazy fuck, not even doing the most basic fact-checking of your smear-job propaganda pieces.
Try that on for a "written answer".
> There is now a Reticulum_CE, a “community edition” fork that exists mainly to carry a redistributable licence.
What psychosis are you getting these from? [Reticulum_CE](https://github.com/Reticulum-Community/Reticulum_CE) is published under the Reticulum License. Did you read their repo? No, you did not:
*"This is a community maintained fork of the Reticulum Network Stack and is intended as a public forum for discussion and issue reporting, as well as a repository to collect community enhancements to RNS."*
> There is RetiNet, an AGPL fork of the Python stack.
Which has been [defunct and abandoned for 8 months](https://codeberg.org/SkyGuy/RetiNet). Are you not equipped with visual perception or basic literacy?
> There are side-loaded F-Droid repositories so people can install the apps at all.
What nonsense is this? You are literally so horny to paint a picture of a dead system that you have to assume people cannot think for two seconds straight?
There has never been a point in time where people could not install Reticulum-based apps on Android. These have always, intentionally, been distributed as *re-distributable* APK files so people could share them offline. Sideband can host and share it's own APK. This is the point, Archimedes.
The F-droid repository was made as service to users by a volunteer in the *real* Reticulum community (not your abhorrent confabulation of what it is) who *has never once objected to the license and project directions*, and who just wanted to help make things easier!
Don't use them as ammunition in your perverse fantasies!
> Every one of those is real maintenance work, done by volunteers, spent entirely on the problem of being allowed to share the software.
A copy of my repo, then pasting my (Reticulum Licensed) work into four files, then abandoning the project is not "real maintenance work".
What I have been doing, for over a decade, is real maintenance work.
> four independent teams have rebuilt Reticulum from scratch in Rust.
Those [have not "rebuild"](https://reticulum.network/manual/brandolinis.html#copyright-the-two-legal-layers) anything ["from scratch"](https://reticulum.network/manual/brandolinis.html#the-clean-room-delusion).
They are feeding the source code of RNS directly into an LLM, running automated software loops, and copying it, blatantly, bit for bit, without any modicum of creative input, work or understanding.
Such a connoisseur of the commons you are.
> The people rebuilding the project to keep it alive
The project is already alive, and has been for a decade. You are lying so hard that it's fucking raining.
Those people, are in their own words, "Killing Reticulum" (quote, Lew Palm, "author" of "Leviculum"), so they can appropriate it for their own profit and control.
> The two that put enclosure resistance first, Leviculum and the rsReticulum stack behind Ratspeak, both chose AGPLv3.
And all of those are [void and null](https://reticulum.network/manual/brandolinis.html#void-grants-legal-foundations-of-machine-generated-code), because what they *really* chose is **copyright violation**, and complete disrespect ([as is now becoming tradition](https://reticulum.network/manual/brandolinis.html#licensing-analysis)) for open source development practices, attribution and basic decency.
*(**Important Note From The Future, added November 2026**: When the maintainer of Ratspeak discovered he had made made a mistake in the licensing, he promptly contacted me, and corrected everything within a few hours. As of this update, all the Ratspeak implementations and derivatives are correctly distributed under the Reticulum License.)*
> Underneath the licence choice sits a harder problem, and it is the one this site keeps coming back to: a system can be decentralised in its architecture and completely centralised in who decides things.
Oooooh... Tension. Drama. Tell me more, let's have the juicy details, huh? Outrage incoming, I can feel it! Ramp up your audience like a good boy, Leo!
The only thing that keeps coming back here is nonsense.
The decisions follow those responsible, over years, and the actual work. Not cry-babies like you. Get used to it.
> Relicensing a codebase other people contributed to, under MIT terms, with a single unilateral commit, is itself a small act of enclosure.
I am responsible for over 98% of commits in that repository. Not a single one of the other contributors disagreed to the license decision.
But your ideology runs so deep now, *that we cannot ourselves decide how we license our own software*? Yes, let's consider for a moment **who the fucking dictator is here**...
It's *almost* like you've decided you already personally own Reticulum.
> The pattern shows up off the codebase too. In late August 2026, after a badly-behaved crawler hammered his NomadNet git node with tens of thousands of requests
Ah yes, so I got personally mad because it hit *my* node, and had a tantrum?
The scraper was hitting *every node across the entire fucking intermesh*, from a high-bandwidth VPS, with no throttling, no regards for people's radio links, airtime concerns, metered connections or limited compute resources.
For what reason? To gouge tens of **millions** of requests to a broken piece of shit software that an idiot deployed and left running without oversight, trying to infintely download **everything that everyone had ever put on their nodes several times over** on a recursive loop.
Who made that stop? I did, because I [contacted](https://github.com/roogle-dev/reticulum-phantom/issues/4) the idiot and talked him out of it.
Let's hear more of your brilliant ideas about how to "protect the commons" you fucking ape.
> Qvist removed the volunteer moderator of the Reticulum subreddit, deleted roughly three years of posts, and closed it with a notice that ended “Well done, vibecoders. You ruined it for everyone.”
The moderator was not moderating anything, and hadn't been for months. The precise original agreement with him for allowing him moderator rights was exactly the arrangement, that if the space was not moderated, it would be closed, and that I would maintain the final decision on that.
This is now being spun into yet another [bottle of piss](https://reticulum.network/manual/brandolinis.html#recycled-wine-fermenting) by the dipshits like you. Well done, moron.
> Around the same window the project’s Matrix room shut down and the GitHub issue tracker was hidden.
Does time not exist in your alternate universe? The events you allude to are separated by month or years, and the rationales were clearly stated. You deliberately ignored all of that.
Further, I did not manage or administrate the Matrix room. It was a community room, run by volunteers, flooded with spam and bad-faith actors. I had *zero* admin rights on it. The moderators of the room made that decision themselves, long after I had left - *to focus on actual work*.
The development was *never* centered on the GitHub issue tracker, it ran over Reticulum long before the preposterous madhouse of GitHub was left behind for good, as it continues to do today, as is documented and stated several places (including the front-page of the repository you refuse to link to).
Signal to noise ratio has gone up *significantly* because of that. The community is *more* alive, and more productive.
You are literally making things up as you go.
> One community member’s summary: for a supposedly decentralised network, the founder holds and uses a lot of centralised levers.
Yes, let's just [invent](https://reticulum.network/manual/brandolinis.html#the-movement-of-a-dozen) another concerned "community member". Who? Some random nutjob on reddit?
A random stranger of your choosing is not a "community member" that gets to decide how a highly complex engineering project is run. But damn man, you wish it worked that way. You wish it badly.
> None of this is meant as a pile-on.
Oh, *thanks*, you're SO *considerate* and full of empathy Leo!
> The Reticulum Licence is presented as a shield for the commons
**Because that is what it actually is.**
How about trying to actually *prove* otherwise, with *a coherent argument*, instead of getting an LLM to spin a load of high-strung nonsense to further your deranged religion?
> If your users are European public bodies
Oh, those poor administrative organizations! Do you think I dedicated my life to empowering European Public Bodies, or actual people?
> Then spend the effort you saved on the part that is actually hard: governance that survives the founder losing interest.
[Ze Governance](https://reticulum.network/manual/brandolinis.html#a-history-of-what-works), again, I see.
Maybe when you've worked 12 hours a day on a project for ten years, which is persistently flooded with this kind of profound idiocy, and kept going, can you begin to consider expounding your finer points on "losing interest".
> The commons is not protected by a clause forbidding bad uses.
It sure as hell ain't protected by pretending the world is made of candy-floss either, like you seem to prefer.
> It is protected by how many people can fork it, ship it, and keep it running without asking anyone’s permission.
Oh isn't that convenient now? How many can do that today? 99.999999% of the worlds population. What's the equation if we change to your beloved AGPL again, you moron?
> Reticulum’s licence made that number smaller.
[Mathematical failure](https://reticulum.network/manual/brandolinis.html#on-freedom).
> The intention was to guard the commons. The effect was to fence off a piece of it.
On the intent, [correct](https://reticulum.network/manual/brandolinis.html#what-the-reticulum-license-is).
On the effect, no: The effect apparently was to induce the uncomfortable feeling of grinding against your own inconsistent dogmas in dishonest, propagandistic nutjobs like you, because *you* now feel *fenced out*.
There's an important difference there.
The self-referential circle-jerk you've got going on here is *exactly* the definition of a fence; one made of hysteria and propaganda with the ulterior motive of usurping power away from a community of thousands of people, and putting it, squarely, into your own, trembling, wanting and lazy little hands.
> Reticulum is also not a toy. People run it over LoRa and packet radio for emergency and off-grid communication, so “someone degrades the network” is a safety question, not an inconvenience.
**Bingo**, Leo.
Which is why you need a *real* [responsible, accountable and competent person](https://reticulum.network/manual/brandolinis.html#a-history-of-what-works), with a decade-long track-record to run the project, not a bunch of kids with a fixation on Claude, and their sugardaddies like you.
> Here is the part that frustrates me.
Yes, you must *surely* be frustrated Leo, how very frustrating this must be for you and your ChatGPT subscription to have to go through all of this.
Tell me about frustration once you have actually done something that required more than pillow-fucking your own sense of grandiosity.
All your github repos are minor personal projects with up to... count them... coooount them... five stars. No users. You must surely know a lot about software development, distribution, and these commons you are protecting so heroically.
I can see you have a lot of skin in the game here, and a lot of work and experience on your back to bolster your authority.
Fuck off, you fraud.
> Every few years a piece of infrastructure I care about gets a new licence
You don't care one fucking bit about infrastructure Leo. *You don't even know what it is*, least of all what it takes to create it.
I do. Because I did it.
Go home, you're drunk.
Thanks for the material, though. You've provided a textbook example of the incessant bullshit, appropriation, deliberate enclosure and intentionally weaponized manipulation that [this project is up against](https://reticulum.network/manual/brandolinis.html#intent-consequence).
*In ideology and war, self-justification wears the same coat, and feeds the same destruction, perpetually.*
Yours truly,
Mark Qvist
September 18th, 2026
---
That it is, I am no longer angry. Back to work.

View file

@ -1,253 +0,0 @@
## Void Grants: Legal Foundations of Machine-Generated Code
*From the [Brandolini's Reference](https://reticulum.network/manual/brandolinis.html) chapter of the Reticulum Manual.*
A grant of rights is only as real as the rights behind it. A license conveys what its issuer holds, and nothing more. A derivative may be distributed only under whatever license covers what it took. And over a work that a machine originated, no human holds anything at all.
Each of these statements is a matter of settled statute and case law, and this document sets out the authorities. The conclusions they support are:
1. A software implementation generated from a licensed work by a machine, published under its operator's claimed copyright and license, **conveys rights the operator never possessed**.
2. **Its grant is void**. The projects, products, and promises built upon such grants stand on nothing. A field of void grants is a minefield for everyone who crosses it.
This document is a general examination of copyright law as it applies to machine-generated code. It is not legal advice.
### I. The Foundational Statute: Translation Is Copying
The starting point is the statutory definition of the class of works at issue. Title 17, United States Code, section 101 provides the definition of a derivative work:
> A "derivative work" is a work based upon one or more preexisting works, such as a *translation*, musical arrangement, dramatization, fictionalization, motion picture version, sound recording, art reproduction, abridgment, condensation, or *any other form in which a work may be recast, transformed, or adapted*.
Section 101 further provides that *"the terms 'including' and 'such as' are illustrative and not limitative"*. As such, the list is a list of *examples*, and the very first example is translation. The definition also reaches, in its own words, *"any other form in which a work may be recast, transformed, or adapted"*.
The exclusive right at issue is defined in section 106:
> Subject to sections 107 through 122, the owner of copyright under this title has the exclusive rights to do and to authorize any of the following:
> (1) to reproduce the copyrighted work in copies or phonorecords;
> (2) to prepare derivative works based upon the copyrighted work;
> (3) to distribute copies or phonorecords of the copyrighted work to the public by sale or other transfer of ownership, or by rental, lease, or lending;
Violation of any of these rights is infringement. 17 U.S.C. § 501(a).
Taken together, the provisions yield a construction that is difficult to improve upon and impossible to escape:
1. **A port is a translation.** A work that takes a program written in one language and from it produces a program written in another language, implementing the same behavior, is a translation of the original - whatever mechanism performed it, and however the surface presentation differs.
2. **A translation is a derivative work.** Section 101 says so, explicitly.
3. **Preparing a derivative work is the copyright owner's exclusive right.** Section 106(2) says so explicitly. A derivative work prepared without authorization, or outside the scope of any authorization, infringes that right.
The common objection - "the code is in a different language, so it is completely different" - is not a legal argument; it is the definition of what the statute already classifies.
Translation *is the paradigm* of derivative authorship, not an exception to it. If translating a work without permission was a defense, the derivative-work right would be meaningless; every infringing adaptation would simply need to be "different enough" to count as a translation of its source.
Nor does the identity of the translator change the analysis. The statute asks whether the work is "based upon" a preexisting work. It does not ask who or what performed the translation. A machine that ingests a licensed work and reproduces a translated transformation of it has performed mechanical translation; the output is a derivative of the ingested work, and the law governing derivative works applies to it exactly as it applies to a translation made by hand.
### II. Ideas Are Free, Expressions Protected
The second foundation is the distinction between what a work *is* - its ideas, its processes, its systems - and the *expression* in which those ideas are embodied. The distinction is codified in section 102(b):
> In no case does copyright protection for an original work of authorship extend to any idea, procedure, process, system, method of operation, concept, principle, or discovery, regardless of the form in which it is described, explained, illustrated, or embodied in such work.
The principle is older than the statute, and its application to software is settled. An author may own the particular code in which a method is expressed; the method itself, the process, the algorithm, belongs to no one - because *"the sine qua non of copyright is originality"* (*Feist Publications, Inc. v. Rural Telephone Service Co.*, 499 U.S. 340, 345. 1991). And ideas are not original to anyone in particular, *Baker v. Selden*, 101 U.S. 99 (1879), established the line: Copyright in a book describing a system does not prevent others from practicing the system.
Applied to a communications protocol, the structure is clean:
---
**An implementation of a public-domain protocol, honestly reasoned and written by humans, is welcome and unencumbered, and owes nothing to any license. An implementation generated from a licensed implementation is a derivative of licensed work, subject to its conditions and to the ordinary copyright of its source.**
---
The public-domain status of the protocol's ideas changes nothing about the copyright in an implementation's expression. A person who copies an implementation - by machine or by hand - cannot defend the copying by pointing to the protocol, because the protocol covers the ideas, not the symbols in which someone expressed them.
A further misconception can be dismissed here; the recently popularized "clean-room" claim about the absence of copying. A [clean-room](https://reticulum.network/manual/brandolinis.html#the-clean-room-delusion) is a not a magic label achievable by LLM, but a specific, institutionalized procedure for proving to a court that copying did not occur: Physically separated teams, written functional specifications, attestable non-exposure, and no access to the original by the implementers (*Altai*, 982 F.2d at 709–11).
*"I created a clean-room implementation by feeding the original source code into a machine"* attempts to *claim* the certified, legally established procedure for proving the absence of copying by describing, in the same breath, a process **composed entirely of copying**. The word is deployed because the procedure it names is what the claimant would *need to have happened*.
### III. Machine Output Has No Author
The third foundation is the human authorship requirement. It is settled at both levels of the federal courts and in the Copyright Office's administering practice.
In *Thaler v. Perlmutter*, the district court held that *"copyright law protects only works of human creation,"* and that *"human authorship is a bedrock requirement of copyright"* (687 F. Supp. 3d 140, 146, D.D.C. 2023). On appeal, the United States Court of Appeals for the District of Columbia Circuit affirmed, holding that the Copyright Act *"requires all eligible works to be authored in the first instance by a human being"* (No. 23-5233, D.C. Cir. Mar. 18, 2025).
The requirement traces to the ordinary meaning of the statutory term. In *Burrow-Giles Lithographic Co. v. Sarony*, the Supreme Court defined an "author" as *"he to whom anything owes its origin; originator; maker; one who completes a work of science or literature"* (111 U.S. 53, 57–58, 1884), and more recently described the author as *"the person who translates an idea into a fixed, tangible expression entitled to copyright protection"* (*Community for Creative Non-Violence v. Reid*, 490 U.S. 730, 737, 1989).
---
**A machine does not owe expression its origin; it reproduces it. No court has recognized copyright in material created by non-humans, and the courts that have addressed the question have rejected it. If you create by machine generation, you do not own the output.**
---
The Copyright Office's registration guidance applies the requirement to generative systems in terms that are directly on point. The Office asks:
> whether the "work" is basically one of human authorship, with the computer [or other device] merely being an assisting instrument, or whether the traditional elements of authorship in the work (literary, artistic, or musical expression or elements of selection, arrangement, etc.) were actually conceived and executed not by man but by a machine.
- *Copyright Registration Guidance: Works Containing Material Generated by Artificial Intelligence*, 88 Fed. Reg. 16190, 16192 (Mar. 16, 2023)
And it states the consequence: *"If a work's traditional elements of authorship were produced by a machine, the work lacks human authorship and the Office will not register it"* (*Id.*).
On the role of prompts, the Office is equally explicit: when a system receives a prompt and produces a work in response, *"these prompts function more like instructions to a commissioned artist - they identify what the prompter wishes to have depicted, but the machine determines how those instructions are implemented in its output"* (*Id.*).
The agency applied this standard and excluded machine-originated material from registration in the leading administrative case. In *Zarya of the Dawn* (Cancellation Decision, Feb. 21, 2023, Registration No. VAu001480196), the Office cancelled registration of images generated by the Midjourney service, holding that *"it was Midjourney-not [the author]-that originated the 'traditional elements of authorship' in the images"*, while preserving protection for the human-authored text and the human selection and arrangement of the work.
The Copyright Office's 2025 report on AI copyrightability reaches the same conclusion:
> Based on the functioning of current generally available technology, prompts do not alone provide sufficient control.
- U.S. Copyright Office, *Copyright and Artificial Intelligence, Part 2: Copyrightability*, at iii (Jan. 29, 2025)
The report's analysis is explicit: *"The Office concludes that, given current generally available technology, prompts alone do not provide sufficient human control to make users of an AI system the authors of the output. Prompts essentially function as instructions that convey unprotectible ideas"* (*Id.* at 18).
Its guiding distinction is stated plainly: *"The use of AI tools to assist rather than stand in for human creativity does not affect the availability of copyright protection for the output"* while *"Copyright does not extend to purely AI-generated material, or material where there is insufficient human control over the expressive elements"* (*Id.* at iii).
The boundary is therefore not "AI or no AI". It is *assistance or substitution*: Whether a human exercised sufficient creative control over the expressive elements and can answer for them, or whether the machine originated them and the human merely supplied instructions and a signature. That is a question of fact in each case, determined from the record of how the work was made.
Two facts follow, and stated together, they form the base legal reality:
1. **A work whose expressive elements were originated by a machine contains no valid new copyright: It has no human author. To the extent that the same work reproduces a preexisting copyrighted work without explicit license, it is at the same time an unauthorized derivative. It is unowned for anything new it contains, and unlicensed for everything taken.**
2. **Its operator therefore holds no copyright in it and no grant over it - and a person who holds neither copyright nor grant can convey neither.**
The first fact makes the work **unownable** by its operator. The second makes it infringing against its source. Neither replaces the other:
- The absence of copyright in machine output does not make the output free for anyone to take; it makes it unowned.
- This is a different thing entirely, especially when what it contains is someone else's expression.
### IV. License Conditions Bound the Grant
The fourth foundation governs what a license *is*. A copyright owner may grant permission to use the work on conditions. The conditions define the scope of the permission. Use outside the scope is use that was never authorized.
In *Jacobsen v. Katzer*, 535 F.3d 1373 (Fed. Cir. 2008), the Federal Circuit held that the conditions of an open-source license are *conditions of the copyright license itself*, not mere contractual promises, and that a licensee operating outside them is an infringer:
> Copyright holders who engage in open source licensing have the right to control the modification and distribution of copyrighted material.
> The choice to exact consideration in the form of compliance with the open source requirements of disclosure and explanation of changes, rather than as a dollar-denominated fee, is entitled to no less legal recognition.
> It is outside the scope of the [license] to modify and distribute the copyrighted materials without copyright notices and a tracking of modifications from the original computer files.
- *Id.* at 1381–82
The doctrinal mechanism is settled: *"A copyright owner who grants a nonexclusive license to use his copyrighted material waives his right to sue the licensee for copyright infringement"* within the scope of the license. If, however *"the licensee acts outside the scope, the licensor can bring an action for copyright infringement"*: An act outside the granted scope **is infringement** (*Sun Microsystems, Inc. v. Microsoft Corp.*, 188 F.3d 1115, 1121, 9th Cir. 1999; *Graham v. James*, 144 F.3d 229, 236, 2d Cir. 1998; *S.O.S., Inc. v. Payday, Inc.*, 886 F.2d 1081, 1087, 9th Cir. 1989).
Whether a given license term is a condition of the grant or a mere covenant is determined from the license's own language; conditions that limit what may be copied, modified, and distributed - the attribution terms, the notice terms, the modification-tracking terms, the field-of-use terms - are the boundaries of the copyright grant (*MDY Industries, LLC v. Blizzard Entertainment, Inc.*, 629 F.3d 928, 940, 9th Cir. 2010).
Three consequences follow for the analysis at hand. They are not hypothetical.
**First, the conditions are measured at the moment of copying**
A work is licensed under whatever conditions applied when the reproduction, adaptation, and distribution occurred. There is **no** period in which copying a licensed work is "free" because the copy was made by a machine; the machine is the *licensee's tool*, and the licensee is bound by the conditions regardless. A derivative generated without satisfying the conditions - without the attribution, without authorization for the manner of use - was never within the scope of the license, from the first instant of its existence.
**Second, breach does not restore rights that never attached**
The district court in *Artifex Software, Inc. v. Hancom, Inc.*, No. 16-cv-06982-JSC (N.D. Cal. Sept. 12, 2017), analyzing the GPL's termination provisions, observed that *"the language of the GPL suggests that [the licensee's] obligations persisted beyond termination of its rights to propagate software using Ghostscript,"* because the source-code obligation recurs each time a covered work is conveyed. The same structure operates a fortiori where a derivative was produced *outside* the license entirely: No grant ever attached, and no subsequent event creates one.
**Third, the settled conditions are the enforceable core**
The attribution and notice conditions rest on the settled doctrine above. Novel conditions (restrictions on field of use, restrictions on how the work may be consumed) are enforceable as conditions of the grant when clearly drafted, but their enforcement history is thinner. The argument that a derivative is unlicensed does *not* depend on the novel conditions being vindicated; it needs **only** the settled ones, and the ordinary law of derivative works, to stand.
### V. The Void Grant
The preceding sections assemble into a single structure. Let us apply it to an example situation:
- A project presented as an independent implementation of a protocol; "ground-up", "from scratch" or "clean-room" work.
- The actual history shows continuous, machine-mediated ingestion of a licensed reference implementation.
- The output is generated at a scale no human could produce or review, with the provenance of the process obscured.
**Step One: The derivative**
The project's output is a translation of the licensed implementation: A derivative work, produced by a copy process. 17 U.S.C. §§ 101, 106(2). Preparing it without authorization infringed from the first moment of its existence. *Id.* § 501(a).
**Step Two: The license**
The derivative was produced outside the conditions of the license; without the notices, without the attribution. The license never attached (*Jacobsen*, 535 F.3d at 1381–82).
**Step Three: Authorship**
The expression was originated by a machine. No human can account for its creative decisions, and no human authored it. It contains no valid new copyright (*Thaler*, 687 F. Supp. 3d at 146; No. 23-5233, D.C. Cir. 2025; 88 Fed. Reg. at 16192).
**Consequence: Void Grant**
The statute supplies the consequence. Section 103(a) provides:
> The subject matter of copyright as specified by section 102 includes compilations and derivative works, but protection for a work employing preexisting material in which copyright subsists does not extend to any part of the work in which such material has been used unlawfully.
And section 103(b) provides that copyright in a derivative work *"extends only to the material contributed by the author of such work."* The legislative history states the point without ambiguity: *"An unauthorized translation of a novel could not be copyrighted at all"* (H.R. Rep. No. 94-1476, at 57, 1976).
In our example, the operator of the derivative's copy process decides to assert copyright over the output, and licenses it to the world under terms of the operator's choosing, for example the Apache License.
But the operator holds no copyright in machine-originated content (section III) and no license over the source from which it was derived (sections I and IV).
For the derived material, the operator's copyright claim **is an assertion of a right that never existed**. The material was used unlawfully, and section 103(a) withholds protection from *every* part of the work in which it appears. For the machine-originated remainder, the operator holds no copyright either, for lack of any human author.
The operator's license to the world is a promise the operator had no power to make. Every person who receives the work believing they received a valid grant has received **a void grant**, and stands, as a matter of settled law, in the position of a downstream user of an unauthorized derivative, whatever they were told at the time.
This is a cascading consequence, and it is why the pattern is a hazard. No combination of self-granted licenses - "MIT," "Apache," "public domain," anything at all - can transfer rights the operator never held. Every downstream project, product, or deployment built on the derivative is built on unlicensed reproduction of a third party's work, whether or not it knew, whether or not it was told, and whether or not the marketing was plausible.
The longer the derivative circulates unexamined, the more it becomes embedded, forked, vendored, depended upon, and the more expensive its correction becomes for everyone who trusted it. A field of superficially real projects, repositories, and licenses that **in reality confer nothing on anyone who relies on them is a minefield**, in the most precise sense the law allows: The rights they purport to grant *were never possessed*.
The law defines what may be asserted, not what will be. The copyright owner may choose whether and how to assert these rights, and at what point in time. The exposure exists regardless, and for *every* downstream that trusted the void grant. This is the scope of the hazard that the conduct exemplified here produces.
### VI. Fair Use Does Not Rescue This Pattern
Every defendant raises fair use. It is an essential and important limitation in the service of free speech, accountability, criticism, education and other important domains. Therefore, it must be considered, and in the case exemplified here, it fails.
Fair use is a limitation on the exclusive rights, evaluated case by case under four factors: The purpose and character of the use; the nature of the copyrighted work; the amount and substantiality of the portion used; and the effect of the use on the potential market for or value of the work (17 U.S.C. § 107).
On the first factor, the governing standard is *Andy Warhol Foundation for the Visual Arts, Inc. v. Goldsmith*, 598 U.S. 508 (2023), in which the Supreme Court held that:
> In sum, the first fair use factor considers whether the use of a copyrighted work has a further purpose or different character, which is a matter of degree, and the degree of difference must be balanced against the commercial nature of the use
The question is whether the use serves a further purpose, not whether the surface changed. A derivative that exists to serve as the same thing as the original, marketed to the people who would otherwise use the original, has not acquired a further purpose by being translated; the surface changed, the function did not, and the use is commercial.
The remaining factors resolve the same way. Source code is expressive creative work, not bare fact. The pattern under examination copies the whole; a complete implementation, ingested and translated in its entirety. And the market effect is the decisive factor, because it is engineered: A derivative marketed as a drop-in replacement for the work it was copied from is not a complement to the original's market, it is a substitute for it. The Supreme Court has long held that use of a work to satisfy the market demand for the original is the paradigm of an *unfair* use (*Harper & Row, Publishers, Inc. v. Nation Enterprises*, 471 U.S. 539, 566–67, 1985).
The one case that is always raised in response, *Google LLC v. Oracle America, Inc.*, 593 U.S. 1 (2021), holds less than it is believed to. The Court there assumed *"for argument's sake, that the material was copyrightable"* and held *"that the copying here at issue nonetheless constituted a fair use"* (*Id.* at 11).
The material in question was a thin slice of *declaring code*, which the Court found sits, *"if copyrightable at all, further than are most computer programs (such as the implementing code) from the core of copyright"* (*id.*), in a use that built a new platform for a new environment, with the copier **having written the vast majority of its own code**. By comparison: A wholesale translation of an entire implementation, marketed as a substitute for it, is **not** that case; it is the other end of every factor.
The valuable zones of fair use remain, and they are important. **Commentary and criticism**, quoting from a work to review, discuss or criticize it. **Teaching and scholarship**, explaining concepts and quoting excerpts, is its traditional beneficiary.
### Conclusion
The law is neither obscure nor new. The conclusion it yields can now be stated:
- A machine translation of a protected implementation **is a derivative work**, and an unauthorized derivative **is an infringement**, whatever language it speaks and whatever machinery produced it.
- A work whose expression a machine originated has no human author, no valid new copyright, and without valid copyright **no license can be granted on it**.
- § 103 withholds protection from every part of a derivative in which protected material was used unlawfully. A license is a grant with conditions, and a derivative made outside the conditions **never received a grant**.
- A person who holds no copyright and no grant **can convey neither**, and *every* downstream consumer who received from them received a **void grant**.
This is a *catastrophic* failure state for all who want to create things good and useful: The projects that inherit only void grants and liability believing they have inherited foundations, the users who depend on code that no one on earth can answer for, the honest builders whose work is buried under a field of plausible-looking impostors.
### Sources and Authorities
Statutory references are to Title 17, United States Code. Case citations are to the official or generally cited reporters. Online links are for reference only. Section numbers refer to the sections of this document in which each authority is principally relied upon.
**Statutes**
- [17 U.S.C. § 101 (definitions: "derivative work," "including," "publication") - I, II](https://www.law.cornell.edu/uscode/text/17/101)
- [17 U.S.C. § 102(b) (idea/expression) - II](https://www.law.cornell.edu/uscode/text/17/102)
- [17 U.S.C. § 103(a)–(b) (derivative works employing unlawfully used preexisting material) - V](https://www.law.cornell.edu/uscode/text/17/103)
- [17 U.S.C. § 106(1)–(3) (exclusive rights: reproduction, derivative works, distribution) - I](https://www.law.cornell.edu/uscode/text/17/106)
- [17 U.S.C. § 107 (fair use) - VI](https://www.law.cornell.edu/uscode/text/17/107)
- [17 U.S.C. § 501(a) (infringement) - I, V](https://www.law.cornell.edu/uscode/text/17/501)
**Judicial decisions**
- [*Jacobsen v. Katzer*, 535 F.3d 1373 (Fed. Cir. 2008) - IV, V](https://www.casemine.com/judgement/us/5914b290add7b04934760a54)
- [*Sun Microsystems, Inc. v. Microsoft Corp.*, 188 F.3d 1115 (9th Cir. 1999) - IV](https://law.justia.com/cases/federal/appellate-courts/F3/188/1115/629501/)
- [*Graham v. James*, 144 F.3d 229 (2d Cir. 1998) - IV](https://caselaw.findlaw.com/court/us-2nd-circuit/1011522.html)
- [*S.O.S., Inc. v. Payday, Inc.*, 886 F.2d 1081 (9th Cir. 1989) - IV](https://law.justia.com/cases/federal/appellate-courts/F2/886/1081/19091/)
- [*MDY Industries, LLC v. Blizzard Entertainment, Inc.*, 629 F.3d 928 (9th Cir. 2010) - IV](https://law.justia.com/cases/federal/appellate-courts/ca9/09-15932/09-15932-2011-02-25.html)
- [*Artifex Software, Inc. v. Hancom, Inc.*, No. 16-cv-06982-JSC (N.D. Cal. Sept. 12, 2017) (settled; dismissed with prejudice Jan. 2018) - IV](https://storage.courtlistener.com/recap/gov.uscourts.cand.305835/gov.uscourts.cand.305835.54.0_1.pdf)
- [*Thaler v. Perlmutter*, 687 F. Supp. 3d 140 (D.D.C. 2023), aff'd, No. 23-5233 (D.C. Cir. Mar. 18, 2025) - III, V](https://www.bitlaw.com/source/cases/copyright/Thaler-Perlmutter-Dct.html)
- [*Burrow-Giles Lithographic Co. v. Sarony*, 111 U.S. 53 (1884) - III](https://supreme.justia.com/cases/federal/us/111/53/)
- [*Community for Creative Non-Violence v. Reid*, 490 U.S. 730 (1989) - III](https://supreme.justia.com/cases/federal/us/490/730/)
- [*Feist Publications, Inc. v. Rural Telephone Service Co.*, 499 U.S. 340 (1991) - II](https://supreme.justia.com/cases/federal/us/499/340/)
- [*Baker v. Selden*, 101 U.S. 99 (1879) - II](https://supreme.justia.com/cases/federal/us/101/99/)
- [*Computer Associates International, Inc. v. Altai, Inc.*, 982 F.2d 693 (2d Cir. 1992) - II](https://law.justia.com/cases/federal/appellate-courts/F2/982/693/137252/)
- [*Andy Warhol Foundation for the Visual Arts, Inc. v. Goldsmith*, 598 U.S. 508 (2023) - VI](https://www.supremecourt.gov/opinions/22pdf/21-869_87ad.pdf)
- [*Google LLC v. Oracle America, Inc.*, 593 U.S. 1 (2021) - VI](https://supreme.justia.com/cases/federal/us/593/18-956/)
- [*Harper & Row, Publishers, Inc. v. Nation Enterprises*, 471 U.S. 539 (1985) - VI](https://supreme.justia.com/cases/federal/us/471/539/)
**Administrative and policy authorities**
- [*Copyright Registration Guidance: Works Containing Material Generated by Artificial Intelligence*, 88 Fed. Reg. 16190 (Mar. 16, 2023) - III](https://www.federalregister.gov/documents/2023/03/16/2023-05321/copyright-registration-guidance-works-containing-material-generated-by-artificial-intelligence)
- [*Zarya of the Dawn*, U.S. Copyright Office (Cancellation Decision, Feb. 21, 2023), Registration No. VAu001480196 - III](https://www.copyright.gov/docs/zarya-of-the-dawn.pdf)
- [U.S. Copyright Office, *Copyright and Artificial Intelligence, Part 2: Copyrightability* (Jan. 29, 2025) - III](https://www.copyright.gov/ai/Copyright-and-Artificial-Intelligence-Part-2-Copyrightability-Report.pdf)
- [H.R. Rep. No. 94-1476 (1976) - V](https://law.resource.org/pub/us/works/aba/ibr/H.Rep.94-1476.pdf)

View file

@ -1,4 +1,4 @@
# Sphinx build info version 1 # Sphinx build info version 1
# This file records the configuration used when building these files. When it is not found, a full rebuild will be done. # This file records the configuration used when building these files. When it is not found, a full rebuild will be done.
config: 6795d8fd07f6250f1b0bffe6d5069dbb config: d77ea3044971177c0a1e0c192353c10a
tags: 645f666f9bcd5a90fca523b33c5a78b7 tags: 645f666f9bcd5a90fca523b33c5a78b7

Binary file not shown.

Before

Width:  |  Height:  |  Size: 63 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 35 KiB

After

Width:  |  Height:  |  Size: 25 KiB

Before After
Before After

Some files were not shown because too many files have changed in this diff Show more