Updated logging

Fix msgpack import

.dockerignore

@@ -0,0 +1 @@
+.gitignore

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+  - name: ✨ Feature Request or Idea
+    url: https://github.com/markqvist/Reticulum/discussions/new?category=ideas
+    about: Propose and discuss features and ideas
+  - name: 💬 Questions, Help & Discussion
+    about: Ask anything, or get help
+    url: https://github.com/markqvist/Reticulum/discussions/new/choose
+  - name: 📖 Read the Reticulum Manual
+    url: https://markqvist.github.io/Reticulum/manual/
+    about: The complete documentation for Reticulum

.github/ISSUE_TEMPLATE/🐛-bug-report.md

@@ -0,0 +1,35 @@
+---
+name: "\U0001F41B Bug Report"
+about: Report a reproducible bug
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Read the Contribution Guidelines**
+Before creating a bug report on this issue tracker, you **must** read the [Contribution Guidelines](https://github.com/markqvist/Reticulum/blob/master/Contributing.md). Issues that do not follow the contribution guidelines **will be deleted without comment**.
+
+- The issue tracker is used by developers of this project. **Do not use it to ask general questions, or for support requests**.
+- Ideas and feature requests can be made on the [Discussions](https://github.com/markqvist/Reticulum/discussions). **Only** feature requests accepted by maintainers and developers are tracked and included on the issue tracker. **Do not post feature requests here**.
+- After reading the [Contribution Guidelines](https://github.com/markqvist/Reticulum/blob/master/Contributing.md), **delete this section only** (*"Read the Contribution Guidelines"*) from your bug report, **and fill in all the other sections**.
+
+**Describe the Bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Describe in detail how to reproduce the bug.
+
+**Expected Behavior**
+A clear and concise description of what you expected to happen.
+
+**Logs & Screenshots**
+Please include any relevant log output. If applicable, also add screenshots to help explain your problem.
+
+**System Information**
+- OS and version
+- Python version
+- Program version
+
+**Additional context**
+Add any other context about the problem here.

.github/workflows/publish-container.yml

@@ -0,0 +1,42 @@
+name: Create and publish a Docker image
+
+on:
+  push:
+    branches: ['master']
+    tags: ['*.*.*']
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.12-alpine as build
+
+RUN apk add --no-cache build-base linux-headers libffi-dev cargo
+
+# Create a virtualenv that we'll copy to the published image
+RUN python -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+RUN pip3 install setuptools-rust pyopenssl cryptography
+
+COPY . /app/
+RUN cd /app/ && pip3 install .
+
+# Use multi-stage build, as we don't need rust compilation on the final image
+FROM python:3.12-alpine
+
+LABEL org.opencontainers.image.documentation="https://github.com/markqvist/NomadNet#nomad-network-daemon-with-docker"
+
+ENV PATH="/opt/venv/bin:$PATH"
+ENV PYTHONUNBUFFERED="yes"
+COPY --from=build /opt/venv /opt/venv
+
+VOLUME /root/.reticulum
+VOLUME /root/.nomadnetwork
+
+ENTRYPOINT ["nomadnet"]
+CMD ["--daemon"]

Dockerfile.build

@@ -0,0 +1,28 @@
+FROM python:alpine
+LABEL authors="Petr Blaha petr.blaha@cleverdata.cz"
+USER root
+RUN apk update
+RUN apk add  build-base libffi-dev cargo pkgconfig   linux-headers py3-virtualenv
+
+RUN addgroup -S myuser && adduser -S -G myuser myuser
+USER myuser
+WORKDIR /home/myuser
+
+RUN pip install --upgrade pip
+RUN pip install setuptools-rust pyopenssl cryptography
+
+
+ENV PATH="/home/myuser/.local/bin:${PATH}"
+
+################### BEGIN NomadNet ###########################################
+
+
+COPY --chown=myuser:myuser . .
+
+#Python create virtual environment
+RUN virtualenv /home/myuser/NomadNet/venv
+RUN source /home/myuser/NomadNet/venv/bin/activate
+
+RUN make all
+
+################### END NomadNet ###########################################

Dockerfile.howto

@@ -0,0 +1,6 @@
+# Run docker command one by one(all four), it will build NomadNet artifact and copy to dist directory.
+# No need to build locally and install dependencies
+docker build -t nomadnetdockerimage -f Dockerfile.build .
+docker run -d -it --name nomadnetdockercontainer nomadnetdockerimage /bin/sh
+docker cp nomadnetdockercontainer:/home/myuser/dist .
+docker rm -f nomadnetdockercontainer

FUNDING.yml

@@ -0,0 +1,3 @@
+liberapay: Reticulum
+ko_fi: markqvist
+custom: "https://unsigned.io/donate"

LICENSE

@@ -0,0 +1,674 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<https://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<https://www.gnu.org/licenses/why-not-lgpl.html>.

MIRROR.md

@@ -0,0 +1,33 @@
+This repository is a public mirror. All potential future development is happening elsewhere.
+
+I am stepping back from all public-facing interaction with this project. Reticulum has always been primarily my work, and continuing in the current public, internet-facing model is no longer sustainable.
+
+The software remains available for use as-is. Occasional updates may appear at unpredictable intervals, but there will be no support, no responses to issues, no discussions, and no community management in this or any other public venue. If it doesn't work for you, it doesn't work. That is the entire extent of available troubleshooting assistance I can offer you.
+
+If you've followed this project for a while, you already know what this means. You know who designed, wrote and tested this, and you know how many years of my life it took. You'll also know about both my particular challenges and strengths, and how I believe anything worth building needs to be built and  maintained with our own hands.
+
+Seven months ago, I said I needed to step back, that I was exhausted, and that I needed to recover. I believed a public resolve would be enough to effectuate that, but while striving to get just a few more useful features and protocols out, the unproductive requests and demands also ramped up, and I got pulled back into the same patterns and draining interactions that I'd explicitly said I couldn't sustain anymore.
+
+So here's what you might have already guessed: I'm done playing the game by rules I can't win at.
+
+Everything you need is right here, and by any sensible measure, it's done. Anyone who wants to invest the time, skill and persistence can build on it, or completely re-imagine it with different priorities. That was always the point.
+
+The people who actually contributed - you know who you are, and you know I mean it when I say: Thank you. All of you who've used this to build something real - that was the goal, and you did it without needing me to hold your hand.
+
+The rest of you: You have what you need. Use it or don't. I am not going to be the person who explains it to you anymore.
+
+This is not a temporary break. It's not "see you after some rest", but a recognition that the current model is fundamentally incompatible with my life, my health, and my reality.
+
+If you want to support continued work, you can do so at the donation links listed in this repository. But please understand, that this is not purchasing support or guaranteeing updates. It is support for work that happens on my timeline, according to my capacity, which at the moment is not what it was.
+
+If you want Reticulum to continue evolving, you have the power to make that happen. The protocol is public domain. The code is open source. Everything you need is right here. I've provided the tools, but building what comes next is not my responsibility anymore. It's yours.
+
+To the small group of people who has actually been here, and understood what this work was and what it cost - you already know where to find me if it actually matters.
+
+To everyone else: This is where we part ways. No hard feelings. It's just time.
+
+---
+
+असतो मा सद्गमय
+तमसो मा ज्योतिर्गमय
+मृत्योर्मा अमृतं गमय

Makefile

@@ -0,0 +1,30 @@
+all: release
+
+clean:
+	@echo Cleaning...
+	-rm -r ./build
+	-rm -r ./dist
+
+remove_symlinks:
+	@echo Removing symlinks for build...
+	-rm ./LXMF
+	-rm ./RNS
+
+create_symlinks:
+	@echo Creating symlinks...
+	-ln -s ../Reticulum/RNS ./
+	-ln -s ../LXMF/LXMF ./
+
+build_wheel:
+	python3 setup.py sdist bdist_wheel
+
+release: remove_symlinks build_wheel create_symlinks
+
+upload:
+	@echo Ready to publish release over Reticulum
+	@read VOID
+	rngit release rns://7649a50d84610232d1416b41d2896aff/reticulum/nomadnet create $$(python setup.py --getversion):dist --name nomadnet
+
+upload-pip:
+	@echo Uploading to PyPi...
+	twine upload dist/*.whl dist/*.tar.gz

README.md

@@ -1,41 +1,163 @@
-Nomad Network
-==========
+# Nomad Network - Communicate Freely
 
-Communicate Freely.
+> [!WARNING]  
+> Several hastily launched, incorrect, and in most cases entirely LLM-generated fakes of Nomad Network and associated tools are currently being circulated and marketed. Most of these "projects" violate the open source GPLv3 license that Nomad Network was published under, but claim independent ownership and license grants. Such claims or grants are **not** legally valid, and **not** recognized by the Nomad Network authors and copyright holders.
+>
+> Any claimed assertion of copyright or grant of license that such projects are making are [void grants](https://reticulum.network/manual/brandolinis.html#void-grants-legal-foundations-of-machine-generated-code). Both the violators themselves, and **all** downstream projects using the infringing derivatives, are **directly** and **personally** liable for the legal consequences.
+>
+> For more detailed information, see the relevant sections of the [Brandolini's Reference](https://reticulum.network/manual/brandolinis.html) chapter of the Reticulum Manual. For software and projects recognized by the Reticulum community, see the [Programs Using Reticulum](https://reticulum.network/manual/software.html) chapter.
+>
+> Do **not** use these "projects", they are a hazard to the entire ecosystem we have been carefully building over the last ten years.
 
-![Screenshot](https://github.com/markqvist/NomadNet/raw/master/docs/screenshots/3.png)
+*This repository is [a public mirror](./MIRROR.md). All development is happening elsewhere.*
 
-Nomad Network is built using Reticulum, see [github.com/markqvist/Reticulum](https://github.com/markqvist/Reticulum).
+Off-grid, resilient mesh communication with strong encryption, forward secrecy and extreme privacy.
+
+![Screenshot](https://github.com/markqvist/NomadNet/raw/master/docs/screenshots/1a.png)
+
+Nomad Network allows you to build private and resilient communications platforms that are in complete control and ownership of the people that use them. No signups, no agreements, no handover of any data, no permissions and gatekeepers.
+
+Nomad Network is build on [LXMF](https://github.com/markqvist/LXMF) and [Reticulum](https://github.com/markqvist/Reticulum), which together provides the cryptographic mesh functionality and peer-to-peer message routing that Nomad Network relies on. This foundation also makes it possible to use the program over a very wide variety of communication mediums, from packet radio to fiber optics.
+
+Nomad Network does not need any connections to the public internet to work. In fact, it doesn't even need an IP or Ethernet network. You can use it entirely over packet radio, LoRa or even serial lines. But if you wish, you can bridge islanded networks over the Internet or private ethernet networks, or you can build networks running completely over the Internet. The choice is yours. Since Nomad Network uses Reticulum, it is efficient enough to run even over *extremely* low-bandwidth medium, and has been succesfully used over 300bps radio links.
+
+If you'd rather want to use an LXMF client with a graphical user interface, you may want to take a look at [Sideband](https://github.com/markqvist/sideband), which is available for Linux, Android, Windows and macOS.
+
+Other Nomad Network clients include:
+
+- [MeshChatX](https://meshchatx.com/)
+- [Columba (mobile)](https://github.com/torlando-tech/columba)
+- [Ren Browser](https://github.com/Quad4-Software/Ren-Browser)
 
 ## Notable Features
  - Encrypted messaging over packet-radio, LoRa, WiFi or anything else [Reticulum](https://github.com/markqvist/Reticulum) supports.
  - Zero-configuration, minimal-infrastructure mesh communication
-
-## Current Status
-
-Pre-alpha. At this point Nomad Network is usable as a basic messaging client over Reticulum networks, but only the very core features have been implemented. Development is ongoing and current features being implemented are:
-
- - Propagated messaging and discussion threads
- - Connectable nodes that can host pages, files and other resources
- - Collaborative information sharing and spatial map-style "wikis"
-
-## Dependencies:
- - Python 3
- - RNS
- - LXMF
+ - Distributed and encrypted message store holds messages for offline users
+ - Connectable nodes that can host pages and files
+ - Image rendering support in terminals that support it
+ - Node-side generated pages with PHP, Python, bash or others
+ - Built-in text-based browser for interacting with contents on nodes
+ - Built-in RRC client for live, many-to-many chat
+ - An easy to use and bandwidth efficient markup language for writing pages
+ - Page caching in browser
 
 ## How do I get started?
 The easiest way to install Nomad Network is via pip:
 
 ```bash
 # Install Nomad Network and dependencies
-pip3 install nomadnet
+pip install nomadnet
 
 # Run the client
 nomadnet
+
+# Or alternatively run as a daemon, with no user interface
+nomadnet --daemon
+
+# List options
+nomadnet --help
 ```
 
-To use Nomad Network on packet radio or LoRa, you will need to configure your Reticulum installation to use any relevant packet radio TNCs or LoRa devices on your system. See the Reticulum documentation for info.
+If you are using an operating system that blocks normal user package installation via `pip`, you can return `pip` to normal behaviour by editing the `~/.config/pip/pip.conf` file, and adding the following directive in the `[global]` section:
+
+```text
+[global]
+break-system-packages = true
+```
+
+Alternatively, you can use the `pipx` tool to install Nomad Network in an isolated environment:
+
+```bash
+# Install Nomad Network
+pipx install nomadnet
+
+# Optionally install Reticulum utilities
+pipx install rns
+
+# Optionally install standalone LXMF utilities
+pipx install lxmf
+
+# Run the client
+nomadnet
+
+# Or alternatively run as a daemon, with no user interface
+nomadnet --daemon
+
+# List options
+nomadnet --help
+```
+
+**Please Note**: If this is the very first time you use pip to install a program on your system, you might need to reboot your system for the program to become available. If you get a "command not found" error or similar when running the program, reboot your system and try again.
+
+The first time the program is running, you will be presented with the **Guide section**, which contains all the information you need to start using Nomad Network.
+
+To use Nomad Network on packet radio or LoRa, you will need to configure your Reticulum installation to use any relevant packet radio TNCs or LoRa devices on your system. See the [Reticulum documentation](https://markqvist.github.io/Reticulum/manual/interfaces.html) for info. For a general introduction on how to set up such a system, take a look at [this post](https://unsigned.io/private-messaging-over-lora/).
+
+If you want to try Nomad Network without building your own physical network, you can connect to the [distributed backbone](https://reticulum.network/manual/gettingstartedfast.html#bootstrapping-connectivity) over the Internet, where there is already quite a bit of Nomad Network and LXMF activity. If you connect to the testnet, you can leave nomadnet running for a while and wait for it to receive announces from other nodes on the network that host pages or services, or you can try connecting directly to some nodes listed here:
+
+ - `9ce92808be498e9e05590ff27cbfdfe4` The rns.recipes forum
+ - `a4a5e861626ce97c9aa544d9ecdf6d22` rmap.world
+
+To browse pages on a node that is not currently known, open the URL dialog in the `Network` section of the program by pressing `Ctrl+U`, paste or enter the address and select `Go` or press enter. Nomadnet will attempt to discover and connect to the requested node.
+
+### Install on Android
+You can install Nomad Network on Android using Termux, but there's a few more commands involved than the above one-liner. The process is documented in the [Android Installation](https://markqvist.github.io/Reticulum/manual/gettingstartedfast.html#reticulum-on-android) section of the Reticulum Manual. Once the Reticulum has been installed according to the linked documentation, Nomad Network can be installed as usual with pip.
+
+For a native Android application with a graphical user interface, have a look at [Sideband](https://github.com/markqvist/Sideband).
+
+
+## Tools & Extensions
+
+Nomad Network is a very flexible and extensible platform, and a variety of community-provided tools, utilities and node-side extensions exist:
+
+- [Retipedia](https://github.com/RFnexus/Retipedia)
+- [NomadForum](https://codeberg.org/AutumnSpark1226/nomadForum) ([GitHub mirror](https://github.com/AutumnSpark1226/nomadForum))
+- [NomadForecast](https://github.com/faragher/NomadForecast)
+- [micron-blog](https://github.com/randogoth/micron-blog)
+- [md2mu](https://github.com/randogoth/md2mu)
+- [Any2MicronConverter](https://github.com/SebastianObi/Any2MicronConverter)
+- [Some nomadnet page examples](https://github.com/SebastianObi/NomadNet-Pages)
+- [More nomadnet page examples](https://github.com/epenguins/NomadNet_pages)
+- [LXMF-Bot](https://github.com/randogoth/lxmf-bot)
+- [LXMF Messageboard](https://github.com/chengtripp/lxmf_messageboard)
+- [LXMEvent](https://github.com/faragher/LXMEvent)
+- [POPR](https://github.com/faragher/POPR)
+- [LXMF Tools](https://github.com/SebastianObi/LXMF-Tools)
+
+
+## Support Nomad Network
+You can help support the continued development of open, free and private communications systems by donating via one of the following channels:
+
+- Monero:
+  ```
+  84FpY1QbxHcgdseePYNmhTHcrgMX4nFfBYtz2GKYToqHVVhJp8Eaw1Z1EedRnKD19b3B8NiLCGVxzKV17UMmmeEsCrPyA5w
+  ```
+- Bitcoin
+  ```
+  bc1pgqgu8h8xvj4jtafslq396v7ju7hkgymyrzyqft4llfslz5vp99psqfk3a6
+  ```
+- Ethereum
+  ```
+  0x91C421DdfB8a30a49A71d63447ddb54cEBe3465E
+  ```
+- Liberapay: https://liberapay.com/Reticulum/
+
+- Ko-Fi: https://ko-fi.com/markqvist
+
 
 ## Caveat Emptor
-Nomad Network is experimental software, and should be considered as such. While it has been built with cryptography best-practices very foremost in mind, it _has not_ been externally security audited, and there could very well be privacy-breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch.
+Nomad Network is beta software, and should be considered as such. Use at your own risk and responsibility.
+
+## Screenshots
+
+![Screenshot 1](https://github.com/markqvist/NomadNet/raw/master/docs/screenshots/1a.png)
+
+![Screenshot 2](https://github.com/markqvist/NomadNet/raw/master/docs/screenshots/2.png)
+
+![Screenshot 3](https://github.com/markqvist/NomadNet/raw/master/docs/screenshots/3.png)
+
+![Screenshot 4](https://github.com/markqvist/NomadNet/raw/master/docs/screenshots/4.png)
+
+![Screenshot 5](https://github.com/markqvist/NomadNet/raw/master/docs/screenshots/5.png)
+
+![Screenshot 6](https://github.com/markqvist/NomadNet/raw/master/docs/screenshots/6.png)

README.mu

@@ -0,0 +1,123 @@
+>> Nomad Network - Communicate Freely
+
+Off-grid, resilient mesh communication with strong encryption, forward secrecy and extreme privacy.
+
+Nomad Network allows you to build private and resilient communications platforms that are in complete control and ownership of the people that use them. No signups, no agreements, no handover of any data, no permissions and gatekeepers.
+
+Nomad Network is build on `_`!`[LXMF`a8d24177d946de4f1f0a0fe1af9a1338:/page/repo.mu`g=reticulum|r=lxmf]`!`_ and `_`!`[Reticulum`a8d24177d946de4f1f0a0fe1af9a1338:/page/repo.mu`g=reticulum|r=reticulum]`!`_, which together provides the cryptographic mesh functionality and peer-to-peer message routing that Nomad Network relies on. This foundation also makes it possible to use the program over a very wide variety of communication mediums, from packet radio to fiber optics.
+
+Nomad Network does not need any connections to the public internet to work. In fact, it doesn't even need an IP or Ethernet network. You can use it entirely over packet radio, LoRa or even serial lines. But if you wish, you can bridge islanded networks over the Internet or private ethernet networks, or you can build networks running completely over the Internet. The choice is yours. Since Nomad Network uses Reticulum, it is efficient enough to run even over `*extremely`* low-bandwidth medium, and has been succesfully used over 300bps radio links.
+
+If you'd rather want to use an LXMF client with a graphical user interface, you may want to take a look at `_`!`[Sideband`a8d24177d946de4f1f0a0fe1af9a1338:/page/repo.mu`g=reticulum|r=sideband]`!`_, which is available for Linux, Android, Windows and macOS.
+
+>> Notable Features
+
+ • Encrypted messaging over packet-radio, LoRa, WiFi or anything else `_`!`[Reticulum`a8d24177d946de4f1f0a0fe1af9a1338:/page/repo.mu`g=reticulum|r=reticulum]`!`_ supports.
+ • Zero-configuration, minimal-infrastructure mesh communication
+ • Distributed and encrypted message store holds messages for offline users
+ • Connectable nodes that can host pages and files
+ • Image rendering support in terminals that support it
+ • Node-side generated pages with PHP, Python, bash or others
+ • Built-in text-based browser for interacting with contents on nodes
+ • Built-in RRC client for live, many-to-many chat
+ • An easy to use and bandwidth efficient markup language for writing pages
+ • Page caching in browser
+
+>> How do I get started?
+
+The easiest way to install Nomad Network is via `B333pip`b:
+
+`B333
+`=
+# Install Nomad Network and dependencies
+pip install nomadnet
+
+# Run the client
+nomadnet
+
+# Or alternatively run as a daemon, with no user interface
+nomadnet --daemon
+
+# List options
+nomadnet --help
+`=
+`b
+
+If you are using an operating system that blocks normal user package installation via `B333pip`b, you can return `B333pip`b to normal behaviour by editing the `B333~/.config/pip/pip.conf`b file, and adding the following directive in the `B333[global]`b section:
+
+`B333
+`=
+[global]
+break-system-packages = true
+`=
+`b
+
+Alternatively, you can use the `B333pipx`b tool to install Nomad Network in an isolated environment:
+
+`B333
+`=
+# Install Nomad Network
+pipx install nomadnet
+
+# Optionally install Reticulum utilities
+pipx install rns
+
+# Optionally install standalone LXMF utilities
+pipx install lxmf
+
+# Run the client
+nomadnet
+
+# Or alternatively run as a daemon, with no user interface
+nomadnet --daemon
+
+# List options
+nomadnet --help
+`=
+`b
+
+`!Please Note`!: If this is the very first time you use pip to install a program on your system, you might need to reboot your system for the program to become available. If you get a "command not found" error or similar when running the program, reboot your system and try again.
+
+The first time the program is running, you will be presented with the `!Guide section`!, which contains all the information you need to start using Nomad Network.
+
+To use Nomad Network on packet radio or LoRa, you will need to configure your Reticulum installation to use any relevant packet radio TNCs or LoRa devices on your system. See the `_`!`[Reticulum documentation`a8d24177d946de4f1f0a0fe1af9a1338:/page/blob.mu`g=reticulum|r=reticulum|ref=HEAD|path=docs/markdown/index.md]`!`_ for info.
+
+If you want to try Nomad Network without building your own physical network, you can connect to the `_`!`[distributed RNS backbone`a8d24177d946de4f1f0a0fe1af9a1338:/page/blob.mu`g=reticulum|r=reticulum|ref=HEAD|path=docs/markdown/gettingstartedfast.md|anchor=connect-to-the-distributed-backbone]`!`_ over the Internet, where there is already quite a bit of Nomad Network and LXMF activity. If you connect to the testnet, you can leave nomadnet running for a while and wait for it to receive announces from other nodes on the network that host pages or services, or you can try connecting directly to some nodes listed here:
+
+ • `!`_`[9ce92808be498e9e05590ff27cbfdfe4]`_`! The rns.recipes forum
+ • `!`_`[a4a5e861626ce97c9aa544d9ecdf6d22]`_`! rmap.world
+
+To browse pages on a node that is not currently known, open the URL dialog in the `!Network`! section of the program by pressing `!Ctrl+U`!, paste or enter the address and select `!Go`! or press enter. Nomadnet will attempt to discover and connect to the requested node.
+
+>>> Install on Android
+
+You can install Nomad Network on Android using Termux, but there's a few more commands involved than the above one-liner. The process is documented in the `_`!`[Android Installation`a8d24177d946de4f1f0a0fe1af9a1338:/page/blob.mu`g=reticulum|r=reticulum|ref=HEAD|path=docs/markdown/gettingstartedfast.md|anchor=reticulum-on-android]`!`_ section of the Reticulum Manual. Once the Reticulum has been installed according to the linked documentation, Nomad Network can be installed as usual with pip.
+
+For a native Android application with a graphical user interface, have a look at `_`!`[Sideband`a8d24177d946de4f1f0a0fe1af9a1338:/page/repo.mu`g=reticulum|r=sideband]`!`_.
+
+>> Help & Discussion
+
+For help requests, discussion, sharing ideas or anything else related to Nomad Network, please have a look at the `_`!`[rns.recipes forum`9ce92808be498e9e05590ff27cbfdfe4]`!`_.
+
+>> Support Nomad Network
+
+For this to be possible, I need your help. Please support the continued development of open, free and private communications systems by donating via one of the following channels:
+
+• `!Monero`!
+  84FpY1QbxHcgdseePYNmhTHcrgMX4nFfBYtz2GKYToqHVVhJp8Eaw1Z1EedRnKD19b3B8NiLCGVxzKV17UMmmeEsCrPyA5w
+
+• `!Bitcoin`!
+  bc1pgqgu8h8xvj4jtafslq396v7ju7hkgymyrzyqft4llfslz5vp99psqfk3a6
+
+• `!Ethereum`!
+  0x91C421DdfB8a30a49A71d63447ddb54cEBe3465E
+
+• `!Liberapay`!
+  `[https://liberapay.com/Reticulum/]
+
+• `!Ko-Fi`!
+  `[https://ko-fi.com/markqvist]
+
+>> Caveat Emptor
+
+Nomad Network is experimental software, and should be considered as such. Use at your own risk and responsibility.

nomadnet/Directory.py

@@ -2,98 +2,356 @@ import os
 import RNS
 import LXMF
 import time
+import nomadnet
+import threading
 import RNS.vendor.umsgpack as msgpack
 
+from LXMF import pn_announce_data_is_valid, PN_META_IMPL_NAME, PN_META_VERSION
+from nomadnet.util import strip_modifiers
+from nomadnet.util import sanitize_name
+
+class PNAnnounceHandler:
+    def __init__(self, owner):
+        self.aspect_filter = "lxmf.propagation"
+        self.owner = owner
+
+    def received_announce(self, destination_hash, announced_identity, app_data):
+        try:
+            if pn_announce_data_is_valid(app_data):
+                data = msgpack.unpackb(app_data)
+
+                if data[2] == True:
+                    metadata = data[6]
+                    if not PN_META_IMPL_NAME in metadata or not PN_META_VERSION in metadata: impl_str = ""
+                    else: impl_str = f" running {metadata[PN_META_IMPL_NAME]} {metadata[PN_META_VERSION]}"
+                    RNS.log(f"Received active propagation node announce from {RNS.prettyhexrep(destination_hash)}{impl_str}", RNS.LOG_DEBUG)
+
+                    associated_peer = RNS.Destination.hash_from_name_and_identity("lxmf.delivery", announced_identity)
+                    associated_node = RNS.Destination.hash_from_name_and_identity("nomadnetwork.node", announced_identity)
+
+                    self.owner.app.directory.pn_announce_received(destination_hash, app_data, associated_peer, associated_node)
+                    self.owner.app.autoselect_propagation_node()
+
+        except Exception as e:
+            RNS.log("Error while evaluating propagation node announce, ignoring announce.", RNS.LOG_DEBUG)
+            RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG)
+
 class Directory:
     ANNOUNCE_STREAM_MAXLENGTH = 256
 
+    aspect_filter = "nomadnetwork.node"
+    @staticmethod
+    def received_announce(destination_hash, announced_identity, app_data):
+        try:
+            app = nomadnet.NomadNetworkApp.get_shared_instance()
+
+            if not destination_hash in app.ignored_list:
+                associated_peer = RNS.Destination.hash_from_name_and_identity("lxmf.delivery", announced_identity)
+
+                app.directory.node_announce_received(destination_hash, app_data, associated_peer)
+                app.autoselect_propagation_node()
+                
+            else:
+                RNS.log("Ignored announce from "+RNS.prettyhexrep(destination_hash), RNS.LOG_DEBUG)
+
+        except Exception as e:
+            RNS.log("Error while evaluating LXMF destination announce, ignoring announce.", RNS.LOG_DEBUG)
+            RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG)
+
+
+    @property
+    def announce_stream(self):
+        return self._node_announces+self._peer_announces+self._pn_announces
+
+    def _clean_node_announces(self):
+        while len(self._node_announces) > Directory.ANNOUNCE_STREAM_MAXLENGTH:
+            self._node_announces.pop()
+
+    def _clean_peer_announces(self):
+        while len(self._peer_announces) > Directory.ANNOUNCE_STREAM_MAXLENGTH:
+            self._peer_announces.pop()
+
+    def _clean_pn_announces(self):
+        while len(self._pn_announces) > Directory.ANNOUNCE_STREAM_MAXLENGTH:
+            self._pn_announces.pop()
+
     def __init__(self, app):
         self.directory_entries = {}
-        self.announce_stream = []
+        self._node_announces = []
+        self._peer_announces = []
+        self._pn_announces   = []
         self.app = app
+        self.announce_lock = threading.Lock()
         self.load_from_disk()
 
+        self.pn_announce_handler = PNAnnounceHandler(self)
+        RNS.Transport.register_announce_handler(self.pn_announce_handler)
+
+
     def save_to_disk(self):
         try:
             packed_list = []
             for source_hash in self.directory_entries:
                 e = self.directory_entries[source_hash]
-                packed_list.append((e.source_hash, e.display_name, e.trust_level, e.hosts_node))
+                packed_list.append((e.source_hash, e.display_name, e.trust_level, e.hosts_node, e.preferred_delivery, e.identify, e.sort_rank, getattr(e, "notes", "")))
 
             directory = {
                 "entry_list": packed_list,
                 "announce_stream": self.announce_stream
             }
 
-            file = open(self.app.directorypath, "wb")
-            file.write(msgpack.packb(directory))
-            file.close()
+            with open(self.app.directorypath, "wb") as file:
+                file.write(msgpack.packb(directory))
+
         except Exception as e:
             RNS.log("Could not write directory to disk. Then contained exception was: "+str(e), RNS.LOG_ERROR)
 
     def load_from_disk(self):
         if os.path.isfile(self.app.directorypath):
             try:
-                file = open(self.app.directorypath, "rb")
-                unpacked_directory = msgpack.unpackb(file.read())
+                with open(self.app.directorypath, "rb") as file:
+                    unpacked_directory = msgpack.unpackb(file.read())
                 unpacked_list = unpacked_directory["entry_list"]
-                file.close()
 
                 entries = {}
                 for e in unpacked_list:
+
+                    if e[1] == None:
+                        e[1] = "Undefined"
+
                     if len(e) > 3:
                         hosts_node = e[3]
                     else:
                         hosts_node = False
 
-                    entries[e[0]] = DirectoryEntry(e[0], e[1], e[2], hosts_node)
+                    if len(e) > 4:
+                        preferred_delivery = e[4]
+                    else:
+                        preferred_delivery = None
+
+                    if len(e) > 5:
+                        identify = e[5]
+                    else:
+                        identify = False
+
+                    if len(e) > 6:
+                        sort_rank = e[6]
+                    else:
+                        sort_rank = None
+
+                    if len(e) > 7:
+                        notes = e[7]
+                    else:
+                        notes = None
+
+                    entries[e[0]] = DirectoryEntry(e[0], e[1], e[2], hosts_node, preferred_delivery=preferred_delivery, identify_on_connect=identify, sort_rank=sort_rank, notes=notes)
 
                 self.directory_entries = entries
-                self.announce_stream = unpacked_directory["announce_stream"]
+
+                astream = unpacked_directory["announce_stream"]
+                self._node_announces = [e for e in astream if e[3] == "node"]
+                self._peer_announces = [e for e in astream if e[3] == "peer"]
+                self._pn_announces = [e for e in astream if e[3] == "pn"]
 
             except Exception as e:
                 RNS.log("Could not load directory from disk. The contained exception was: "+str(e), RNS.LOG_ERROR)
 
-    def announce_received(self, source_hash, app_data):
-        timestamp = time.time()
-        self.announce_stream.insert(0, (timestamp, source_hash, app_data))
-        while len(self.announce_stream) > Directory.ANNOUNCE_STREAM_MAXLENGTH:
-            self.announce_stream.pop()
+    def lxmf_announce_received(self, source_hash, app_data):
+        with self.announce_lock:
+            if app_data != None:
+                if self.app.compact_stream:
+                    try:
+                        remove_announces = []
+                        for announce in self._peer_announces:
+                            if announce[1] == source_hash: remove_announces.append(announce)
+
+                        for a in remove_announces: self._peer_announces.remove(a)
+                    
+                    except Exception as e:
+                        RNS.log("An error occurred while compacting the announce stream. The contained exception was:"+str(e), RNS.LOG_ERROR)
+
+                timestamp = time.time()
+                self._peer_announces.insert(0, (timestamp, source_hash, app_data, "peer"))
+                self._clean_peer_announces()
+
+                if hasattr(self.app, "ui") and self.app.ui != None:
+                    if hasattr(self.app.ui, "main_display"):
+                        self.app.ui.main_display.sub_displays.network_display.directory_change_callback()
+
+    def node_announce_received(self, source_hash, app_data, associated_peer):
+        with self.announce_lock:
+            if app_data != None:
+                if self.app.compact_stream:
+                    try:
+                        remove_announces = []
+                        for announce in self._node_announces:
+                            if announce[1] == source_hash:
+                                remove_announces.append(announce)
+
+                        for a in remove_announces:
+                            self._node_announces.remove(a)
+                    
+                    except Exception as e:
+                        RNS.log("An error occurred while compacting the announce stream. The contained exception was:"+str(e), RNS.LOG_ERROR)
+
+                timestamp = time.time()
+                self._node_announces.insert(0, (timestamp, source_hash, app_data, "node"))
+                self._clean_node_announces()
+
+                # if self.trust_level(associated_peer) == DirectoryEntry.TRUSTED:
+                #     existing_entry = self.find(source_hash)
+                #     if not existing_entry:
+                #         node_entry = DirectoryEntry(source_hash, display_name=app_data.decode("utf-8"), trust_level=DirectoryEntry.TRUSTED, hosts_node=True)
+                #         self.remember(node_entry)
+                
+                if hasattr(self.app.ui, "main_display"):
+                    self.app.ui.main_display.sub_displays.network_display.directory_change_callback()
+
+    def pn_announce_received(self, source_hash, app_data, associated_peer, associated_node):
+        with self.announce_lock:
+            found_node = None
+            for sh in self.directory_entries:
+                if sh == associated_node:
+                    found_node = True
+                    break
+
+            for e in self._pn_announces:
+                if e[1] == associated_node:
+                    found_node = True
+                    break
+
+            # TODO: Remove debug and rethink this (needs way to set PN when node is saved)
+            if True or not found_node:
+                if self.app.compact_stream:
+                    try:
+                        remove_announces = []
+                        for announce in self._pn_announces:
+                            if announce[1] == source_hash:
+                                remove_announces.append(announce)
+
+                        for a in remove_announces:
+                            self._pn_announces.remove(a)
+                    
+                    except Exception as e:
+                        RNS.log("An error occurred while compacting the announce stream. The contained exception was:"+str(e), RNS.LOG_ERROR)
+
+                timestamp = time.time()
+                self._pn_announces.insert(0, (timestamp, source_hash, app_data, "pn"))
+                self._clean_pn_announces()
+                
+                if hasattr(self.app, "ui") and hasattr(self.app.ui, "main_display"):
+                    self.app.ui.main_display.sub_displays.network_display.directory_change_callback()
+
+    def remove_announce_with_timestamp(self, timestamp):
+        selected_announce = None
+        for announce in self._node_announces:
+            if announce[0] == timestamp:
+                selected_announce = announce
+                break
+
+        if selected_announce != None:
+            self._node_announces.remove(selected_announce)
+            return
+
+        for announce in self._peer_announces:
+            if announce[0] == timestamp:
+                selected_announce = announce
+                break
+
+        if selected_announce != None:
+            self._peer_announces.remove(selected_announce)
+            return
+
+        for announce in self._pn_announces:
+            if announce[0] == timestamp:
+                selected_announce = announce
+                break
+
+        if selected_announce != None:
+            self._pn_announces.remove(selected_announce)
+            return
 
     def display_name(self, source_hash):
         if source_hash in self.directory_entries:
-            return self.directory_entries[source_hash].display_name
+            return strip_modifiers(self.directory_entries[source_hash].display_name)
         else:
             return None
 
-    def simplest_display_str(self, source_hash):
+    def simplest_display_str(self, source_hash, san=True):
+        def s(name):
+            if self.app.config["textui"]["sanitize_names"] and san: return sanitize_name(name)
+            else:                                                   return strip_modifiers(name)
+
         trust_level = self.trust_level(source_hash)
         if trust_level == DirectoryEntry.WARNING or trust_level == DirectoryEntry.UNTRUSTED:
-            return "<"+RNS.hexrep(source_hash, delimit=False)+">"
+            if source_hash in self.directory_entries:
+                dn = s(self.directory_entries[source_hash].display_name)
+                if not dn: return RNS.prettyhexrep(source_hash)
+                else:      return dn+" <"+RNS.hexrep(source_hash, delimit=False)+">"
+            
+            else: return "<"+RNS.hexrep(source_hash, delimit=False)+">"
+        
         else:
             if source_hash in self.directory_entries:
-                return self.directory_entries[source_hash].display_name
-            else:
-                return "<"+RNS.hexrep(source_hash, delimit=False)+">"
+                dn = s(self.directory_entries[source_hash].display_name)
+                if not dn: return RNS.prettyhexrep(source_hash)
+                else:      return dn
+            
+            else: return "<"+RNS.hexrep(source_hash, delimit=False)+">"
+
+    def alleged_display_str(self, source_hash):
+        if source_hash in self.directory_entries:
+            return strip_modifiers(self.directory_entries[source_hash].display_name)
+        else:
+            return None
+
 
     def trust_level(self, source_hash, announced_display_name=None):
         if source_hash in self.directory_entries:
             if announced_display_name == None:
                 return self.directory_entries[source_hash].trust_level
             else:
-                for entry in self.directory_entries:
-                    e = self.directory_entries[entry]
-                    if e.display_name == announced_display_name:
-                        if e.source_hash != source_hash:
-                            return DirectoryEntry.WARNING
+                if not self.directory_entries[source_hash].trust_level == DirectoryEntry.TRUSTED:
+                    for entry in self.directory_entries:
+                        e = self.directory_entries[entry]
+                        if e.display_name == announced_display_name:
+                            if e.source_hash != source_hash:
+                                return DirectoryEntry.WARNING
 
                 return self.directory_entries[source_hash].trust_level
         else:
             return DirectoryEntry.UNKNOWN
 
+    def pn_trust_level(self, source_hash):
+        recalled_identity = RNS.Identity.recall(source_hash)
+        if recalled_identity != None:
+            associated_node = RNS.Destination.hash_from_name_and_identity("nomadnetwork.node", recalled_identity)
+            return self.trust_level(associated_node)
+
+    def sort_rank(self, source_hash):
+        if source_hash in self.directory_entries:
+            return self.directory_entries[source_hash].sort_rank
+        else:
+            return None
+
+    def preferred_delivery(self, source_hash):
+        if source_hash in self.directory_entries:
+            return self.directory_entries[source_hash].preferred_delivery
+        else:
+            return DirectoryEntry.DIRECT
+
     def remember(self, entry):
         self.directory_entries[entry.source_hash] = entry
 
+        identity = RNS.Identity.recall(entry.source_hash)
+        if identity != None:
+            associated_node = RNS.Destination.hash_from_name_and_identity("nomadnetwork.node", identity)
+            if associated_node in self.directory_entries:
+                node_entry = self.directory_entries[associated_node]
+                node_entry.trust_level = entry.trust_level
+
+        self.save_to_disk()
+
     def forget(self, source_hash):
         if source_hash in self.directory_entries:
             self.directory_entries.pop(source_hash)
@@ -116,6 +374,18 @@ class Directory:
         except Exception as e:
             return False
 
+    def should_identify_on_connect(self, source_hash):
+        if source_hash in self.directory_entries:
+            entry = self.directory_entries[source_hash]
+            return entry.identify
+        else:
+            return False
+
+    def set_identify_on_connect(self, source_hash, state):
+        if source_hash in self.directory_entries:
+            entry = self.directory_entries[source_hash]
+            entry.identify = state
+    
     def known_nodes(self):
         node_list = []
         for eh in self.directory_entries:
@@ -123,6 +393,7 @@ class Directory:
             if e.hosts_node:
                 node_list.append(e)
 
+        node_list.sort(key = lambda e: (e.sort_rank if e.sort_rank != None else 2^32, DirectoryEntry.TRUSTED-e.trust_level, e.display_name if e.display_name != None else "_"))
         return node_list
 
     def number_of_known_nodes(self):
@@ -144,14 +415,24 @@ class DirectoryEntry:
     UNKNOWN   = 0x02
     TRUSTED   = 0xFF
 
-    def __init__(self, source_hash, display_name=None, trust_level=UNKNOWN, hosts_node=False):
+    DIRECT        = 0x01
+    PROPAGATED    = 0x02
+    OPPORTUNISTIC = 0x03
+
+    def __init__(self, source_hash, display_name=None, trust_level=UNKNOWN, hosts_node=False, preferred_delivery=None, identify_on_connect=False, sort_rank=None, notes=None):
         if len(source_hash) == RNS.Identity.TRUNCATED_HASHLENGTH//8:
             self.source_hash  = source_hash
             self.display_name = display_name
-            if display_name  == None:
-                display_name  = source_hash
+            self.sort_rank = sort_rank
+
+            if preferred_delivery == None:
+                self.preferred_delivery = DirectoryEntry.DIRECT
+            else:
+                self.preferred_delivery = preferred_delivery
 
             self.trust_level  = trust_level
             self.hosts_node   = hosts_node
+            self.identify     = identify_on_connect
+            self.notes        = notes or ""
         else:
-            raise TypeError("Attempt to add invalid source hash to directory")
+            raise TypeError("Attempt to add invalid source hash to directory")

nomadnet/InterfaceProfiles.py

@@ -0,0 +1,240 @@
+import os
+import threading
+
+import RNS
+import RNS.vendor.umsgpack as msgpack
+from nomadnet.util import write_config_atomic
+
+
+class InterfaceProfiles:
+    VERSION = 1
+
+    def __init__(self, app):
+        self.app = app
+        self.path = os.path.join(app.storagepath, "interface_profiles.msgpack")
+        self._lock = threading.Lock()
+        self.profiles = []
+        self.load()
+
+    def load(self):
+        self.profiles = []
+        self.default_members = None
+        try:
+            if not os.path.isfile(self.path):
+                return
+            with open(self.path, "rb") as f:
+                data = msgpack.unpackb(f.read())
+            if not isinstance(data, dict):
+                return
+            clean = []
+            seen = set()
+            for p in data.get("profiles") or []:
+                if not isinstance(p, dict):
+                    continue
+                pid = str(p.get("id") or "")
+                if not pid or pid in seen:
+                    continue
+                members = [str(m) for m in (p.get("members") or []) if isinstance(m, (str, bytes))]
+                clean.append({"id": pid, "name": str(p.get("name") or ""), "members": members})
+                seen.add(pid)
+            self.profiles = clean
+            dm = data.get("default_members")
+            self.default_members = [str(m) for m in dm] if isinstance(dm, list) else None
+        except Exception as e:
+            RNS.log("Could not load interface profiles: "+str(e), RNS.LOG_ERROR)
+            self.profiles = []
+            self.default_members = None
+
+    def save(self):
+        with self._lock:
+            tmp = self.path + ".tmp"
+            try:
+                payload = {
+                    "version": InterfaceProfiles.VERSION,
+                    "profiles": [{"id": p["id"], "name": p["name"], "members": list(p["members"])} for p in self.profiles],
+                    "default_members": self.default_members,
+                }
+                with open(tmp, "wb") as f:
+                    f.write(msgpack.packb(payload))
+                    f.flush()
+                    try: os.fsync(f.fileno())
+                    except Exception: pass
+                os.replace(tmp, self.path)
+            except Exception as e:
+                RNS.log("Could not save interface profiles: "+str(e), RNS.LOG_ERROR)
+                try: os.remove(tmp)
+                except Exception: pass
+
+    def _interfaces(self):
+        try:
+            return self.app.rns.config["interfaces"]
+        except Exception:
+            return {}
+
+    def interface_names(self):
+        try:
+            return list(self._interfaces().keys())
+        except Exception:
+            return []
+
+    def _is_enabled(self, iface):
+        return str(iface.get("enabled")).lower() not in ('false', 'off', 'no', '0') and \
+               str(iface.get("interface_enabled")).lower() not in ('false', 'off', 'no', '0')
+
+    def enabled_set(self):
+        interfaces = self._interfaces()
+        result = set()
+        for name in interfaces:
+            try:
+                if self._is_enabled(interfaces[name]):
+                    result.add(name)
+            except Exception:
+                pass
+        return result
+
+    def get(self, pid):
+        for p in self.profiles:
+            if p["id"] == pid:
+                return p
+        return None
+
+    def members_existing(self, p):
+        existing = set(self.interface_names())
+        return set(m for m in p["members"] if m in existing)
+
+    def active_profile_id(self):
+        enabled = self.enabled_set()
+        for p in self.profiles:
+            if self.members_existing(p) == enabled:
+                return p["id"]
+        return None
+
+    def profiles_for(self, iface_name):
+        return [p for p in self.profiles if iface_name in p["members"]]
+
+    def label_for(self, iface_name):
+        names = [p["name"] or p["id"] for p in self.profiles_for(iface_name)]
+        return ", ".join(names) if names else None
+
+    def member_count(self, p):
+        return len(self.members_existing(p))
+
+    def _new_id(self):
+        while True:
+            pid = os.urandom(4).hex()
+            if not self.get(pid):
+                return pid
+
+    def create(self, name):
+        pid = self._new_id()
+        self.profiles.append({"id": pid, "name": name or "Profile", "members": []})
+        self.save()
+        return pid
+
+    def save_current_as_profile(self, name):
+        pid = self.create(name)
+        self.set_members(pid, sorted(self.enabled_set()))
+        return pid
+
+    def rename(self, pid, name):
+        p = self.get(pid)
+        if p is not None and name:
+            p["name"] = name
+            self.save()
+
+    def delete(self, pid):
+        self.profiles = [p for p in self.profiles if p["id"] != pid]
+        self.save()
+
+    def set_members(self, pid, members):
+        p = self.get(pid)
+        if p is not None:
+            valid = set(self.interface_names())
+            seen = set()
+            ordered = []
+            for m in members:
+                if m in valid and m not in seen:
+                    ordered.append(m)
+                    seen.add(m)
+            p["members"] = ordered
+            self.save()
+
+    def set_interface_profiles(self, iface_name, profile_ids):
+        pset = set(profile_ids)
+        for p in self.profiles:
+            if p["id"] in pset:
+                if iface_name not in p["members"]:
+                    p["members"].append(iface_name)
+            elif iface_name in p["members"]:
+                p["members"] = [m for m in p["members"] if m != iface_name]
+        self.save()
+
+    def _apply_members(self, members):
+        existing = set(self.interface_names())
+        members = set(m for m in members if m in existing)
+        interfaces = self._interfaces()
+        try:
+            for name in interfaces:
+                value = "true" if name in members else "false"
+                interfaces[name]["interface_enabled"] = value
+                if "enabled" in interfaces[name]:
+                    interfaces[name]["enabled"] = value
+            write_config_atomic(self.app.rns.config)
+            return True
+        except Exception as e:
+            RNS.log("Could not apply interface profile: "+str(e), RNS.LOG_ERROR)
+            return False
+
+    def select_profile(self, pid):
+        p = self.get(pid)
+        if p is None:
+            return False
+        return self._apply_members(self.members_existing(p))
+
+    def select_default(self):
+        if self.default_members is None:
+            return False
+        return self._apply_members(set(self.default_members))
+
+    def update_default_if_custom(self):
+        # Remember the live enabled-set as "Default" whenever it matches no named
+        # profile -- i.e. it's the user's manual setup. Only relevant once profiles exist.
+        if not self.profiles:
+            return
+        enabled = self.enabled_set()
+        matches_named = any(self.members_existing(p) == enabled for p in self.profiles)
+        if matches_named and self.default_members is not None:
+            return
+        new_default = sorted(enabled)
+        if self.default_members != new_default:
+            self.default_members = new_default
+            self.save()
+
+    def remove_interface(self, iface_name):
+        changed = False
+        for p in self.profiles:
+            if iface_name in p["members"]:
+                p["members"] = [m for m in p["members"] if m != iface_name]
+                changed = True
+        if changed:
+            self.save()
+
+    def rename_interface(self, old_name, new_name):
+        changed = False
+        for p in self.profiles:
+            if old_name in p["members"]:
+                p["members"] = [new_name if m == old_name else m for m in p["members"]]
+                changed = True
+        if changed:
+            self.save()
+
+    def prune(self):
+        valid = set(self.interface_names())
+        changed = False
+        for p in self.profiles:
+            pruned = [m for m in p["members"] if m in valid]
+            if len(pruned) != len(p["members"]):
+                p["members"] = pruned
+                changed = True
+        if changed:
+            self.save()

nomadnet/Node.py

@@ -0,0 +1,401 @@
+import os
+import sys
+
+import RNS
+import time
+import threading
+import hashlib
+import shutil
+import subprocess
+import RNS.vendor.umsgpack as msgpack
+
+from RNS.Utilities.rngit.media import convert_file_to_webp
+
+class Node:
+    JOB_INTERVAL = 5
+    SETTINGS_FLUSH_INTERVAL = 60
+    START_ANNOUNCE_DELAY = 6
+
+    def __init__(self, app):
+        RNS.log("Nomad Network Node starting...", RNS.LOG_VERBOSE)
+        self.app = app
+        self.identity = self.app.identity
+        self.destination = RNS.Destination(self.identity, RNS.Destination.IN, RNS.Destination.SINGLE, "nomadnetwork", "node")
+        self.last_announce = time.time()
+        self.last_file_refresh = time.time()
+        self.last_page_refresh = time.time()
+        self.announce_interval = self.app.node_announce_interval
+        self.page_refresh_interval = self.app.page_refresh_interval
+        self.file_refresh_interval = self.app.file_refresh_interval
+        self.job_interval = Node.JOB_INTERVAL
+        self.should_run_jobs = True
+        self.settings_dirty = False
+        self.conversion_cache_lock = threading.Lock()
+        self.last_settings_flush = time.time()
+        self.app_data = None
+        self.name = self.app.node_name
+
+        self.register_pages()
+        self.register_files()
+        self.register_media()
+
+        self.destination.set_link_established_callback(self.peer_connected)
+
+        if self.name == None:
+            self.name = self.app.peer_settings["display_name"]+"'s Node"
+
+        RNS.log("Node \""+self.name+"\" ready for incoming connections on "+RNS.prettyhexrep(self.destination.hash), RNS.LOG_VERBOSE)
+
+        if self.app.node_announce_at_start:
+            def delayed_announce():
+                time.sleep(Node.START_ANNOUNCE_DELAY)
+                self.announce()
+
+            da_thread = threading.Thread(target=delayed_announce)
+            da_thread.setDaemon(True)
+            da_thread.start()
+
+        job_thread = threading.Thread(target=self.__jobs)
+        job_thread.setDaemon(True)
+        job_thread.start()
+
+
+    def register_pages(self):
+        # TODO: Deregister previously registered pages
+        # that no longer exist.
+        self.servedpages = []
+        self.scan_pages(self.app.pagespath)
+
+        if not self.app.pagespath+"index.mu" in self.servedpages:
+            self.destination.register_request_handler(
+                "/page/index.mu",
+                response_generator = self.serve_default_index,
+                allow = RNS.Destination.ALLOW_ALL)
+
+        for page in self.servedpages:
+            request_path = "/page"+page.replace(self.app.pagespath, "")
+            self.destination.register_request_handler(
+                request_path,
+                response_generator = self.serve_page,
+                allow = RNS.Destination.ALLOW_ALL)
+
+    def register_media(self):
+        self.destination.register_request_handler("/media", response_generator = self.serve_media,
+                                                  allow = RNS.Destination.ALLOW_ALL)
+
+    def register_files(self):
+        # TODO: Deregister previously registered files
+        # that no longer exist.
+        self.servedfiles = []
+        self.scan_files(self.app.filespath)
+
+        for file in self.servedfiles:
+            request_path = "/file"+file.replace(self.app.filespath, "")
+            self.destination.register_request_handler(
+                request_path,
+                response_generator = self.serve_file,
+                allow = RNS.Destination.ALLOW_ALL,
+                auto_compress = 32_000_000)
+
+    def scan_pages(self, base_path):
+        files = [file for file in os.listdir(base_path) if os.path.isfile(os.path.join(base_path, file)) and file[:1] != "."]
+        directories = [file for file in os.listdir(base_path) if os.path.isdir(os.path.join(base_path, file)) and file[:1] != "."]
+
+        for file in files:
+            if not file.endswith(".allowed"):
+                self.servedpages.append(base_path+"/"+file)
+
+        for directory in directories:
+            self.scan_pages(base_path+"/"+directory)
+
+    def scan_files(self, base_path):
+        files = [file for file in os.listdir(base_path) if os.path.isfile(os.path.join(base_path, file)) and file[:1] != "."]
+        directories = [file for file in os.listdir(base_path) if os.path.isdir(os.path.join(base_path, file)) and file[:1] != "."]
+
+        for file in files:
+            if not file.endswith(".allowed"):
+                self.servedfiles.append(base_path+"/"+file)
+
+        for directory in directories:
+            self.scan_files(base_path+"/"+directory)
+
+    def serve_page(self, path, data, request_id, link_id, remote_identity, requested_at):
+        RNS.log("Page request "+RNS.prettyhexrep(request_id)+" for: "+str(path), RNS.LOG_VERBOSE)
+        try:
+            self.app.peer_settings["served_page_requests"] += 1
+            self.settings_dirty = True
+            
+        except Exception as e:
+            RNS.log("Could not increase served page request count", RNS.LOG_ERROR)
+
+        file_path = path.replace("/page", self.app.pagespath, 1)
+        request_allowed = self.request_allowed(file_path, remote_identity)
+
+        try:
+            if request_allowed:
+                RNS.log("Serving page: "+file_path, RNS.LOG_VERBOSE)
+                if not RNS.vendor.platformutils.is_windows() and os.access(file_path, os.X_OK):
+                    env_map = {}
+                    if "PATH" in os.environ:
+                        env_map["PATH"] = os.environ["PATH"]
+                    if link_id != None:
+                        env_map["link_id"] = RNS.hexrep(link_id, delimit=False)
+                    if remote_identity != None:
+                        env_map["remote_identity"] = RNS.hexrep(remote_identity.hash, delimit=False)
+
+                    if data != None and isinstance(data, dict):
+                        for e in data:
+                            if isinstance(e, str) and (e.startswith("field_") or e.startswith("var_")):
+                                env_map[e] = data[e]
+
+                    generated = subprocess.run([file_path], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env_map)
+                    return generated.stdout
+                else:
+                    with open(file_path, "rb") as fh:
+                        response_data = fh.read()
+                    return response_data
+            else:
+                RNS.log("Request denied", RNS.LOG_VERBOSE)
+                return DEFAULT_NOTALLOWED.encode("utf-8")
+
+        except Exception as e:
+            RNS.log("Error occurred while handling request "+RNS.prettyhexrep(request_id)+" for: "+str(path), RNS.LOG_ERROR)
+            RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
+            return None
+
+    NATIVE_MEDIA_EXTS = [".webp"]
+    MEDIA_EXTS = [".webp", ".png", ".jpg", ".jpeg", ".bmp", ".gif", ".tiff"]
+    CONVERSION_QUALITY = 85
+    CONVERSION_MAX_DIMENSION = 1200
+    CONVERSION_CACHE_MAX_FILES = 256
+    CONVERSION_CACHE_MAX_BYTES = 192*1024*1024
+    def serve_media(self, path, data, request_id, link_id, remote_identity, requested_at):
+        if not type(data) == dict: return None
+        if not "path" in data: return None
+        if not "key" in data: return None
+
+        jail       = os.path.normpath(self.app.pagespath)
+        media_path = os.path.join(self.app.pagespath, data["path"].replace("/media/", "").lstrip("/"))
+        media_path = os.path.normpath(media_path)
+        base_name  = os.path.basename(media_path)
+        base_ext   = os.path.splitext(base_name)[-1]
+
+        if not base_ext.lower() in self.MEDIA_EXTS:
+            RNS.log(f"Invalid media request type: {media_path}, must be in {self.MEDIA_EXTS}", RNS.LOG_DEBUG)
+            return False
+
+        if not media_path.startswith(jail+os.sep):
+            RNS.log(f"Invalid media request path: {media_path}", RNS.LOG_DEBUG)
+            return False
+
+        if len(media_path) > 512:
+            RNS.log(f"Invalid media request path length: {len(media_path)}", RNS.LOG_DEBUG)
+            return False
+
+        RNS.log(f"Media request {RNS.prettyhexrep(request_id)} for: {media_path}", RNS.LOG_VERBOSE)
+        try:
+            self.app.peer_settings["served_media_requests"] += 1
+            self.settings_dirty = True
+        except Exception as e: RNS.log("Could not increase served page request count", RNS.LOG_ERROR)
+
+        request_allowed = self.request_allowed(media_path, remote_identity)
+        try:
+            if request_allowed:
+                if not base_ext.lower() in self.NATIVE_MEDIA_EXTS:
+                    converted_path = self.convert_media_to_webp(media_path)
+                    if converted_path is False:
+                        RNS.log(f"Could not convert {media_path} to WebP", RNS.LOG_DEBUG)
+                        return False
+                    file_path = converted_path
+                    file_name = os.path.splitext(base_name)[0]+".webp"
+                else:
+                    file_path = media_path
+                    file_name = base_name
+
+                RNS.log(f"Serving media: {file_path}", RNS.LOG_VERBOSE)
+                return [open(file_path, "rb"), {"name": file_name.encode("utf-8")}]
+            
+            else:
+                RNS.log("Request denied", RNS.LOG_VERBOSE)
+                return False
+
+        except Exception as e:
+            RNS.log("Error occurred while handling request "+RNS.prettyhexrep(request_id)+" for: "+str(media_path), RNS.LOG_ERROR)
+            RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
+            return False
+
+    def convert_media_to_webp(self, media_path, quality = CONVERSION_QUALITY, max_dimension = CONVERSION_MAX_DIMENSION):
+        try:
+            hasher = hashlib.sha256()
+            with open(media_path, "rb") as fh:
+                for chunk in iter(lambda: fh.read(65536), b""): hasher.update(chunk)
+            source_hash = hasher.hexdigest()
+        except Exception as e:
+            RNS.log(f"Could not read media source for conversion: {media_path}", RNS.LOG_WARNING)
+            return False
+
+        if quality == None: quality = Node.CONVERSION_QUALITY
+        if max_dimension == None: max_dimension = Node.CONVERSION_MAX_DIMENSION
+
+        cache_name  = f"{source_hash}.q{quality}.d{max_dimension if max_dimension is not None else 0}.webp"
+        cache_path  = os.path.join(self.app.convcachepath, cache_name)
+        return self.__cached_conversion(media_path, cache_path, quality, max_dimension)
+
+    def __cached_conversion(self, media_path, cache_path, quality, max_dimension):
+        with self.conversion_cache_lock:
+            if os.path.isfile(cache_path):
+                RNS.log(f"Using cached media conversion for: {media_path}", RNS.LOG_VERBOSE)
+                try: os.utime(cache_path, None)
+                except Exception: pass
+                return cache_path
+
+            try:
+                os.makedirs(self.app.convcachepath, exist_ok=True)
+                converted_path = convert_file_to_webp(media_path, quality=quality, max_dimension=max_dimension)
+                if converted_path is False: return False
+
+                shutil.move(converted_path, cache_path)
+                self._prune_conversion_cache()
+                RNS.log(f"Converted media: {media_path} -> {cache_path}", RNS.LOG_VERBOSE)
+                return cache_path
+
+            except Exception as e:
+                RNS.log(f"Error during media conversion of {media_path}: {e}", RNS.LOG_WARNING)
+                return False
+
+    def _prune_conversion_cache(self):
+        try:
+            entries = []
+            for name in os.listdir(self.app.convcachepath):
+                path = os.path.join(self.app.convcachepath, name)
+                try:
+                    st = os.stat(path)
+                    if st.st_size > 0:
+                        entries.append((st.st_mtime, st.st_size, path))
+                except OSError: pass
+
+            total = sum(e[1] for e in entries)
+            entries.sort()
+            while len(entries) > Node.CONVERSION_CACHE_MAX_FILES or total > Node.CONVERSION_CACHE_MAX_BYTES:
+                _, size, path = entries.pop(0)
+                total -= size
+                try: os.unlink(path)
+                except OSError: pass
+        except OSError: pass
+
+    def request_allowed(self, file_path, remote_identity):
+        if file_path.lower().endswith(".allowed"): return False
+        allowed_path = file_path+".allowed"
+        if not os.path.isfile(allowed_path): return True
+
+        allowed_list = []
+        try:
+            if os.access(allowed_path, os.X_OK):
+                allowed_input = subprocess.run([allowed_path], stdout=subprocess.PIPE).stdout
+            else:
+                with open(allowed_path, "rb") as fh:
+                    allowed_input = fh.read()
+
+            for hash_str in allowed_input.splitlines():
+                if len(hash_str) == RNS.Identity.TRUNCATED_HASHLENGTH//8*2:
+                    try:
+                        allowed_list.append(bytes.fromhex(hash_str.decode("utf-8")))
+                    except Exception as e:
+                        RNS.log("Could not decode RNS Identity hash from: "+str(hash_str), RNS.LOG_DEBUG)
+                        RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG)
+
+        except Exception as e:
+            RNS.log("Error while fetching list of allowed identities for request: "+str(e), RNS.LOG_ERROR)
+
+        if hasattr(remote_identity, "hash") and remote_identity.hash in allowed_list:
+            return True
+
+        RNS.log("Denying request, remote identity was not in list of allowed identities", RNS.LOG_VERBOSE)
+        return False
+
+    def serve_file(self, path, data, request_id, remote_identity, requested_at):
+        RNS.log("File request "+RNS.prettyhexrep(request_id)+" for: "+str(path), RNS.LOG_VERBOSE)
+        try:
+            self.app.peer_settings["served_file_requests"] += 1
+            self.settings_dirty = True
+            
+        except Exception as e:
+            RNS.log("Could not increase served file request count", RNS.LOG_ERROR)
+
+        file_path = path.replace("/file", self.app.filespath, 1)
+        file_name = os.path.basename(file_path)
+        if not self.request_allowed(file_path, remote_identity):
+            RNS.log("Request denied", RNS.LOG_VERBOSE)
+            return DEFAULT_NOTALLOWED.encode("utf-8")
+
+        try:
+            RNS.log("Serving file: "+file_path, RNS.LOG_VERBOSE)
+            return [open(file_path, "rb"), {"name": file_name.encode("utf-8")}]
+
+        except Exception as e:
+            RNS.log("Error occurred while handling request "+RNS.prettyhexrep(request_id)+" for: "+str(path), RNS.LOG_ERROR)
+            RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
+            return None
+
+    def serve_default_index(self, path, data, request_id, remote_identity, requested_at):
+        RNS.log("Serving default index for request "+RNS.prettyhexrep(request_id)+" for: "+str(path), RNS.LOG_VERBOSE)
+        return DEFAULT_INDEX.encode("utf-8")
+
+    def announce(self):
+        self.app_data = self.name.encode("utf-8")
+        self.last_announce = time.time()
+        self.app.peer_settings["node_last_announce"] = self.last_announce
+        self.destination.announce(app_data=self.app_data)
+        if not self.app.disable_propagation: self.app.message_router.announce_propagation_node()
+
+    def __jobs(self):
+        while self.should_run_jobs:
+            now = time.time()
+            
+            if now > self.last_announce + self.announce_interval*60:
+                self.announce()
+                
+            if self.page_refresh_interval > 0:
+                if now > self.last_page_refresh + self.page_refresh_interval*60:
+                    self.register_pages()
+                    self.last_page_refresh = time.time()
+
+            if self.file_refresh_interval > 0:
+                if now > self.last_file_refresh + self.file_refresh_interval*60:
+                    self.register_files()
+                    self.last_file_refresh = time.time()
+
+            if self.settings_dirty and now > self.last_settings_flush + Node.SETTINGS_FLUSH_INTERVAL:
+                self.settings_dirty = False
+                self.last_settings_flush = now
+                try: self.app.save_peer_settings()
+                except Exception as e: RNS.log("Could not save peer settings: "+str(e), RNS.LOG_ERROR)
+
+            time.sleep(self.job_interval)
+
+    def peer_connected(self, link):
+        RNS.log("Peer connected to "+str(self.destination), RNS.LOG_VERBOSE)
+        try:
+            self.app.peer_settings["node_connects"] += 1
+            self.settings_dirty = True
+
+        except Exception as e:
+            RNS.log("Could not increase node connection count", RNS.LOG_ERROR)
+
+        link.set_link_closed_callback(self.peer_disconnected)
+
+    def peer_disconnected(self, link):
+        RNS.log("Peer disconnected from "+str(self.destination), RNS.LOG_VERBOSE)
+        pass
+
+DEFAULT_INDEX = '''>Default Home Page
+
+This node is serving pages, but the home page file (index.mu) was not found in the page storage directory. This is an auto-generated placeholder.
+
+If you are the node operator, you can define your own home page by creating a file named `*index.mu`* in the page storage directory.
+'''
+
+DEFAULT_NOTALLOWED = '''>Request Not Allowed
+
+You are not authorised to carry out the request.
+'''

nomadnet/__init__.py

@@ -3,11 +3,15 @@ import glob
 
 from .NomadNetworkApp import NomadNetworkApp
 from .Conversation import Conversation
+from .Directory import Directory
+from .Node import Node
 from .ui import *
 
 
-modules = glob.glob(os.path.dirname(__file__)+"/*.py")
-__all__ = [ os.path.basename(f)[:-3] for f in modules if not f.endswith('__init__.py')]
+py_modules  = glob.glob(os.path.dirname(__file__)+"/*.py")
+pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
+modules     = py_modules+pyc_modules
+__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
 
 def panic():
     os._exit(255)

nomadnet/_version.py

@@ -1 +1 @@
-__version__ = "0.0.5"
+__version__ = "1.4.4"

nomadnet/examples/messageboard/README.md

@@ -0,0 +1,18 @@
+# lxmf_messageboard
+Simple message board that can be hosted on a NomadNet node, messages can be posted by 'conversing' with a unique peer, all messages are then forwarded to the message board.
+
+## How Do I Use It?
+A user can submit messages to the message board by initiating a chat with the message board peer, they are assigned a username (based on the first 5 characters of their address) and their messages are added directly to the message board. The message board can be viewed on a page hosted by a NomadNet node.
+
+An example message board can be found on the reticulum testnet hosted on the SolarExpress Node `<d16df67bff870a8eaa2af6957c5a2d7d>` and the message board peer `<ad713cd3fedf36cc190f0cb89c4be1ff>`
+
+## How Does It Work?
+The message board page itself is hosted on a NomadNet node, you can place the message_board.mu into the pages directory. You can then run the message_board.py script which provides the peer that the users can send messages to. The two parts are joined together using umsgpack and a flat file system similar to NomadNet and Reticulum and runs in the background.
+
+## How Do I Set It Up?
+* Turn on node hosting in NomadNet
+* Put the `message_board.mu` file into `pages` directory in the config file for `NomadNet`. Edit the file to customise from the default page.
+* Run the `message_board.py` script (`python3 message_board.py` either in a `screen` or as a system service), this script uses `NomadNet` and `RNS` libraries and has no additional libraries that need to be installed. Take a note of the message boards address, it is printed on starting the board, you can then place this address in `message_board.mu` file to make it easier for users to interact the board.
+
+## Credits
+* This example application was written and contributed by @chengtripp

nomadnet/examples/messageboard/messageboard.mu

@@ -0,0 +1,41 @@
+#!/bin/python3
+import time
+import os
+import RNS.vendor.umsgpack as msgpack
+
+message_board_peer = 'please_replace'
+userdir = os.path.expanduser("~")
+
+if os.path.isdir("/etc/nomadmb") and os.path.isfile("/etc/nomadmb/config"):
+    configdir = "/etc/nomadmb"
+elif os.path.isdir(userdir+"/.config/nomadmb") and os.path.isfile(userdir+"/.config/nomadmb/config"):
+    configdir = userdir+"/.config/nomadmb"
+else:
+    configdir = userdir+"/.nomadmb"
+
+storagepath  = configdir+"/storage"
+if not os.path.isdir(storagepath):
+    os.makedirs(storagepath)
+
+boardpath = configdir+"/storage/board"
+
+print('`!`F222`Bddd`cNomadNet Message Board')
+
+print('-')
+print('`a`b`f')
+print("")
+print("To add a message to the board just converse with the NomadNet Message Board at `[lxmf@{}]".format(message_board_peer))
+time_string = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
+print("Last Updated: {}".format(time_string))
+print("")
+print('>Messages')
+print("  Date       Time    Username     Message")
+f = open(boardpath, "rb")
+board_contents = msgpack.unpack(f)
+board_contents.reverse()
+
+for content in board_contents:
+    print("`a{}".format(content.rstrip()))
+    print("")
+
+f.close()

nomadnet/examples/messageboard/messageboard.py

@@ -0,0 +1,187 @@
+# Simple message board that can be hosted on a NomadNet node, messages can be posted by 'conversing' with a unique peer, all messages are then forwarded to the message board.
+# https://github.com/chengtripp/lxmf_messageboard
+
+import RNS
+import LXMF
+import os, time
+from queue import Queue
+import RNS.vendor.umsgpack as msgpack
+
+display_name = "NomadNet Message Board"
+max_messages = 20
+
+def setup_lxmf():
+    if os.path.isfile(identitypath):
+        identity = RNS.Identity.from_file(identitypath)
+        RNS.log('Loaded identity from file', RNS.LOG_INFO)
+    else:
+        RNS.log('No Primary Identity file found, creating new...', RNS.LOG_INFO)
+        identity = RNS.Identity()
+        identity.to_file(identitypath)
+
+    return identity
+
+def lxmf_delivery(message):
+    # Do something here with a received message
+    RNS.log("A message was received: "+str(message.content.decode('utf-8')))
+
+    message_content = message.content.decode('utf-8')
+    source_hash_text = RNS.hexrep(message.source_hash, delimit=False)
+
+    #Create username (just first 5 char of your addr)
+    username = source_hash_text[0:5]
+
+    RNS.log('Username: {}'.format(username), RNS.LOG_INFO)
+
+    time_string = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(message.timestamp))
+    new_message = '{} {}: {}\n'.format(time_string, username, message_content)
+
+    # Push message to board
+    # First read message board (if it exists
+    if os.path.isfile(boardpath):
+        f = open(boardpath, "rb")
+        message_board = msgpack.unpack(f)
+        f.close()
+    else:
+        message_board = []
+
+    #Check we aren't doubling up (this can sometimes happen if there is an error initially and it then gets fixed)
+    if new_message not in message_board:
+        # Append our new message to the list
+        message_board.append(new_message)
+
+    # Prune the message board if needed
+    while len(message_board) > max_messages:
+        RNS.log('Pruning Message Board')
+        message_board.pop(0)
+
+    # Now open the board and write the updated list
+    f = open(boardpath, "wb")
+    msgpack.pack(message_board, f)
+    f.close()
+
+    # Send reply
+    message_reply = '{}_{}_Your message has been added to the messageboard'.format(source_hash_text, time.time())
+    q.put(message_reply)
+
+def announce_now(lxmf_destination):
+    lxmf_destination.announce()
+
+def send_message(destination_hash, message_content):
+    try:
+      # Make a binary destination hash from a hexadecimal string
+      destination_hash = bytes.fromhex(destination_hash)
+
+    except Exception as e:
+      RNS.log("Invalid destination hash", RNS.LOG_ERROR)
+      return
+
+    # Check that size is correct
+    if not len(destination_hash) == RNS.Reticulum.TRUNCATED_HASHLENGTH//8:
+      RNS.log("Invalid destination hash length", RNS.LOG_ERROR)
+
+    else:
+      # Length of address was correct, let's try to recall the
+      # corresponding Identity
+      destination_identity = RNS.Identity.recall(destination_hash)
+
+      if destination_identity == None:
+        # No path/identity known, we'll have to abort or request one
+        RNS.log("Could not recall an Identity for the requested address. You have probably never received an announce from it. Try requesting a path from the network first. In fact, let's do this now :)", RNS.LOG_ERROR)
+        RNS.Transport.request_path(destination_hash)
+        RNS.log("OK, a path was requested. If the network knows a path, you will receive an announce with the Identity data shortly.", RNS.LOG_INFO)
+
+      else:
+        # We know the identity for the destination hash, let's
+        # reconstruct a destination object.
+        lxmf_destination = RNS.Destination(destination_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, "lxmf", "delivery")
+
+        # Create a new message object
+        lxm = LXMF.LXMessage(lxmf_destination, local_lxmf_destination, message_content, title="Reply", desired_method=LXMF.LXMessage.DIRECT)
+
+        # You can optionally tell LXMF to try to send the message
+        # as a propagated message if a direct link fails
+        lxm.try_propagation_on_fail = True
+
+        # Send it
+        message_router.handle_outbound(lxm)
+
+def announce_check():
+    if os.path.isfile(announcepath):
+        f = open(announcepath, "r")
+        announce = int(f.readline())
+        f.close()
+    else:
+        RNS.log('failed to open announcepath', RNS.LOG_DEBUG)
+        announce = 1
+
+    if announce > int(time.time()):
+        RNS.log('Recent announcement', RNS.LOG_DEBUG)
+    else:
+        f = open(announcepath, "w")
+        next_announce = int(time.time()) + 1800
+        f.write(str(next_announce))
+        f.close()
+        announce_now(local_lxmf_destination)
+        RNS.log('Announcement sent, expr set 1800 seconds', RNS.LOG_INFO)
+
+#Setup Paths and Config Files
+userdir = os.path.expanduser("~")
+
+if os.path.isdir("/etc/nomadmb") and os.path.isfile("/etc/nomadmb/config"):
+    configdir = "/etc/nomadmb"
+elif os.path.isdir(userdir+"/.config/nomadmb") and os.path.isfile(userdir+"/.config/nomadmb/config"):
+    configdir = userdir+"/.config/nomadmb"
+else:
+    configdir = userdir+"/.nomadmb"
+
+storagepath  = configdir+"/storage"
+if not os.path.isdir(storagepath):
+    os.makedirs(storagepath)
+
+identitypath = configdir+"/storage/identity"
+announcepath = configdir+"/storage/announce"
+boardpath = configdir+"/storage/board"
+
+# Message Queue
+q = Queue(maxsize = 5)
+
+# Start Reticulum and print out all the debug messages
+reticulum = RNS.Reticulum(loglevel=RNS.LOG_VERBOSE)
+
+# Create a Identity.
+current_identity = setup_lxmf()
+
+# Init the LXMF router
+message_router = LXMF.LXMRouter(identity = current_identity, storagepath = configdir)
+
+# Register a delivery destination (for yourself)
+# In this example we use the same Identity as we used
+# to instantiate the LXMF router. It could be a different one,
+# but it can also just be the same, depending on what you want.
+local_lxmf_destination = message_router.register_delivery_identity(current_identity, display_name=display_name)
+
+# Set a callback for when a message is received
+message_router.register_delivery_callback(lxmf_delivery)
+
+# Announce node properties
+
+RNS.log('LXMF Router ready to receive on: {}'.format(RNS.prettyhexrep(local_lxmf_destination.hash)), RNS.LOG_INFO)
+announce_check()
+
+while True:
+
+    # Work through internal message queue
+    for i in list(q.queue):
+        message_id = q.get()
+        split_message = message_id.split('_')
+        destination_hash = split_message[0]
+        message = split_message[2]
+        RNS.log('{} {}'.format(destination_hash, message), RNS.LOG_INFO)
+        send_message(destination_hash, message)
+
+    # Check whether we need to make another announcement
+    announce_check()
+
+    #Sleep
+    time.sleep(10)

nomadnet/examples/various/input_fields.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+import os
+env_string = ""
+for e in os.environ:
+  env_string += "{}={}\n".format(e, os.environ[e])
+
+template = """>Fields and Submitting Data
+
+Nomad Network let's you use simple input fields for submitting data to node-side applications. Submitted data, along with other session variables will be available to the node-side script / program as environment variables. This page contains a few examples.
+
+>> Read Environment Variables
+
+{@ENV}
+>>Examples of Fields and Submissions
+
+The following section contains a simple set of fields, and a few different links that submit the field data in different ways.
+
+-=
+
+
+>>>Text Fields
+An input field    : `B444`<username`Entered data>`b
+
+An masked field   : `B444`<!|password`Value of Field>`b
+
+An small field    : `B444`<8|small`test>`b, and some more text.
+
+A multi-row field : `B444`<40x4|notes`>`b
+
+Two fields        : `B444`<8|one`One>`b `B444`<8|two`Two>`b
+
+The data can be `!`[submitted`:/page/input_fields.mu`username|two]`!.
+
+>> Checkbox Fields
+
+`B444`<?|sign_up|1|*`>`b Sign me up 
+
+>> Radio group
+
+Select your favorite color:
+
+`B900`<^|color|Red`>`b  Red
+
+`B090`<^|color|Green`>`b Green
+
+`B009`<^|color|Blue`>`b Blue
+
+
+>>> Submitting data
+
+You can `!`[submit`:/page/input_fields.mu`one|password|small|color]`! other fields, or just `!`[a single one`:/page/input_fields.mu`username]`!
+
+Or simply `!`[submit them all`:/page/input_fields.mu`*]`!.
+
+Submission links can also `!`[include pre-configured variables`:/page/input_fields.mu`username|two|entitiy_id=4611|action=view]`!.
+
+Or take all fields and `!`[pre-configured variables`:/page/input_fields.mu`*|entitiy_id=4611|action=view]`!.
+
+Or only `!`[pre-configured variables`:/page/input_fields.mu`entitiy_id=4688|task=something]`!
+
+-=
+
+"""
+print(template.replace("{@ENV}", env_string))

nomadnet/nomadnet.py

@@ -2,18 +2,27 @@
 
 from ._version import __version__
 
+import io
 import argparse
 import nomadnet
 
 
-def program_setup(configdir, rnsconfigdir):
-    app = nomadnet.NomadNetworkApp(configdir = configdir, rnsconfigdir = rnsconfigdir)
+def program_setup(configdir, rnsconfigdir, daemon, console):
+    app = nomadnet.NomadNetworkApp(
+        configdir = configdir,
+        rnsconfigdir = rnsconfigdir,
+        daemon = daemon,
+        force_console = console,
+    )
 
 def main():
     try:
         parser = argparse.ArgumentParser(description="Nomad Network Client")
         parser.add_argument("--config", action="store", default=None, help="path to alternative Nomad Network config directory", type=str)
         parser.add_argument("--rnsconfig", action="store", default=None, help="path to alternative Reticulum config directory", type=str)
+        parser.add_argument("-t", "--textui", action="store_true", default=False, help="run Nomad Network in text-UI mode")
+        parser.add_argument("-d", "--daemon", action="store_true", default=False, help="run Nomad Network in daemon mode")
+        parser.add_argument("-c", "--console", action="store_true", default=False, help="in daemon mode, log to console instead of file")
         parser.add_argument("--version", action="version", version="Nomad Network Client {version}".format(version=__version__))
         
         args = parser.parse_args()
@@ -28,7 +37,18 @@ def main():
         else:
             rnsconfigarg = None
 
-        program_setup(configarg, rnsconfigarg)
+        console = False
+        if args.daemon:
+            daemon = True
+            if args.console:
+                console = True
+        else:
+            daemon = False
+
+        if args.textui:
+            daemon = False
+
+        program_setup(configarg, rnsconfigarg, daemon, console)
 
     except KeyboardInterrupt:
         print("")

nomadnet/ui/MenuUI.py

@@ -4,5 +4,5 @@ import nomadnet
 class MenuUI:
 
     def __init__(self):
-        RNS.log("Menu UI not implemented", RNS.LOG_ERROR)
+        RNS.log("Menu UI not implemented", RNS.LOG_ERROR, _override_destination=True)
         nomadnet.panic()

nomadnet/ui/NoneUI.py

@@ -0,0 +1,19 @@
+import RNS
+import nomadnet
+import time
+
+from nomadnet import NomadNetworkApp
+
+class NoneUI:
+
+    def __init__(self):
+        self.app = NomadNetworkApp.get_shared_instance()
+        self.app.ui = self
+
+        if not self.app.force_console_log:
+            RNS.log("Nomad Network started in daemon mode, all further messages are logged to "+str(self.app.logfilepath), RNS.LOG_INFO, _override_destination=True)
+        else:
+            RNS.log("Nomad Network daemon started", RNS.LOG_INFO)
+
+        while True:
+            time.sleep(1)

nomadnet/ui/TextUI.py

@@ -1,9 +1,13 @@
 import RNS
-import importlib
+import urwid
 import time
+import os
+import platform
 
 import nomadnet
+from nomadnet.ui import THEME_DARK, THEME_LIGHT
 from nomadnet.ui.textui import *
+from nomadnet.ui.textui.images import ImageScreen
 from nomadnet import NomadNetworkApp
 
 COLORMODE_MONO = 1
@@ -11,61 +15,244 @@ COLORMODE_16   = 16
 COLORMODE_88   = 88
 COLORMODE_256  = 256
 COLORMODE_TRUE = 2**24
-THEME_DARK     = 0x01
-THEME_LIGHT    = 0x02
+
+IMGLOAD_NEVER  = 0x00
+IMGLOAD_MANUAL = 0x01
+IMGLOAD_AUTO   = 0x02
+IMGLOAD_ALWAYS = 0x03
 
 THEMES = {
-    THEME_DARK: [
-        # Style name                    # 16-color style                        # Monochrome style          # 88, 256 and true-color style
-        ('heading',                     'light gray,underline', 'default',      'underline',                'g93,underline', 'default'),
-        ('menubar',                     'black', 'light gray',                  'standout',                 '#111', '#bbb'),
-        ('shortcutbar',                 'black', 'light gray',                  'standout',                 '#111', '#bbb'),
-        ('body_text',                   'white', 'default',                     'default',                  '#0a0', 'default'),
-        ('error_text',                  'dark red', 'default',                  'default',                  'dark red', 'default'),
-        ('warning_text',                'yellow', 'default',                    'default',                  '#ba4', 'default'),
-        ('inactive_text',               'dark gray', 'default',                 'default',                  'dark gray', 'default'),
-        ('buttons',                     'light green,bold', 'default',          'default',                  '#00a533', 'default'),
-        ('msg_editor',                  'black', 'light cyan',                  'standout',                 '#111', '#0bb'),
-        ("msg_header_ok",               'black', 'light green',                 'standout',                 '#111', '#6b2'),
-        ("msg_header_caution",          'black', 'yellow',                      'standout',                 '#111', '#fd3'),
-        ("msg_header_sent",             'black', 'light gray',                  'standout',                 '#111', '#ddd'),
-        ("msg_header_delivered",        'black', 'light blue',                  'standout',                 '#111', '#28b'),
-        ("msg_header_failed",           'black', 'dark gray',                   'standout',                 'black', 'dark gray'),
-        ("msg_warning_untrusted",       'black', 'dark red',                    'standout',                 '#111', 'dark red'),
-        ("list_focus",                  "black", "light gray",                  "standout",                 "#111", "#bbb"),
-        ("list_off_focus",              "black", "dark gray",                   "standout",                 "#111", "dark gray"),
-        ("list_trusted",                "light green", "default",               "default",                  "#6b2", "default"),
-        ("list_focus_trusted",          "black", "light gray",                  "standout",                 "#180", "#bbb"),
-        ("list_unknown",                "dark gray", "default",                 "default",                  "light gray", "default"),
-        ("list_untrusted",              "dark red", "default",                  "default",                  "dark red", "default"),
-        ("list_focus_untrusted",        "black", "light gray",                  "standout",                 "#810", "#bbb"),
-    ]
+    THEME_DARK: {
+        "urwid_theme": [
+            # Style name                    # 16-color style                        # Monochrome style          # 88, 256 and true-color style
+            ("heading",                     "light gray,underline", "default",      "underline",                "g93,underline", "default"),
+            ("menubar",                     "black", "light gray",                  "standout",                 "#111", "#bbb"),
+            ("scrollbar",                   "light gray", "default",                "default",                  "#444", "default"),
+            ("shortcutbar",                 "black", "light gray",                  "standout",                 "#111", "#bbb"),
+            ("body_text",                   "light gray", "default",                "default",                  "#ddd", "default"),
+            ("error_text",                  "dark red", "default",                  "default",                  "dark red", "default"),
+            ("warning_text",                "yellow", "default",                    "default",                  "#ba4", "default"),
+            ("inactive_text",               "dark gray", "default",                 "default",                  "dark gray", "default"),
+            ("browser_inactive",            "dark gray", "default",                 "default",                  "#444", "default"),
+            ("buttons",                     "light green,bold", "default",          "default",                  "#00a533", "default"),
+            ("msg_editor",                  "black", "light cyan",                  "standout",                 "#111", "#0bb"),
+            ("msg_header_ok",               "black", "light green",                 "standout",                 "#111", "#6b2"),
+            ("msg_header_caution",          "black", "yellow",                      "standout",                 "#111", "#fd3"),
+            ("msg_header_sent",             "black", "light gray",                  "standout",                 "#111", "#ddd"),
+            ("msg_header_propagated",       "black", "light blue",                  "standout",                 "#111", "#28b"),
+            ("msg_header_delivered",        "black", "light blue",                  "standout",                 "#111", "#28b"),
+            ("msg_header_failed",           "black", "dark gray",                   "standout",                 "#000", "#777"),
+            ("msg_warning_untrusted",       "black", "dark red",                    "standout",                 "#111", "dark red"),
+            ("msg_notice_unread",           "light blue", "default",                "standout",                 "#28b", "default"),
+            ("msg_notice_caution",          "yellow", "default",                    "standout",                 "#fd3", "default"),
+            ("list_focus",                  "black", "light gray",                  "standout",                 "#111", "#aaa"),
+            ("list_off_focus",              "black", "dark gray",                   "standout",                 "#111", "#777"),
+            ("list_trusted",                "dark green", "default",                "default",                  "#6b2", "default"),
+            ("list_focus_trusted",          "black", "light gray",                  "standout",                 "#150", "#aaa"),
+            ("list_unknown",                "dark gray", "default",                 "default",                  "#bbb", "default"),
+            ("list_normal",                 "dark gray", "default",                 "default",                  "#bbb", "default"),
+            ("list_untrusted",              "dark red", "default",                  "default",                  "#a22", "default"),
+            ("list_focus_untrusted",        "black", "light gray",                  "standout",                 "#810", "#aaa"),
+            ("list_unresponsive",           "yellow", "default",                    "default",                  "#b92", "default"),
+            ("list_focus_unresponsive",     "black", "light gray",                  "standout",                 "#530", "#aaa"),
+            ("topic_list_normal",           "light gray", "default",                "default",                  "#ddd", "default"),
+            ("browser_controls",            "light gray", "default",                "default",                  "#bbb", "default"),
+            ("progress_full",               "black", "light gray",                  "standout",                 "#111", "#bbb"),
+            ("progress_empty",              "light gray", "default",                "default",                  "#ddd", "default"),
+            ("interface_title",             "", "",                                 "default",                  "", ""),
+            ("interface_title_selected",    "bold", "",                             "bold",                     "", ""),
+            ("interface_tile_focus",        "light blue,bold", "default",           "bold",                     "#5af,bold", "default"),
+            ("connected_status",            "dark green", "default",                "default",                  "dark green", "default"),
+            ("disconnected_status",         "dark red", "default",                   "default",                  "dark red", "default"),
+            ("disabled_status",             "dark red", "default",                   "default",                  "dark red", "default"),
+            ("placeholder",                 "dark gray", "default",                 "default",                  "dark gray", "default"),
+            ("placeholder_text",            "dark gray", "default",                 "default",                  "dark gray", "default"),
+            ("error",                       "light red,blink",                      "default", "blink",         "#f44,blink", "default"),
+            ("irc_ts",                      "dark gray", "default",                 "default",                  "#888", "default"),
+            ("irc_nick_self",               "light green", "default",               "default",                  "#6c5", "default"),
+            ("irc_nick_peer",               "light cyan", "default",                "default",                  "#3cd", "default"),
+            ("irc_notice",                  "yellow", "default",                    "default",                  "#fd3", "default"),
+            ("irc_error",                   "light red", "default",                 "default",                  "#f55", "default"),
+            ("irc_system",                  "dark gray", "default",                 "default",                  "#888", "default"),
+            ("irc_mention",                 "light red,bold", "default",            "bold",                     "#fb4,bold", "default"),
+
+        ],
+    },
+    THEME_LIGHT: {
+        "urwid_theme": [
+            # Style name                    # 16-color style                        # Monochrome style          # 88, 256 and true-color style
+            ("heading",                     "dark gray,underline", "default",      "underline",                "g93,underline", "default"),
+            ("menubar",                     "black", "dark gray",                  "standout",                 "#111", "#bbb"),
+            ("scrollbar",                   "dark gray", "default",                "default",                  "#444", "default"),
+            ("shortcutbar",                 "black", "dark gray",                  "standout",                 "#111", "#bbb"),
+            ("body_text",                   "dark gray", "default",                "default",                  "#222", "default"),
+            ("error_text",                  "dark red", "default",                 "default",                  "dark red", "default"),
+            ("warning_text",                "yellow", "default",                   "default",                  "#ba4", "default"),
+            ("inactive_text",               "light gray", "default",               "default",                  "dark gray", "default"),
+            ("buttons",                     "light green,bold", "default",         "default",                  "#00a533", "default"),
+            ("msg_editor",                  "black", "dark cyan",                  "standout",                 "#111", "#0bb"),
+            ("msg_header_ok",               "black", "dark green",                 "standout",                 "#111", "#6b2"),
+            ("msg_header_caution",          "black", "yellow",                     "standout",                 "#111", "#fd3"),
+            ("msg_header_sent",             "black", "dark gray",                  "standout",                 "#111", "#ddd"),
+            ("msg_header_propagated",       "black", "light blue",                 "standout",                 "#111", "#28b"),
+            ("msg_header_delivered",        "black", "light blue",                 "standout",                 "#111", "#28b"),
+            ("msg_header_failed",           "black", "dark gray",                  "standout",                 "#000", "#777"),
+            ("msg_warning_untrusted",       "black", "dark red",                   "standout",                 "#111", "dark red"),
+            ("msg_notice_unread",           "dark blue", "default",               "standout",                  "#069", "default"),
+            ("msg_notice_caution",          "yellow", "default",                   "standout",                 "#fd3", "default"),
+            ("list_focus",                  "black", "dark gray",                  "standout",                 "#111", "#aaa"),
+            ("list_off_focus",              "black", "dark gray",                  "standout",                 "#111", "#777"),
+            ("list_trusted",                "dark green", "default",               "default",                  "#4a0", "default"),
+            ("list_focus_trusted",          "black", "dark gray",                  "standout",                 "#150", "#aaa"),
+            ("list_unknown",                "dark gray", "default",                "default",                  "#444", "default"),
+            ("list_normal",                 "dark gray", "default",                "default",                  "#444", "default"),
+            ("list_untrusted",              "dark red", "default",                 "default",                  "#a22", "default"),
+            ("list_focus_untrusted",        "black", "dark gray",                  "standout",                 "#810", "#aaa"),
+            ("list_unresponsive",           "yellow", "default",                   "default",                  "#b92", "default"),
+            ("list_focus_unresponsive",     "black", "light gray",                 "standout",                 "#530", "#aaa"),
+            ("topic_list_normal",           "dark gray", "default",                "default",                  "#222", "default"),
+            ("browser_controls",            "dark gray", "default",                "default",                  "#444", "default"),
+            ("progress_full",               "black", "dark gray",                  "standout",                 "#111", "#bbb"),
+            ("progress_empty",              "dark gray", "default",                "default",                  "#ddd", "default"),
+            ("interface_title",             "dark gray", "default",                "default",                  "#444", "default"),
+            ("interface_title_selected",    "dark gray,bold", "default",           "bold",                     "#444,bold", "default"),
+            ("interface_tile_focus",        "dark blue,bold", "default",           "bold",                     "#06c,bold", "default"),
+            ("connected_status",            "dark green", "default",               "default",                  "#4a0", "default"),
+            ("disconnected_status",         "dark red", "default",                  "default",                  "#a22", "default"),
+            ("disabled_status",             "dark red", "default",                  "default",                  "#a22", "default"),
+            ("placeholder",                 "light gray", "default",               "default",                  "#999", "default"),
+            ("placeholder_text",            "light gray", "default",               "default",                  "#999", "default"),
+            ("error",                       "dark red,blink", "default",           "blink",                    "#a22,blink", "default"),
+            ("irc_ts",                      "dark gray", "default",                "default",                  "#888", "default"),
+            ("irc_nick_self",               "dark green", "default",               "default",                  "#3a0", "default"),
+            ("irc_nick_peer",               "dark cyan", "default",                "default",                  "#077", "default"),
+            ("irc_notice",                  "brown", "default",                    "default",                  "#a70", "default"),
+            ("irc_error",                   "dark red", "default",                 "default",                  "#a22", "default"),
+            ("irc_system",                  "dark gray", "default",                "default",                  "#888", "default"),
+            ("irc_mention",                 "dark red,bold", "default",             "bold",                     "#c50,bold", "default"),
+        ],
+    }
+}
+
+CONFIG_SECTION_COLORS = {
+    "logging":   ("yellow",        "#fd3", "brown",        "#a80"),
+    "client":    ("light cyan",    "#3cd", "dark cyan",    "#077"),
+    "textui":    ("light green",   "#6c5", "dark green",   "#2a0"),
+    "utilities": ("light gray",    "#bbb", "dark gray",    "#555"),
+    "rrc":       ("light magenta", "#d7f", "dark magenta", "#a2a"),
+    "node":      ("light blue",    "#5af", "dark blue",    "#06c"),
+    "printing":  ("brown",         "#b86", "brown",        "#852"),
+    "reticulum": ("light magenta", "#a6f", "dark magenta", "#63c"),
+}
+
+for _section, (_d16, _dhi, _l16, _lhi) in CONFIG_SECTION_COLORS.items():
+    THEMES[THEME_DARK]["urwid_theme"].append(("config_title_"+_section, _d16, "default", "default", _dhi, "default"))
+    THEMES[THEME_DARK]["urwid_theme"].append(("config_title_"+_section+"_selected", _d16+",bold", "default", "bold", _dhi+",bold", "default"))
+    THEMES[THEME_LIGHT]["urwid_theme"].append(("config_title_"+_section, _l16, "default", "default", _lhi, "default"))
+    THEMES[THEME_LIGHT]["urwid_theme"].append(("config_title_"+_section+"_selected", _l16+",bold", "default", "bold", _lhi+",bold", "default"))
+
+GLYPHSETS = {
+    "plain": 1,
+    "unicode": 2,
+    "nerdfont": 3
+}
+
+if platform.system() == "Darwin":
+    urm_char = " \uf0e0"
+    ur_char = "\uf0e0 "
+else:
+    urm_char = " \uf003"
+    ur_char = "\uf003 "
+
+GLYPHS = {
+    # Glyph name        # Plain      # Unicode      # Nerd Font
+    ("check",           "=",         "\u2713",      "\u2713"),
+    ("cross",           "X",         "\u2715",      "\u2715"),
+    ("unknown",         "?",         "?",           "?"),
+    ("encrypted",       "",          "\u26BF",      "\uf023"),
+    ("plaintext",       "!",         "!",           "\uf06e "),
+    ("arrow_r",         "->",        "\u2192",      "\u2192"),
+    ("arrow_l",         "<-",        "\u2190",      "\u2190"),
+    ("arrow_u",         "/\\",       "\u2191",      "\u2191"),
+    ("arrow_d",         "\\/",       "\u2193",      "\u2193"),
+    ("warning",         "!",         "\u26a0",      "\uf12a"),
+    ("info",            "i",         "\u2139",      "\U000f064e"),
+    ("unread",          "[!]",       "\u2709",      ur_char),
+    ("divider1",        "-",         "\u2504",      "\u2504"),
+    ("peer",            "[P]",       "\u24c5 ",     "\uf415"),
+    ("node",            "[N]",       "\u24c3 ",     "\U000f0002"),
+    ("page",            "",          "\u25a4 ",     "\uf719 "),
+    ("speed",           "",          "\u25F7 ",     "\U000f04c5 "),
+    ("decoration_menu", " +",        " +",          " \U000f043b"),
+    ("unread_menu",     " !",        " \u2709",     urm_char),
+    ("globe",           "",          "",            "\uf484"),
+    ("sent",            "/\\",       "\u2191",      "\U000f0cd8"),
+    ("papermsg",        "P",         "\u25a4",      "\uf719"),
+    ("qrcode",          "QR",        "\u25a4",      "\uf029"),
+    ("selected",        "[*] ",      "\u25CF",      "\u25CF"),
+    ("unselected",      "[ ] ",      "\u25CB",      "\u25CB"),
+    ("focus_arrow",     ">",         "\u25B8",      "\u25B8"),
+    ("sep_dot",         "|",         "\u00B7",      "\u00B7"),
+    ("connected",       "+",         "\u2713",      "\U000f0201"),
+    ("disconnected",    "x",         "\u2717",      "\U000f0202"),
+    ("file",            "[F]",       "\u25a4",      "\uf15b"),
+    ("image",           "[I]",       "\u25a3",      "\uf1c5"),
+    ("audio",           "[~]",       "\u266b",      "\uf1c7"),
+    ("pin",             "*",         "\u2605",      "\uf08d"),
+    ("copy",            "[C]",       "\u29c9",      "\uf0c5"),
+    ("folder",          "[+]",       "\u25b8",      "\uf07b"),
+    ("folder_open",     "[-]",       "\u25be",      "\uf07c"),
+    ("fold_open",       "-",         "▾",      "▾"),
+    ("fold_closed",     "+",         "▸",      "▸"),
+    ("dropdown",        " [v]",      " ▾",          " "),
 }
 
 class TextUI:
 
     def __init__(self):
+        self.restore_ixon = False
+
+        try:
+            rval = os.system("stty -a | grep \"\\-ixon\"")
+            if rval == 0:
+                pass
+
+            else:
+                os.system("stty -ixon")
+                self.restore_ixon = True
+
+        except Exception as e:
+            RNS.log("Could not configure terminal flow control sequences, some keybindings may not work.", RNS.LOG_WARNING)
+            RNS.log("The contained exception was: "+str(e))
+
         self.app = NomadNetworkApp.get_shared_instance()
         self.app.ui = self
         self.loop = None
 
-        if importlib.util.find_spec("urwid") != None:
-            import urwid
-        else:
-            RNS.log("The text-mode user interface requires Urwid to be installed on your system.", RNS.LOG_ERROR)
-            RNS.log("You can install it with the command: pip3 install urwid", RNS.LOG_ERROR)
-            nomadnet.panic()
-
         urwid.set_encoding("UTF-8")
 
-        intro_timeout = self.app.config["textui"]["intro_time"]
-        colormode     = self.app.config["textui"]["colormode"]
-        theme         = self.app.config["textui"]["theme"]
+        intro_timeout  = self.app.config["textui"]["intro_time"]
+        colormode      = self.app.config["textui"]["colormode"]
+        theme          = self.app.config["textui"]["theme"]
+        mouse_enabled  = self.app.config["textui"]["mouse_enabled"]
 
-        palette       = THEMES[theme]
+        self.palette   = THEMES[theme]["urwid_theme"]
 
-        self.screen = urwid.raw_display.Screen()
-        self.screen.register_palette(palette)
+        if self.app.config["textui"]["glyphs"] == "plain":
+            glyphset = "plain"
+        elif self.app.config["textui"]["glyphs"] == "unicode":
+            glyphset = "unicode"
+        elif self.app.config["textui"]["glyphs"] == "nerdfont":
+            glyphset = "nerdfont"
+        else:
+            glyphset = "unicode"
+
+        self.glyphs = {}
+        for glyph in GLYPHS:
+            self.glyphs[glyph[0]] = glyph[GLYPHSETS[glyphset]]
+
+        self.screen = ImageScreen()
+        self.screen.register_palette(self.palette)
         
         self.main_display = Main.MainDisplay(self, self.app)
         
@@ -75,18 +262,44 @@ class TextUI:
         else:
             initial_widget = self.main_display.widget
 
-        self.loop = urwid.MainLoop(initial_widget, screen=self.screen)
+        self.loop = urwid.MainLoop(initial_widget, unhandled_input=self.unhandled_input, screen=self.screen, handle_mouse=mouse_enabled)
 
         if intro_timeout > 0:
             self.loop.set_alarm_in(intro_timeout, self.display_main)
-        
+
+        if "KONSOLE_VERSION" in os.environ:
+            if colormode > 16:
+                RNS.log("", RNS.LOG_WARNING, _override_destination = True)
+                RNS.log("", RNS.LOG_WARNING, _override_destination = True)
+                RNS.log("You are using the terminal emulator Konsole.", RNS.LOG_WARNING, _override_destination = True)
+                RNS.log("If you are not seeing the user interface, it is due to a bug in Konsole/urwid.", RNS.LOG_WARNING, _override_destination = True)
+                RNS.log("", RNS.LOG_WARNING, _override_destination = True)
+
+                RNS.log("To circumvent this, use another terminal emulator, or launch nomadnet within a", RNS.LOG_WARNING, _override_destination = True)
+                RNS.log("screen session, using a command like the following:", RNS.LOG_WARNING, _override_destination = True)
+                RNS.log("", RNS.LOG_WARNING, _override_destination = True)
+                RNS.log("screen nomadnet", RNS.LOG_WARNING, _override_destination = True)
+                RNS.log("", RNS.LOG_WARNING, _override_destination = True)
+                RNS.log("Press ctrl-c to exit now and try again.", RNS.LOG_WARNING, _override_destination = True)
+
         self.set_colormode(colormode)
 
+        self.main_display.start()
+        if intro_timeout <= 0 and self.app.peer_settings_notice: self.main_display.show_notice(self.app.peer_settings_notice)
         self.loop.run()
 
     def set_colormode(self, colormode):
+        self.colormode = colormode
         self.screen.set_terminal_properties(colormode)
-        self.screen.reset_default_terminal_palette()
+        
+        if self.colormode < 256:
+            self.screen.reset_default_terminal_palette()
+            self.restore_palette = True
+
+    def unhandled_input(self, key):
+        if key == "ctrl q":   self.main_display.quit()
+        elif key == "ctrl e": pass
 
     def display_main(self, loop, user_data):
         self.loop.widget = self.main_display.widget
+        if self.app.peer_settings_notice: self.main_display.show_notice(self.app.peer_settings_notice)

nomadnet/ui/__init__.py

@@ -3,20 +3,28 @@ import glob
 import RNS
 import nomadnet
 
-modules = glob.glob(os.path.dirname(__file__)+"/*.py")
-__all__ = [ os.path.basename(f)[:-3] for f in modules if not f.endswith('__init__.py')]
+py_modules  = glob.glob(os.path.dirname(__file__)+"/*.py")
+pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
+modules     = py_modules+pyc_modules
+__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))
 
+THEME_DARK     = 0x01
+THEME_LIGHT    = 0x02
 
 UI_NONE       = 0x00
 UI_MENU       = 0x01
 UI_TEXT       = 0x02
 UI_GRAPHICAL  = 0x03
 UI_WEB        = 0x04
-UI_MODES = [UI_MENU, UI_TEXT, UI_GRAPHICAL, UI_WEB]
+UI_MODES = [UI_NONE, UI_MENU, UI_TEXT, UI_GRAPHICAL, UI_WEB]
 
 def spawn(uimode):
     if uimode in UI_MODES:
-        RNS.log("Starting user interface...", RNS.LOG_INFO)
+        if uimode == UI_NONE:
+            RNS.log("Starting Nomad Network daemon...", RNS.LOG_INFO)
+        else:
+            RNS.log("Starting user interface...", RNS.LOG_INFO)
+
         if uimode == UI_MENU:
             from .MenuUI import MenuUI
             return MenuUI()
@@ -29,8 +37,11 @@ def spawn(uimode):
         elif uimode == UI_WEB:
             from .WebUI import WebUI
             return WebUI()
+        elif uimode == UI_NONE:
+            from .NoneUI import NoneUI
+            return NoneUI()
         else:
             return None
     else:
-        RNS.log("Invalid UI mode", RNS.LOG_ERROR)
+        RNS.log("Invalid UI mode", RNS.LOG_ERROR, _override_destination=True)
         nomadnet.panic()

nomadnet/ui/textui/Directory.py

@@ -15,7 +15,7 @@ class DirectoryDisplay():
         ])
 
         self.shortcuts_display = DirectoryDisplayShortcuts(self.app)
-        self.widget = urwid.Filler(pile, 'top')
+        self.widget = urwid.Filler(pile, urwid.TOP)
 
     def shortcuts(self):
         return self.shortcuts_display

nomadnet/ui/textui/Extras.py

@@ -5,57 +5,15 @@ class IntroDisplay():
 
         font = urwid.font.HalfBlock5x4Font()
 
-        big_text = urwid.BigText(("intro_title", "Nomad Network"), font)
-        big_text = urwid.Padding(big_text, align="center", width="clip")
+        big_text = urwid.BigText(("intro_title", self.app.config["textui"]["intro_text"]), font)
+        big_text = urwid.Padding(big_text, align=urwid.CENTER, width=urwid.CLIP)
 
         intro = urwid.Pile([
             big_text,
-            urwid.Text(("Version %s" % (str(self.app.version))), align="center"),
+            urwid.Text(("Version %s" % (str(self.app.version))), align=urwid.CENTER),
             urwid.Divider(),
-            urwid.Text(("-= Starting =- "), align="center"),
+            urwid.Text(("-= Starting =- "), align=urwid.CENTER),
         ])
 
         self.widget = urwid.Filler(intro)
 
-
-class DemoDisplay():
-    def __init__(self, ui, app):
-        import urwid
-
-        def color_mono(btn):
-            ui.set_colormode(nomadnet.ui.COLORMODE_MONO)
-
-        def color_16(btn):
-            ui.set_colormode(nomadnet.ui.COLORMODE_16)
-
-        def color_88(btn):
-            ui.set_colormode(nomadnet.ui.COLORMODE_88)
-
-        def color_8bit(btn):
-            ui.set_colormode(nomadnet.ui.COLORMODE_256)
-
-        def color_true(btn):
-            ui.set_colormode(nomadnet.ui.COLORMODE_TRUE)
-
-        # pile = urwid.Pile([
-        #     urwid.Text(("heading", "This is a heading")),
-        #     urwid.Text(("body_text", "Hello World \U0001F332")),
-        #     urwid.Button(("buttons", "Monochrome"), color_mono),
-        #     urwid.Button(("buttons", "16 color"), color_16),
-        #     urwid.Button(("buttons", "88 color"), color_88),
-        #     urwid.Button(("buttons", "256 color"), color_8bit),
-        #     urwid.Button(("buttons", "True color"), color_true),
-        # ])
-
-        gf = urwid.GridFlow([
-            urwid.Text(("heading", "This is a heading")),
-            urwid.Text(("body_text", "Hello World \U0001F332")),
-            urwid.Button(("buttons", "Monochrome"), color_mono),
-            urwid.Button(("buttons", "16 color"), color_16),
-            urwid.Button(("buttons", "88 color"), color_88),
-            urwid.Button(("buttons", "256 color"), color_8bit),
-            urwid.Button(("buttons", "True color"), color_true),
-        ], cell_width=20, h_sep=0, v_sep=0, align="left")
-
-        self.widget = urwid.Filler(urwid.Padding((urwid.Text("Test"),urwid.Text("Test 2"))), 'top')
-        #self.widget = urwid.Filler(pile, 'top')

nomadnet/ui/textui/FileBrowser.py

@@ -0,0 +1,234 @@
+import os
+import nomadnet
+import urwid
+
+class FileBrowserEntry(urwid.WidgetWrap):
+    signals = ["click"]
+
+    def __init__(self, name, full_path, is_dir=False, is_parent=False, selected=False):
+        self.full_path = full_path
+        self.name = name
+        self.is_dir = is_dir
+        self.is_parent = is_parent
+        self.selected = selected
+        g = nomadnet.NomadNetworkApp.get_shared_instance().ui.glyphs
+        if is_parent:
+            display = g["arrow_l"]+" .."
+        elif is_dir:
+            display = g["arrow_r"]+" "+name+"/"
+        elif selected:
+            display = g["check"]+" "+name
+        else:
+            display = "  "+name
+        self.text_widget = urwid.SelectableIcon(display, 0)
+        if is_dir or is_parent:
+            style = "list_trusted"
+            focus_style = "list_focus"
+        elif selected:
+            style = "list_trusted"
+            focus_style = "list_focus_trusted"
+        else:
+            style = "list_unknown"
+            focus_style = "list_focus"
+        display_widget = urwid.AttrMap(self.text_widget, style, focus_style)
+        super().__init__(display_widget)
+
+    def keypress(self, size, key):
+        if key == "enter":
+            self._emit("click")
+        else:
+            return key
+
+    def mouse_event(self, size, event, button, x, y, focus):
+        if button == 1 and urwid.util.is_mouse_press(event):
+            self._emit("click")
+            return True
+        return False
+
+class FileBrowser(urwid.WidgetWrap):
+    def __init__(self, on_done=None, on_cancel=None, on_change=None, title="Select File", multi_select=True, start_path=None, selected=None, max_file_size=None):
+        self.on_done_cb = on_done
+        self.on_cancel_cb = on_cancel
+        self.on_change_cb = on_change
+        self.multi_select = multi_select
+        self.max_file_size = max_file_size
+        self.selected = list(selected) if selected else []
+
+        app = nomadnet.NomadNetworkApp.get_shared_instance()
+        self.g = app.ui.glyphs
+        self.current_path = os.path.expanduser(start_path) if start_path else os.path.expanduser("~")
+
+        self.path_label = urwid.Text("")
+        self.status_label = urwid.Text("")
+        self.file_walker = urwid.SimpleFocusListWalker([])
+        self.file_listbox = urwid.ListBox(self.file_walker)
+
+        self.button_columns = urwid.Columns([
+            (urwid.WEIGHT, 0.45, urwid.Button("Done", on_press=self._dismiss)),
+            (urwid.WEIGHT, 0.1, urwid.Text("")),
+            (urwid.WEIGHT, 0.45, urwid.Button("Cancel", on_press=self._cancel)),
+        ])
+
+        header_pile = urwid.Pile([
+            self.path_label,
+            self.status_label,
+            urwid.Divider(self.g["divider1"]),
+        ])
+        footer_pile = urwid.Pile([
+            urwid.Divider(self.g["divider1"]),
+            self.button_columns,
+        ])
+
+        self._populate()
+
+        self.browser_frame = urwid.Frame(self.file_listbox, header=header_pile, footer=footer_pile)
+        self._main_widget = urwid.LineBox(self.browser_frame, title=title)
+        self._notice_open = False
+        super().__init__(self._main_widget)
+
+    def _show_notice(self, message, title="Notice"):
+        def dismiss(_b=None):
+            self._notice_open = False
+            self._w = self._main_widget
+
+        box = urwid.LineBox(urwid.Pile([
+            urwid.Text(message, align=urwid.CENTER),
+            urwid.Divider(),
+            urwid.Padding(urwid.Button("OK", on_press=dismiss), align=urwid.CENTER, width=8),
+        ]), title=title)
+        self._notice_open = True
+        self._w = urwid.Overlay(box, self._main_widget, align=urwid.CENTER, width=("relative", 75), valign=urwid.MIDDLE, height=urwid.PACK)
+
+    def _dismiss_notice(self):
+        self._notice_open = False
+        self._w = self._main_widget
+
+    def _update_status(self):
+        if self.selected:
+            names = [os.path.basename(p) for p in self.selected]
+            self.status_label.set_text("  "+self.g["file"]+" "+str(len(self.selected))+" selected: "+", ".join(names))
+        elif self.multi_select:
+            self.status_label.set_text("  No files selected")
+        else:
+            self.status_label.set_text("  Select a file")
+
+    def _populate(self):
+        self.path_label.set_text("  "+self.current_path)
+        self._update_status()
+
+        focus_pos = None
+        try:
+            focus_pos = self.file_listbox.focus_position
+        except Exception:
+            pass
+
+        entries = []
+        parent = os.path.dirname(self.current_path)
+        if parent != self.current_path:
+            entry = FileBrowserEntry("..", parent, is_parent=True)
+            urwid.connect_signal(entry, "click", self._entry_clicked, entry)
+            entries.append(entry)
+
+        try:
+            items = sorted(os.listdir(self.current_path))
+        except PermissionError:
+            entries.append(urwid.Text(("error_text", "  Permission denied")))
+            self.file_walker[:] = entries
+            return
+        except Exception:
+            entries.append(urwid.Text(("error_text", "  Cannot read directory")))
+            self.file_walker[:] = entries
+            return
+
+        dirs = []
+        files = []
+        for item in items:
+            if item.startswith("."):
+                continue
+            full = os.path.join(self.current_path, item)
+            if os.path.isdir(full):
+                dirs.append((item, full))
+            elif os.path.isfile(full):
+                files.append((item, full))
+
+        for name, full in dirs:
+            entry = FileBrowserEntry(name, full, is_dir=True)
+            urwid.connect_signal(entry, "click", self._entry_clicked, entry)
+            entries.append(entry)
+
+        for name, full in files:
+            entry = FileBrowserEntry(name, full, selected=(full in self.selected))
+            urwid.connect_signal(entry, "click", self._entry_clicked, entry)
+            entries.append(entry)
+
+        if not dirs and not files:
+            entries.append(urwid.Text(("inactive_text", "  (empty)")))
+
+        self.file_walker[:] = entries
+        if focus_pos is not None and focus_pos < len(entries):
+            self.file_listbox.set_focus(focus_pos)
+        elif entries:
+            self.file_listbox.set_focus(0)
+
+    def _entry_clicked(self, entry_widget, user_data=None):
+        entry = user_data if user_data else entry_widget
+        if entry.is_dir or entry.is_parent:
+            self.current_path = entry.full_path
+            self._populate()
+            return
+
+        if self.max_file_size is not None and entry.full_path not in self.selected:
+            try:
+                file_size = os.path.getsize(entry.full_path)
+            except Exception:
+                file_size = 0
+            if file_size > self.max_file_size:
+                limit_mb = self.max_file_size / (1024 * 1024)
+                size_mb = file_size / (1024 * 1024)
+                self._show_notice(
+                    "%s is %.1f MB.\nAttachments are limited to %.0f MB." % (entry.name, size_mb, limit_mb),
+                    title="File too large",
+                )
+                return
+
+        if self.multi_select:
+            if entry.full_path in self.selected:
+                self.selected.remove(entry.full_path)
+            else:
+                self.selected.append(entry.full_path)
+            if self.on_change_cb is not None:
+                self.on_change_cb(list(self.selected))
+            self._populate()
+        else:
+            self.selected = [entry.full_path]
+            if self.on_change_cb is not None:
+                self.on_change_cb(list(self.selected))
+            if self.on_done_cb is not None:
+                self.on_done_cb(list(self.selected))
+
+    def _dismiss(self, sender):
+        if self.on_done_cb is not None:
+            self.on_done_cb(list(self.selected))
+
+    def _cancel(self, sender):
+        self.selected = []
+        if self.on_cancel_cb is not None:
+            self.on_cancel_cb()
+
+    def keypress(self, size, key):
+        if self._notice_open:
+            if key in ("esc", "enter"):
+                self._dismiss_notice()
+                return None
+            return super().keypress(size, key)
+        if key == "esc":
+            self._cancel(None)
+            return None
+        result = super().keypress(size, key)
+        if result == "down" and self.browser_frame.focus_position == "body":
+            self.browser_frame.focus_position = "footer"
+            return None
+        elif result == "up" and self.browser_frame.focus_position == "footer":
+            self.browser_frame.focus_position = "body"
+            return None
+        return result

nomadnet/ui/textui/Helpers.py

@@ -0,0 +1,46 @@
+import sys
+import base64
+import urwid
+import RNS
+
+
+def osc52_copy(text):
+    if not text:
+        return False
+    try:
+        encoded = base64.b64encode(text.encode("utf-8")).decode("ascii")
+        sys.stdout.write("\x1b]52;c;" + encoded + "\x07")
+        sys.stdout.flush()
+        return True
+    except Exception as e:
+        RNS.log("Could not emit clipboard escape sequence: "+str(e), RNS.LOG_ERROR)
+        return False
+
+
+def qr_ascii(data):
+    try:
+        import qrcode
+        import io
+        qr = qrcode.QRCode(error_correction=qrcode.constants.ERROR_CORRECT_L, box_size=1, border=1)
+        qr.add_data(data)
+        qr.make()
+        buf = io.StringIO()
+        qr.print_ascii(out=buf, invert=False)
+        return buf.getvalue().rstrip("\n")
+    except Exception as e:
+        RNS.log("QR generation failed: "+str(e), RNS.LOG_ERROR)
+        return None
+
+
+class ClickableIcon(urwid.Text):
+    _selectable = False
+
+    def __init__(self, text, on_click=None):
+        super().__init__(text)
+        self._on_click = on_click
+
+    def mouse_event(self, size, event, button, x, y, focus):
+        if button == 1 and "press" in event and self._on_click is not None:
+            self._on_click()
+            return True
+        return False

nomadnet/ui/textui/Log.py

@@ -0,0 +1,132 @@
+import os
+import sys
+import itertools
+import mmap
+import urwid
+import nomadnet
+from nomadnet.util import strip_modifiers
+
+
+class LogDisplayShortcuts():
+    def __init__(self, app):
+        import urwid
+        self.app = app
+
+        self.widget = urwid.AttrMap(urwid.Text(""), "shortcutbar")
+
+
+class LogDisplay():
+    def __init__(self, app):
+        self.app = app
+
+        self.shortcuts_display = LogDisplayShortcuts(self.app)
+        self.widget = None
+
+    @property
+    def log_term(self):
+        return self.widget
+
+    def show(self):
+        if self.widget is None:
+            self.widget = log_widget(self.app)
+
+    def kill(self):
+        if self.widget is not None:
+            self.widget.terminate()
+            self.widget = None
+        
+    def shortcuts(self):
+        return self.shortcuts_display
+
+
+class LogTerminal(urwid.WidgetWrap):
+    def __init__(self, app):
+        self.app = app
+        self.log_term = urwid.Terminal(
+            ("tail", "-fn50", self.app.logfilepath),
+            encoding='utf-8',
+            escape_sequence="up",
+            main_loop=self.app.ui.loop,
+        )
+        self.widget = urwid.LineBox(self.log_term)
+        super().__init__(self.widget)
+
+    def terminate(self):
+        self.log_term.terminate()
+
+
+    def keypress(self, size, key):
+        if key == "ctrl q":
+            nomadnet.NomadNetworkApp.get_shared_instance().ui.main_display.quit()
+            return None
+
+        if key == "up":
+            nomadnet.NomadNetworkApp.get_shared_instance().ui.main_display.frame.focus_position = "header"
+
+        return super(LogTerminal, self).keypress(size, key)
+
+
+class LogTail(urwid.WidgetWrap):
+    def __init__(self, app):
+        self.app = app
+        self.log_tail = urwid.Text(strip_modifiers(tail(self.app.logfilepath, 50)))
+        self.log = urwid.Scrollable(self.log_tail)
+        self.log.set_scrollpos(-1)
+        self.log_scrollbar = urwid.ScrollBar(self.log)
+        # We have this here because ui.textui.Main depends on this field to kill it
+        self.log_term = None
+
+        super().__init__(self.log_scrollbar)
+
+    def terminate(self):
+        pass
+
+
+def log_widget(app, platform=sys.platform):
+    if platform == "win32":
+        return LogTail(app)
+    else:
+        return LogTerminal(app)
+
+# https://stackoverflow.com/a/34029605/3713120
+def _tail(f_name, n, offset=0):
+    def skip_back_lines(mm: mmap.mmap, numlines: int, startidx: int) -> int:
+        '''Factored out to simplify handling of n and offset'''
+        for _ in itertools.repeat(None, numlines):
+            startidx = mm.rfind(b'\n', 0, startidx)
+            if startidx < 0:
+                break
+        return startidx
+
+    # Open file in binary mode
+    with open(f_name, 'rb') as binf, mmap.mmap(binf.fileno(), 0, access=mmap.ACCESS_READ) as mm:
+        # len(mm) - 1 handles files ending w/newline by getting the prior line
+        startofline = skip_back_lines(mm, offset, len(mm) - 1)
+        if startofline < 0:
+            return []  # Offset lines consumed whole file, nothing to return
+            # If using a generator function (yield-ing, see below),
+            # this should be a plain return, no empty list
+
+        endoflines = startofline + 1  # Slice end to omit offset lines
+
+        # Find start of lines to capture (add 1 to move from newline to beginning of following line)
+        startofline = skip_back_lines(mm, n, startofline) + 1
+
+        # Passing True to splitlines makes it return the list of lines without
+        # removing the trailing newline (if any), so list mimics f.readlines()
+        # return mm[startofline:endoflines].splitlines(True)
+        # If Windows style \r\n newlines need to be normalized to \n
+        return mm[startofline:endoflines].replace(os.linesep.encode(sys.getdefaultencoding()), b'\n').splitlines(True)
+
+
+def tail(f_name, n):
+    """
+    Return the last n lines of a given file name, f_name.
+    Akin to `tail -<n> <f_name>`
+    """
+    def decode(b):
+        return b.decode(encoding)
+
+    encoding = sys.getdefaultencoding()
+    lines = map(decode, _tail(f_name=f_name, n=n))
+    return ''.join(lines)

nomadnet/ui/textui/Main.py

@@ -1,10 +1,17 @@
 import RNS
-import time
 
 from .Network import *
 from .Conversations import *
+from .Channels import *
 from .Directory import *
+from .Config import *
+from .Interfaces import *
+from .Transport import *
+from .Utilities import *
 from .Map import *
+from .Log import *
+from .Guide import *
+from .Editor import *
 import urwid
 
 class SubDisplays():
@@ -12,10 +19,26 @@ class SubDisplays():
         self.app = app
         self.network_display = NetworkDisplay(self.app)
         self.conversations_display = ConversationsDisplay(self.app)
+        self.channels_display = ChannelsDisplay(self.app)
         self.directory_display = DirectoryDisplay(self.app)
+        self.config_display = ConfigDisplay(self.app)
+        self.interface_display = InterfaceDisplay(self.app)
+        self.utilities_display = UtilitiesDisplay(self.app) if self._utilities_enabled(app) else None
         self.map_display = MapDisplay(self.app)
+        self.log_display = LogDisplay(self.app)
+        self.guide_display = GuideDisplay(self.app)
+        self.page_editor_display = PageEditorDisplay(self.app) if app.enable_node else None
 
-        self.active_display = self.conversations_display
+        if app.firstrun:
+            self.active_display = self.guide_display
+        else:
+            self.active_display = self.conversations_display
+
+    def _utilities_enabled(self, app):
+        try:
+            return app.config["utilities"].as_bool("enable_utilities")
+        except Exception:
+            return False
 
     def active(self):
         return self.active_display
@@ -68,11 +91,23 @@ class MainFrame(urwid.Frame):
     def keypress(self, size, key):
         self.keypress_focus_check()
         
-        if key == "ctrl q":
-            raise urwid.ExitMainLoop
+        #if key == "ctrl q":
+        #    raise urwid.ExitMainLoop
 
         return super(MainFrame, self).keypress(size, key)
 
+class QuitDialogLineBox(urwid.LineBox):
+    def __init__(self, *args, on_cancel=None, **kwargs):
+        self.on_cancel = on_cancel
+        super().__init__(*args, **kwargs)
+
+    def keypress(self, size, key):
+        if key == "esc":
+            if self.on_cancel is not None:
+                self.on_cancel()
+            return None
+        return super().keypress(size, key)
+
 class MainDisplay():
     def __init__(self, ui, app):
         self.ui = ui
@@ -93,6 +128,11 @@ class MainDisplay():
         self.sub_displays.active_display = self.sub_displays.conversations_display
         self.update_active_sub_display()
 
+    def show_channels(self, user_data):
+        self.sub_displays.active_display = self.sub_displays.channels_display
+        self.update_active_sub_display()
+        self.sub_displays.channels_display.start()
+
     def show_directory(self, user_data):
         self.sub_displays.active_display = self.sub_displays.directory_display
         self.update_active_sub_display()
@@ -101,38 +141,206 @@ class MainDisplay():
         self.sub_displays.active_display = self.sub_displays.map_display
         self.update_active_sub_display()
 
+    def show_config(self, user_data):
+        self.sub_displays.active_display = self.sub_displays.config_display
+        self.update_active_sub_display()
+
+    def show_interfaces(self, user_data):
+        self.sub_displays.active_display = self.sub_displays.interface_display
+        self.update_active_sub_display()
+        self.sub_displays.interface_display.start()
+
+    def show_utilities(self, user_data):
+        if self.sub_displays.utilities_display is None:
+            return
+        self.sub_displays.active_display = self.sub_displays.utilities_display
+        self.update_active_sub_display()
+        self.sub_displays.utilities_display.start()
+
+    def show_log(self, user_data):
+        self.sub_displays.active_display = self.sub_displays.log_display
+        self.sub_displays.log_display.show()
+        self.update_active_sub_display()
+
+    def show_guide(self, user_data):
+        self.sub_displays.active_display = self.sub_displays.guide_display
+        self.update_active_sub_display()
+
+    def show_page_editor(self, user_data):
+        if not self.app.enable_node:
+            return
+        self.sub_displays.active_display = self.sub_displays.page_editor_display
+        self.update_active_sub_display()
+        self.sub_displays.page_editor_display.start()
+        self.frame.focus_position = "body"
+
     def update_active_sub_display(self):
         self.frame.contents["body"] = (self.sub_displays.active().widget, None)
         self.update_active_shortcuts()
+        if self.sub_displays.active_display != self.sub_displays.log_display:
+            self.sub_displays.log_display.kill()
 
     def update_active_shortcuts(self):
         self.frame.contents["footer"] = (self.sub_displays.active().shortcuts().widget, None)
 
+    def request_redraw(self, extra_delay=0.0):
+        self.app.ui.loop.set_alarm_in(0.25+extra_delay, self.redraw_now)
+    
+    def redraw_now(self, sender=None, data=None):
+        self.app.ui.loop.screen.clear()
+        #self.app.ui.loop.draw_screen()
+
+    def start(self):
+        self.menu_display.start()
+
     def quit(self, sender=None):
+        if self.app.peer_settings["quit_confirm"]: self.show_quit_dialog()
+        else: self.do_quit()
+
+    def show_quit_dialog(self, sender=None):
+        if getattr(self, "_quit_dialog_open", False): return
+
+        active = self.sub_displays.active_display
+        if active is self.sub_displays.log_display:
+            self.do_quit()
+            return
+        if active is self.sub_displays.config_display and self.sub_displays.config_display.editor_term is not None:
+            self.do_quit()
+            return
+
+        dont_ask_again = urwid.CheckBox("Don't ask again")
+
+        def confirm(button=None):
+            if dont_ask_again.get_state():
+                try:
+                    self.app.peer_settings["quit_confirm"] = False
+                    self.app.save_peer_settings()
+                except Exception: pass
+            self.do_quit()
+
+        def cancel(button=None):
+            self._quit_dialog_open = False
+            self.app.ui.loop.widget = self.frame
+
+        buttons = urwid.Columns([
+            (urwid.WEIGHT, 0.45, urwid.Button("Yes", on_press=confirm)),
+            (urwid.WEIGHT, 0.1, urwid.Text("")),
+            (urwid.WEIGHT, 0.45, urwid.Button("No", on_press=cancel)),
+        ])
+        try: buttons.focus_position = 2  # default to "No"
+        except Exception: pass
+
+        pile = urwid.Pile([
+            urwid.Text("Are you sure you want to quit?", align="center"),
+            urwid.Divider(),
+            buttons,
+            urwid.Divider(),
+            dont_ask_again,
+        ])
+
+        dialog = QuitDialogLineBox(
+            urwid.Filler(pile, urwid.TOP),
+            title="Quit NomadNet",
+            on_cancel=cancel,
+        )
+
+        self._quit_dialog_open = True
+
+        self.app.ui.loop.widget = urwid.Overlay(
+            dialog, self.frame,
+            align="center", width=44,
+            valign="middle", height=7, min_width=20,
+        )
+
+    def show_notice(self, text):
+        def close(button=None): self.app.ui.loop.widget = self.frame
+        pile = urwid.Pile([urwid.Text(text, align="center"), urwid.Divider(), urwid.Button("OK", on_press=close)])
+        dialog = QuitDialogLineBox(urwid.Filler(pile, urwid.TOP), title="Notice", on_cancel=close)
+        self.app.ui.loop.widget = urwid.Overlay(dialog, self.frame, align="center", width=64, valign="middle", height=9, min_width=20)
+
+    def do_quit(self, sender=None):
+        logterm_pid = None
+        if True or RNS.vendor.platformutils.is_android():
+            if self.sub_displays.log_display != None and self.sub_displays.log_display.log_term != None:
+                if self.sub_displays.log_display.log_term.log_term != None:
+                    logterm_pid = self.sub_displays.log_display.log_term.log_term.pid
+                    if logterm_pid != None:
+                        import os, signal
+                        os.kill(logterm_pid, signal.SIGKILL)
+
         raise urwid.ExitMainLoop
 
 
 class MenuColumns(urwid.Columns):
     def keypress(self, size, key):
         if key == "tab" or key == "down":
-            self.handler.frame.set_focus("body")
+            self.handler.frame.focus_position = "body"
 
         return super(MenuColumns, self).keypress(size, key)
 
 class MenuDisplay():
+    UPDATE_INTERVAL = 2
+
     def __init__(self, app, handler):
         self.app = app
+        self.update_interval = MenuDisplay.UPDATE_INTERVAL
+        self.g = self.app.ui.glyphs
 
-        menu_text            = ("pack", urwid.Text(" \U00002638"))
-        button_network       = (11, MenuButton("Network", on_press=handler.show_network))
-        button_conversations = (17, MenuButton("Conversations", on_press=handler.show_conversations))
-        button_directory     = (13, MenuButton("Directory", on_press=handler.show_directory))
-        button_map           = (7, MenuButton("Map", on_press=handler.show_map))
-        button_quit          = (8, MenuButton("Quit", on_press=handler.quit))
+        self.menu_indicator  = urwid.Text("")
+
+        menu_text             = (urwid.PACK, self.menu_indicator)
+        button_network        = (11, MenuButton("Network", on_press=handler.show_network))
+        button_conversations  = (17, MenuButton("Conversations", on_press=handler.show_conversations))
+        button_channels       = (12, MenuButton("Channels", on_press=handler.show_channels))
+        button_directory      = (13, MenuButton("Directory", on_press=handler.show_directory))
+        button_map            = (7,  MenuButton("Map", on_press=handler.show_map))
+        button_log            = (7,  MenuButton("Log", on_press=handler.show_log))
+        button_config         = (10, MenuButton("Config", on_press=handler.show_config))
+        button_interfaces     = (14, MenuButton("Interfaces", on_press=handler.show_interfaces))
+        button_utilities      = (13, MenuButton("Utilities", on_press=handler.show_utilities))
+        button_guide          = (9,  MenuButton("Guide", on_press=handler.show_guide))
+        button_mynode         = (11, MenuButton("My Node", on_press=handler.show_page_editor))
+        button_quit           = (8,  MenuButton("Quit", on_press=handler.quit))
 
         # buttons = [menu_text, button_conversations, button_node, button_directory, button_map]
-        buttons = [menu_text, button_conversations, button_network, button_quit]
+        buttons = [menu_text, button_conversations, button_network]
+        if self.app.enable_node:
+            buttons.append(button_mynode)
+        buttons += [button_channels, button_log, button_interfaces]
+        enable_utilities = False
+        try:
+            enable_utilities = self.app.config["utilities"].as_bool("enable_utilities")
+        except Exception:
+            enable_utilities = False
+        if enable_utilities:
+            buttons.append(button_utilities)
+        buttons.append(button_config)
+        if not self.app.config["textui"]["hide_guide"]:
+            buttons.append(button_guide)
+        buttons.append(button_quit)
+
         columns = MenuColumns(buttons, dividechars=1)
         columns.handler = handler
 
+        self.update_display()
+
         self.widget = urwid.AttrMap(columns, "menubar")
+
+    def start(self):
+        self.update_display_job()
+
+    def update_display_job(self, event = None, sender = None):
+        self.update_display()
+        self.app.ui.loop.set_alarm_in(self.update_interval, self.update_display_job)
+
+    def update_display(self):
+        if self.app.has_unread_conversations():
+            self.indicate_unread()
+        else:
+            self.indicate_normal()
+
+    def indicate_normal(self):
+        self.menu_indicator.set_text(self.g["decoration_menu"])
+
+    def indicate_unread(self):
+        self.menu_indicator.set_text(self.g["unread_menu"])

nomadnet/ui/textui/Map.py

@@ -15,7 +15,7 @@ class MapDisplay():
         ])
 
         self.shortcuts_display = MapDisplayShortcuts(self.app)
-        self.widget = urwid.Filler(pile, 'top')
+        self.widget = urwid.Filler(pile, urwid.TOP)
 
     def shortcuts(self):
         return self.shortcuts_display

nomadnet/ui/textui/ReadlineEdit.py

@@ -0,0 +1,150 @@
+import urwid
+
+
+_KILL_KEYS = frozenset(("ctrl u", "ctrl k", "ctrl w", "ctrl l"))
+
+
+class _KillRing:
+    """Module-global kill buffer shared across all ReadlineMixin widgets.
+
+    Mirrors GNU readline: consecutive kills accumulate into the same entry
+    (forward kills append, backward kills prepend); any non-kill keypress
+    breaks the chain so the next kill replaces the buffer.
+    """
+    text = ""
+    last_was_kill = False
+
+    @classmethod
+    def reset_chain(cls):
+        cls.last_was_kill = False
+
+    @classmethod
+    def kill(cls, killed, direction):
+        if not killed:
+            return
+        if cls.last_was_kill:
+            cls.text = cls.text + killed if direction == "forward" else killed + cls.text
+        else:
+            cls.text = killed
+        cls.last_was_kill = True
+
+
+class ReadlineMixin:
+    """Mixin adding readline-style editing keys to an urwid.Edit-derived widget.
+
+    Bindings (GNU readline defaults, plus ctrl-l as kill-whole-buffer):
+        ctrl-a        beginning-of-line
+        ctrl-e        end-of-line
+        ctrl-u        unix-line-discard      (kill from cursor to beginning of line)
+        ctrl-k        kill-line              (kill from cursor to end of line)
+        ctrl-w        unix-word-rubout       (kill previous whitespace-delimited word)
+        ctrl-l        kill-whole-buffer      (kill the entire edit buffer)
+        ctrl-y        yank                   (insert most-recently-killed text)
+        ctrl-left     backward-word          (alphanumeric boundary)
+        ctrl-right    forward-word           (alphanumeric boundary)
+
+    "Line" is the current logical line within the edit buffer, delimited by
+    newlines -- so on multiline Edits these act on the line under the cursor,
+    not the entire buffer.
+
+    The kill buffer is shared across all widgets using this mixin, so text
+    killed in one Edit can be yanked into another.
+    """
+
+    def keypress(self, size, key):
+        if   key == "ctrl a":     self._rl_beg_of_line()
+        elif key == "ctrl e":     self._rl_end_of_line()
+        elif key == "ctrl u":     self._rl_kill_to_beg()
+        elif key == "ctrl k":     self._rl_kill_to_end()
+        elif key == "ctrl w":     self._rl_kill_word_back()
+        elif key == "ctrl l":     self._rl_kill_whole_buffer()
+        elif key == "ctrl y":     self._rl_yank()
+        elif key == "ctrl left":  self._rl_backward_word()
+        elif key == "ctrl right": self._rl_forward_word()
+        else:
+            result = super().keypress(size, key)
+            if key not in _KILL_KEYS:
+                _KillRing.reset_chain()
+            return result
+        if key not in _KILL_KEYS:
+            _KillRing.reset_chain()
+        return None
+
+    def _rl_line_bounds(self):
+        text, pos = self.edit_text, self.edit_pos
+        bol = text.rfind("\n", 0, pos)
+        bol = 0 if bol == -1 else bol + 1
+        eol = text.find("\n", pos)
+        eol = len(text) if eol == -1 else eol
+        return bol, eol
+
+    def _rl_delete(self, start, end, kill_direction=None):
+        if start == end:
+            return
+        text = self.edit_text
+        if kill_direction is not None:
+            _KillRing.kill(text[start:end], kill_direction)
+        self.set_edit_text(text[:start] + text[end:])
+        self.set_edit_pos(start)
+
+    @staticmethod
+    def _rl_is_word_char(ch):
+        return ch.isalnum() or ch == "_"
+
+    def _rl_beg_of_line(self):
+        bol, _ = self._rl_line_bounds()
+        self.set_edit_pos(bol)
+
+    def _rl_end_of_line(self):
+        _, eol = self._rl_line_bounds()
+        self.set_edit_pos(eol)
+
+    def _rl_kill_to_beg(self):
+        bol, _ = self._rl_line_bounds()
+        self._rl_delete(bol, self.edit_pos, kill_direction="backward")
+
+    def _rl_kill_to_end(self):
+        _, eol = self._rl_line_bounds()
+        self._rl_delete(self.edit_pos, eol, kill_direction="forward")
+
+    def _rl_kill_word_back(self):
+        text, pos = self.edit_text, self.edit_pos
+        p = pos
+        while p > 0 and text[p - 1].isspace():
+            p -= 1
+        while p > 0 and not text[p - 1].isspace():
+            p -= 1
+        self._rl_delete(p, pos, kill_direction="backward")
+
+    def _rl_kill_whole_buffer(self):
+        self._rl_delete(0, len(self.edit_text), kill_direction="forward")
+
+    def _rl_yank(self):
+        if not _KillRing.text:
+            return
+        pos = self.edit_pos
+        text = self.edit_text
+        self.set_edit_text(text[:pos] + _KillRing.text + text[pos:])
+        self.set_edit_pos(pos + len(_KillRing.text))
+
+    def _rl_backward_word(self):
+        text, pos = self.edit_text, self.edit_pos
+        while pos > 0 and not self._rl_is_word_char(text[pos - 1]):
+            pos -= 1
+        while pos > 0 and self._rl_is_word_char(text[pos - 1]):
+            pos -= 1
+        self.set_edit_pos(pos)
+
+    def _rl_forward_word(self):
+        text, pos = self.edit_text, self.edit_pos
+        n = len(text)
+        while pos < n and not self._rl_is_word_char(text[pos]):
+            pos += 1
+        while pos < n and self._rl_is_word_char(text[pos]):
+            pos += 1
+        self.set_edit_pos(pos)
+
+
+class ReadlineEdit(ReadlineMixin, urwid.Edit):
+    """Drop-in urwid.Edit replacement with readline-style editing keys."""
+    pass

nomadnet/ui/textui/Transport.py

@@ -0,0 +1,495 @@
+import RNS
+import time
+import urwid
+
+def _fmt_secs(secs):
+    secs = int(secs)
+    if secs < 0:
+        secs = 0
+    if secs < 60:
+        return "%ds" % secs
+    if secs < 3600:
+        return "%dm" % (secs // 60)
+    if secs < 86400:
+        return "%dh" % (secs // 3600)
+    return "%dd" % (secs // 86400)
+
+def _rel_future(ts):
+    if not ts:
+        return "-"
+    remaining = ts - time.time()
+    if remaining <= 0:
+        return "expired"
+    return _fmt_secs(remaining)
+
+def _rel_past(ts):
+    if not ts:
+        return "-"
+    return _fmt_secs(time.time() - ts) + " ago"
+
+def _fmt_duration(secs):
+    if not secs:
+        return "n/a"
+    secs = int(secs)
+    if secs < 0:
+        secs = 0
+    days = secs // 86400
+    hours = (secs % 86400) // 3600
+    minutes = (secs % 3600) // 60
+    if days:
+        return "%dd %dh" % (days, hours)
+    if hours:
+        return "%dh %dm" % (hours, minutes)
+    if minutes:
+        return "%dm" % minutes
+    return "%ds" % secs
+
+def _fmt_rate(hz):
+    try:
+        hz = float(hz or 0)
+    except Exception:
+        return "0"
+    if hz <= 0:
+        return "0"
+    per_hour = hz * 3600
+    if per_hour >= 1:
+        return "%.0f/h" % per_hour
+    return "%.1f/d" % (hz * 86400)
+
+def _mode_str(mode):
+    return {1: "Full", 2: "Point-to-Point", 3: "Access Point", 4: "Roaming", 5: "Boundary", 6: "Gateway"}.get(mode, "Full")
+
+class TransportRow(urwid.WidgetWrap):
+    def __init__(self, columns):
+        self._selectable = True
+        super().__init__(urwid.AttrMap(columns, "list_normal", focus_map="list_focus"))
+
+    def selectable(self):
+        return True
+
+    def keypress(self, size, key):
+        return key
+
+class LazyTableWalker(urwid.ListWalker):
+    def __init__(self):
+        self.items = []
+        self.factory = None
+        self.empty_widget = None
+        self.focus = 0
+
+    def configure(self, items, factory, empty_widget=None):
+        self.items = items or []
+        self.factory = factory
+        self.empty_widget = empty_widget
+        if self.focus >= len(self.items):
+            self.focus = max(0, len(self.items) - 1)
+        self._modified()
+
+    def _build(self, position):
+        return self.factory(self.items[position], position)
+
+    def get_focus(self):
+        if not self.items:
+            if self.empty_widget is not None:
+                return (self.empty_widget, 0)
+            return (None, None)
+        if self.focus >= len(self.items):
+            self.focus = len(self.items) - 1
+        if self.focus < 0:
+            self.focus = 0
+        return (self._build(self.focus), self.focus)
+
+    def set_focus(self, position):
+        self.focus = position
+        self._modified()
+
+    def get_next(self, position):
+        n = position + 1
+        if not self.items or n >= len(self.items):
+            return (None, None)
+        return (self._build(n), n)
+
+    def get_prev(self, position):
+        p = position - 1
+        if not self.items or p < 0:
+            return (None, None)
+        return (self._build(p), p)
+
+    def positions(self, reverse=False):
+        count = len(self.items)
+        if count == 0:
+            return [0] if self.empty_widget is not None else []
+        if reverse:
+            return range(count - 1, -1, -1)
+        return range(count)
+
+    def __len__(self):
+        return len(self.items)
+
+    def __getitem__(self, position):
+        if not self.items:
+            if position == 0 and self.empty_widget is not None:
+                return self.empty_widget
+            raise IndexError(position)
+        if position < 0 or position >= len(self.items):
+            raise IndexError(position)
+        return self._build(position)
+
+class TransportFiller(urwid.WidgetWrap):
+    def __init__(self, widget, display):
+        self.display = display
+        super().__init__(widget)
+
+    def keypress(self, size, key):
+        if key == "tab":
+            self.display.cycle_tab()
+            return None
+        if key == "ctrl r" or key == "f5":
+            self.display.refresh()
+            return None
+        if key == "esc":
+            self.display.go_back()
+            return None
+        result = super().keypress(size, key)
+        if key == "up" and result == "up":
+            self.display.go_back_to_header()
+            return None
+        return result
+
+class TransportDisplayShortcuts:
+    def __init__(self, app):
+        self.app = app
+        self.widget = urwid.AttrMap(
+            urwid.Text("[Tab] Switch Table  [Up] Filter  [C-r] Refresh  [Esc] Back"),
+            "shortcutbar"
+        )
+
+class TransportDisplay:
+    POLL_INTERVAL = 5
+
+    HASH_W = 34
+    HOPS_W = 6
+    VIA_W = 34
+    EXP_W = 12
+    LAST_W = 16
+    VIOL_W = 11
+    STATUS_W = 10
+    PATHS_W = 8
+    HELD_W = 8
+    QUEUE_W = 8
+    RATE_W = 10
+    MODE_W = 12
+
+    def __init__(self, app, parent=None):
+        self.app = app
+        self.parent = parent
+        self.started = False
+        self.poll_scheduler = False
+        self.glyphset = self.app.config["textui"]["glyphs"]
+        self.g = self.app.ui.glyphs
+
+        self.active_tab = "paths"
+        self.paths = []
+        self.rates = []
+        self.interfaces = []
+        self.paths_per_interface = {}
+        self.link_count = 0
+        self.transport_id = None
+        self.network_id = None
+        self.transport_uptime = None
+        self.held_announces = 0
+        self.filter_text = ""
+
+        self.summary_text = urwid.Text("")
+        self.list_walker = LazyTableWalker()
+        self.list_box = urwid.ListBox(self.list_walker)
+
+        self.filter_edit = urwid.Edit("", "")
+        urwid.connect_signal(self.filter_edit, "change", self._on_filter_change)
+        self.filter_row = urwid.Columns([
+            (9, urwid.Text(("key", "Filter:"))),
+            urwid.AttrMap(self.filter_edit, "list_normal", focus_map="list_focus"),
+        ], dividechars=1)
+
+        self.pile = urwid.Pile([
+            ('pack', self.summary_text),
+            ('pack', self._tab_bar()),
+            ('pack', self.filter_row),
+            ('pack', self._header_row()),
+            ('pack', urwid.Divider("─")),
+            ('weight', 1, self.list_box),
+        ])
+
+        self.transport_view = TransportFiller(self.pile, self)
+        self.shortcuts_display = TransportDisplayShortcuts(self.app)
+        self.widget = self.transport_view
+
+        self._refresh_data()
+        self._rebuild()
+
+    def shortcuts(self):
+        return self.shortcuts_display
+
+    def _tab_bar(self):
+        def tab(label, key):
+            attr = "list_focus" if self.active_tab == key else "list_normal"
+            return (urwid.PACK, urwid.Text((attr, " %s " % label)))
+
+        return urwid.Columns([
+            tab("Paths", "paths"),
+            tab("Announce Rates", "rates"),
+            tab("Interfaces", "interfaces"),
+            (urwid.WEIGHT, 1, urwid.Text("")),
+        ], dividechars=1)
+
+    def _header_row(self):
+        if self.active_tab == "paths":
+            cols = [
+                (self.HASH_W, urwid.Text(("key", "Destination"), wrap="clip")),
+                (self.HOPS_W, urwid.Text(("key", "Hops"), wrap="clip")),
+                (urwid.WEIGHT, 1, urwid.Text(("key", "Interface"), wrap="clip")),
+                (self.VIA_W, urwid.Text(("key", "Via"), wrap="clip")),
+                (self.EXP_W, urwid.Text(("key", "Expires"), wrap="clip")),
+            ]
+        elif self.active_tab == "interfaces":
+            cols = [
+                (urwid.WEIGHT, 1, urwid.Text(("key", "Interface"), wrap="clip")),
+                (self.STATUS_W, urwid.Text(("key", "Status"), wrap="clip")),
+                (self.PATHS_W, urwid.Text(("key", "Paths"), wrap="clip")),
+                (self.HELD_W, urwid.Text(("key", "Held"), wrap="clip")),
+                (self.QUEUE_W, urwid.Text(("key", "Queue"), wrap="clip")),
+                (self.RATE_W, urwid.Text(("key", "Ann in"), wrap="clip")),
+                (self.MODE_W, urwid.Text(("key", "Mode"), wrap="clip")),
+            ]
+        else:
+            cols = [
+                (self.HASH_W, urwid.Text(("key", "Destination"), wrap="clip")),
+                (self.LAST_W, urwid.Text(("key", "Last announce"), wrap="clip")),
+                (self.VIOL_W, urwid.Text(("key", "Violations"), wrap="clip")),
+                (urwid.WEIGHT, 1, urwid.Text(("key", "Blocked"), wrap="clip")),
+            ]
+        return urwid.Columns(cols, dividechars=1)
+
+    def _refresh_data(self):
+        try:
+            self.paths = self.app.rns.get_path_table() or []
+        except Exception:
+            self.paths = []
+        try:
+            self.rates = self.app.rns.get_rate_table() or []
+        except Exception:
+            self.rates = []
+        try:
+            self.link_count = self.app.rns.get_link_count()
+        except Exception:
+            self.link_count = 0
+
+        try:
+            stats = self.app.rns.get_interface_stats() or {}
+        except Exception:
+            stats = {}
+        self.transport_id = stats.get("transport_id")
+        self.network_id = stats.get("network_id")
+        self.transport_uptime = stats.get("transport_uptime")
+        self.interfaces = stats.get("interfaces", []) or []
+        try:
+            self.held_announces = sum(int(i.get("held_announces", 0) or 0) for i in self.interfaces)
+        except Exception:
+            self.held_announces = 0
+
+        self.paths_per_interface = {}
+        for entry in self.paths:
+            iface = entry.get("interface", "")
+            self.paths_per_interface[iface] = self.paths_per_interface.get(iface, 0) + 1
+
+        try:
+            self.interfaces.sort(key=lambda i: self.paths_per_interface.get(i.get("name", ""), 0), reverse=True)
+        except Exception:
+            pass
+
+        try:
+            self.paths.sort(key=lambda e: (e.get("interface", ""), e.get("hops", 0)))
+        except Exception:
+            pass
+        try:
+            self.rates.sort(key=lambda e: e.get("last", 0), reverse=True)
+        except Exception:
+            pass
+
+    def _make_path_row(self, entry, position):
+        dest = RNS.prettyhexrep(entry["hash"]) if entry.get("hash") else "?"
+        via = RNS.prettyhexrep(entry["via"]) if entry.get("via") else "?"
+        hops = entry.get("hops", 0)
+        iface = entry.get("interface", "")
+        expires = _rel_future(entry.get("expires", 0))
+
+        columns = urwid.Columns([
+            (self.HASH_W, urwid.Text(("list_normal", dest), wrap="clip")),
+            (self.HOPS_W, urwid.Text(str(hops), wrap="clip")),
+            (urwid.WEIGHT, 1, urwid.Text(iface, wrap="clip")),
+            (self.VIA_W, urwid.Text(via, wrap="clip")),
+            (self.EXP_W, urwid.Text(expires, wrap="clip")),
+        ], dividechars=1)
+        return TransportRow(columns)
+
+    def _make_rate_row(self, entry, position):
+        now = time.time()
+        dest = RNS.prettyhexrep(entry["hash"]) if entry.get("hash") else "?"
+        last = _rel_past(entry.get("last", 0))
+        violations = str(entry.get("rate_violations", 0))
+        blocked_until = entry.get("blocked_until", 0) or 0
+        if blocked_until > now:
+            blocked = ("disconnected_status", "blocked %s" % _rel_future(blocked_until))
+        else:
+            blocked = ("connected_status", "no")
+
+        columns = urwid.Columns([
+            (self.HASH_W, urwid.Text(("list_normal", dest), wrap="clip")),
+            (self.LAST_W, urwid.Text(last, wrap="clip")),
+            (self.VIOL_W, urwid.Text(violations, wrap="clip")),
+            (urwid.WEIGHT, 1, urwid.Text(blocked, wrap="clip")),
+        ], dividechars=1)
+        return TransportRow(columns)
+
+    def _make_iface_row(self, entry, position):
+        short_name = entry.get("short_name")
+        if short_name == "None": short_name = None
+        name = short_name or entry.get("name") or "?"
+        RNS.log(name)
+        name = name.replace("\x00", "")
+        paths = self.paths_per_interface.get(entry.get("name", ""), 0)
+        held = entry.get("held_announces", 0) or 0
+        queue = entry.get("announce_queue")
+        queue_str = "-" if queue is None else str(queue)
+        rate = _fmt_rate(entry.get("incoming_announce_frequency", 0))
+        mode = _mode_str(entry.get("mode"))
+
+        if entry.get("status"):
+            status = ("connected_status", "up")
+        else:
+            status = ("disconnected_status", "down")
+
+        columns = urwid.Columns([
+            (urwid.WEIGHT, 1, urwid.Text(("list_normal", name), wrap="clip")),
+            (self.STATUS_W, urwid.Text(status, wrap="clip")),
+            (self.PATHS_W, urwid.Text(str(paths), wrap="clip")),
+            (self.HELD_W, urwid.Text(str(held), wrap="clip")),
+            (self.QUEUE_W, urwid.Text(queue_str, wrap="clip")),
+            (self.RATE_W, urwid.Text(rate, wrap="clip")),
+            (self.MODE_W, urwid.Text(mode, wrap="clip")),
+        ], dividechars=1)
+        return TransportRow(columns)
+
+    def _rebuild(self):
+        blocked_now = 0
+        try:
+            now = time.time()
+            blocked_now = sum(1 for r in self.rates if (r.get("blocked_until", 0) or 0) > now)
+        except Exception:
+            pass
+
+        transport_id = RNS.prettyhexrep(self.transport_id) if self.transport_id else "n/a"
+        network_id = RNS.prettyhexrep(self.network_id) if self.network_id else "n/a"
+        uptime = _fmt_duration(self.transport_uptime)
+
+        summary = [
+            ("key", "Transport ID: "), ("value", transport_id),
+            ("key", "   Network ID: "), ("value", network_id),
+            ("key", "   Uptime: "), ("value", uptime), "\n",
+            ("key", "Active links: "), ("value", str(self.link_count)),
+            ("key", "   Known paths: "), ("value", str(len(self.paths))),
+            ("key", "   Held announces: "), ("value", str(self.held_announces)),
+            ("key", "   Announce sources: "), ("value", str(len(self.rates))),
+            ("key", "   Rate-limited: "), ("value", str(blocked_now)),
+        ]
+
+        if self.active_tab == "paths":
+            data = self._filtered(self.paths)
+            factory = self._make_path_row
+            empty_msg = "No paths in the routing table yet."
+        elif self.active_tab == "interfaces":
+            data = self._filtered(self.interfaces)
+            factory = self._make_iface_row
+            empty_msg = "No interface statistics available."
+        else:
+            data = self._filtered(self.rates)
+            factory = self._make_rate_row
+            empty_msg = "No announce rate data available."
+
+        if self.filter_text.strip():
+            summary.append(("inactive_text", "   [filter: %d shown]" % len(data)))
+        self.summary_text.set_text(summary)
+
+        self.pile.contents[1] = (self._tab_bar(), self.pile.options('pack'))
+        self.pile.contents[3] = (self._header_row(), self.pile.options('pack'))
+
+        empty = urwid.Text(("inactive_text", empty_msg), align=urwid.CENTER)
+        self.list_walker.configure(data, factory, empty)
+
+    def cycle_tab(self):
+        order = ["paths", "rates", "interfaces"]
+        i = order.index(self.active_tab) if self.active_tab in order else 0
+        self.active_tab = order[(i + 1) % len(order)]
+        self._rebuild()
+
+    def _on_filter_change(self, widget, new_text):
+        self.filter_text = new_text
+        self._rebuild()
+
+    def _haystack(self, entry):
+        parts = []
+        for key in ("hash", "via"):
+            value = entry.get(key)
+            if value:
+                parts.append(RNS.prettyhexrep(value))
+        for key in ("interface", "name", "short_name"):
+            value = entry.get(key)
+            if value:
+                parts.append(str(value))
+        return " ".join(parts).lower()
+
+    def _filtered(self, items):
+        needle = self.filter_text.strip().lower()
+        if not needle:
+            return items
+        return [entry for entry in items if needle in self._haystack(entry)]
+
+    def go_back(self):
+        if self.parent is not None:
+            self.parent.show_landing()
+
+    def go_back_to_header(self):
+        try:
+            self.app.ui.main_display.frame.focus_position = "header"
+        except Exception:
+            pass
+
+    def _is_visible(self):
+        md = self.app.ui.main_display
+        if self.parent is not None:
+            return md.sub_displays.active_display is self.parent and getattr(self.parent, "active", None) == "transport"
+        return md.sub_displays.active_display is self
+
+    def refresh(self):
+        self._refresh_data()
+        self._rebuild()
+
+    def start(self):
+        self.started = True
+        self._refresh_data()
+        self._rebuild()
+        if not self.poll_scheduler and self.app.ui.loop is not None:
+            self.poll_scheduler = True
+            self.app.ui.loop.set_alarm_in(self.POLL_INTERVAL, self._poll)
+
+    def _poll(self, loop, user_data):
+        if not self.started or not self._is_visible():
+            self.poll_scheduler = False
+            return
+        self._refresh_data()
+        self._rebuild()
+        try:
+            loop.draw_screen()
+        except Exception:
+            pass
+        loop.set_alarm_in(self.POLL_INTERVAL, self._poll)

nomadnet/ui/textui/__init__.py

@@ -1,5 +1,7 @@
 import os
 import glob
 
-modules = glob.glob(os.path.dirname(__file__)+"/*.py")
-__all__ = [ os.path.basename(f)[:-3] for f in modules if not f.endswith('__init__.py')]
+py_modules  = glob.glob(os.path.dirname(__file__)+"/*.py")
+pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
+modules     = py_modules+pyc_modules
+__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))

nomadnet/ui/textui/images/__init__.py

@@ -0,0 +1,18 @@
+# A minimal, dependency-free image widget for urwid, rendering images through
+# the Kitty terminal graphics protocol.
+
+from ._termlib import get_cell_size, is_kitty_supported, reset_caches, diagnostics
+from ._imagedata import ImageData, parse_image_header
+from .widget import ImageWidget, ImageCanvas, UI_MARGIN
+from .screen import ImageScreen
+
+__all__ = ( "ImageWidget",
+            "ImageCanvas",
+            "ImageScreen",
+            "ImageData",
+            "parse_image_header",
+            "is_kitty_supported",
+            "get_cell_size",
+            "diagnostics",
+            "reset_caches",
+            "UI_MARGIN" )

nomadnet/ui/textui/images/_ctlseqs.py

@@ -0,0 +1,78 @@
+# Kitty Terminal Graphics Protocol control sequences
+# 
+# Minimal set of control sequences needed for the Kitty graphics protocol
+# side-channel image rendering used by the urwid image widget.
+
+from base64 import standard_b64encode
+import re
+
+# Basic building blocks
+APC = b"\x1b_" # Application Program Command
+ST = b"\x1b\\" # String Terminator
+CSI = b"\x1b["
+
+# Kitty graphics: transmit-and-display with chunking support.
+#   ESC _ G <control-data> ; <base64 payload> ESC \
+KITTY_TRANSMISSION = b"\x1b_G%s;%s\x1b\\"
+
+# Delete images: d=A (all placements visible on screen), d=i (one
+# placement, data kept), d=I (whole image, data freed).
+KITTY_DELETE_ALL = b"\x1b_Ga=d,d=A\x1b\\"
+KITTY_DELETE_PLACEMENT = b"\x1b_Ga=d,d=i,i=%d,p=%d\x1b\\"
+KITTY_DELETE_IMAGE = b"\x1b_Ga=d,d=I,i=%d\x1b\\"
+
+# Support query (payload is base64 of 4 zero bytes; expects "i=31;OK")
+KITTY_QUERY = b"\x1b_Ga=q,t=d,i=31,f=24,s=1,v=1,C=1,c=1,r=1;AAAA\x1b\\"
+KITTY_QUERY_RESPONSE_RE = re.compile(b"\x1b_Gi=31;(?P<message>[^\x1b]*)\x1b\\\\")
+
+# Cursor: absolute positioning, erase characters, cursor forward
+CURSOR_POSITION = b"\x1b[%d;%dH"
+ERASE_CHARS = b"\x1b[%dX"
+
+# Synchronized output (DEC 2026) - reduces flicker/tearing when redrawing
+BEGIN_SYNCED_UPDATE = b"\x1b[?2026h"
+END_SYNCED_UPDATE = b"\x1b[?2026l"
+
+# Terminal queries: cell size in pixels (XTWINOPS 16) + primary DA
+CELL_SIZE_PX = b"\x1b[16t"
+DA1 = b"\x1b[c"
+# Response: CSI 4 ; <height-px> ; <width-px> t
+CELL_SIZE_RESPONSE_RE = re.compile(b"\x1b\\[4;(?P<h>\\d+);(?P<w>\\d+)t")
+
+# Maximum payload (base64) per transmission chunk, as recommended by the
+# protocol documentation
+CHUNK_SIZE = 4096
+
+# Control data for a transmit-only (a=t) with a quiet mode (q=1)
+def kitty_transmit_control(image_id): return b"a=t,i=%d,f=100,t=d,q=1" % image_id
+
+# A placement command (a=p) for a previously transmitted image.
+# Re-sending the same (image id, placement id) pair replaces the
+# placement in place - used to move/resize/slice without flicker.
+# y/h select the source rectangle (image pixels) when a slice of
+# the image is displayed.
+def kitty_placement(image_id, placement_id, cols, rows, z, y=None, h=None):
+    control = b"a=p,i=%d,p=%d,c=%d,r=%d,z=%d,C=1,q=1" % (image_id, placement_id, cols, rows, z)
+    if y is not None and h is not None: control += b",y=%d,h=%d" % (y, h)
+    return KITTY_TRANSMISSION % (control, b"")
+
+# Encodes a kitty graphics transmission, chunking the base64-encoded
+# payload into ``chunk_size``-byte pieces.
+def kitty_transmission(control, payload, chunk_size=CHUNK_SIZE):
+    encoded = standard_b64encode(payload)
+    chunks = [encoded[i:i + chunk_size] for i in range(0, len(encoded), chunk_size)]
+    if not chunks:
+        chunks = [b""]
+
+    def control_for(index, n_chunks):
+        if n_chunks == 1:
+            return control
+        if index == 0:
+            # The control data appears only in the first chunk; the chunk
+            # marker says more chunks are coming.
+            return control + b",m=1"
+        if index == n_chunks - 1:
+            return b"m=0"
+        return b"m=1"
+
+    return b"".join(KITTY_TRANSMISSION % (control_for(i, len(chunks)), chunk) for i, chunk in enumerate(chunks))

nomadnet/ui/textui/images/_imagedata.py

@@ -0,0 +1,142 @@
+# Image file handling for the side-channel renderer
+#
+# The Kitty graphics protocol can decode and scale images in the terminal
+# itself (data format ``f=100``), which means the widget needs no image
+# decoding library at all. It only needs the *dimensions* of the image to lay
+# it out correctly in the urwid canvas; those are trivially parseable from the
+# file headers of the supported formats.
+# 
+# Supported source formats:
+#  - WebP (VP8, VP8L and VP8X)
+#  - PNG
+# 
+# The payload transmitted to the terminal is always a PNG file: PNG sources
+# are passed through verbatim, and WebP sources are converted transparently
+# by the multi-backend conversion chain in :mod:`._webp` (no required
+# dependencies). The conversion result is cached on disk, so each unique
+# WebP is converted at most once per system.
+
+import hashlib
+import os
+
+MAX_PAYLOAD_BYTES           = 8 * 1024 * 1024
+MAX_CONVERTED_PAYLOAD_BYTES = 16 * 1024 * 1024
+_PNG_SIGNATURE              = b"\x89PNG\r\n\x1a\n"
+
+# Returns (width, height) from a PNG file.
+def _parse_png(data):
+    if not data.startswith(_PNG_SIGNATURE): raise ValueError("not a PNG")
+    if len(data) < 24 or data[12:16] != b"IHDR": raise ValueError("PNG missing IHDR chunk")
+    width = int.from_bytes(data[16:20], "big")
+    height = int.from_bytes(data[20:24], "big")
+    if not (width > 0 and height > 0): raise ValueError("PNG has null dimensions")
+    return width, height
+
+# Returns (width, height) from a WebP file (VP8, VP8L or VP8X).
+def _parse_webp(data):
+    if not data.startswith(b"RIFF") or data[8:12] != b"WEBP": raise ValueError("not a WebP")
+    if len(data) < 30: raise ValueError("WebP file too short")
+    fourcc = data[12:16]
+    if fourcc == b"VP8X":
+        # 10-byte chunk: flags(1) reserved(3) canvas-w-1(3 LE) canvas-h-1(3 LE)
+        width = int.from_bytes(data[24:27], "little") + 1
+        height = int.from_bytes(data[27:30], "little") + 1
+    elif fourcc == b"VP8 ":
+        # frame tag(3) start code 0x9d012a(3) width(2 LE, 14 bits) height(2 LE, 14 bits)
+        width = int.from_bytes(data[26:28], "little") & 0x3FFF
+        height = int.from_bytes(data[28:30], "little") & 0x3FFF
+    elif fourcc == b"VP8L":
+        # signature(1) then 4 bytes: w-1 (14 bits) | h-1 (14 bits) | ...
+        bits = int.from_bytes(data[21:25], "little")
+        width = (bits & 0x3FFF) + 1
+        height = ((bits >> 14) & 0x3FFF) + 1
+    else: raise ValueError("unsupported WebP variant %r" % fourcc)
+    if not (width > 0 and height > 0): raise ValueError("WebP has null dimensions")
+    return width, height
+
+
+def parse_image_header(data):
+    if data[:8] == _PNG_SIGNATURE:
+        w, h = _parse_png(data)
+        return "PNG", w, h
+    if data[:4] == b"RIFF" and data[8:12] == b"WEBP":
+        w, h = _parse_webp(data)
+        return "WebP", w, h
+    raise ValueError("unsupported image format")
+
+# Loads an image file and exposes its PNG bytes and dimensions.
+# The payload is always a PNG file, transmitted verbatim to the terminal
+# (Kitty's f=100 data format). The dimensions are used only for layout.
+# WebP sources are converted transparently to PNG on load.
+class ImageData(object):
+    def __init__(self, path, max_bytes=MAX_PAYLOAD_BYTES, remote_source=False):
+        self.path = path
+        self.error = None
+        self.format = None
+        self.source_format = None
+        self.conversion_backend = None
+        self.width = 0
+        self.height = 0
+        self.data = None
+        try:
+            with open(path, "rb") as f: data = f.read()
+        except OSError as e:
+            self.error = "Could not read file: %s" % e
+            return
+        
+        if len(data) > max_bytes:
+            self.error = "File too large (%.1f MiB > %d MiB)" % (len(data) / (1024 * 1024), max_bytes // (1024 * 1024))
+            return
+        
+        if not data:
+            self.error = "Empty file"
+            return
+        
+        try: self.format, self.width, self.height = parse_image_header(data)
+        except ValueError as e:
+            self.error = str(e)
+            return
+        
+        self.source_format = self.format
+        if self.format == "WebP":
+            from . import _webp
+            png = _webp.convert_webp_to_png(data)
+            if png is None:
+                self.error = "WebP conversion failed (no working backend available)"
+                return
+            if len(png) > MAX_CONVERTED_PAYLOAD_BYTES:
+                self.error = "Converted image too large (%.1f MiB > %d MiB)" % (len(png) / (1024 * 1024), MAX_CONVERTED_PAYLOAD_BYTES // (1024 * 1024))
+                return
+
+            self.data = png
+            self.format = "PNG"
+            self.conversion_backend = _webp.last_backend()
+        
+        else:
+            if remote_source:
+                self.error = "Invalid image format, remote images must be WebP"
+                return
+            else:
+                self.data = data
+        
+        # Content key for de-duplication: identical bytes = same key, so
+        # the same image transmitted once can be placed many times.
+        self._key = hashlib.sha256(self.data).hexdigest()
+
+    @property
+    def ok(self): return self.data is not None
+
+    @property
+    def key(self): return getattr(self, "_key", None)
+
+    @property
+    def size(self): return (self.width, self.height)
+
+    def description(self):
+        if self.ok:
+            if self.source_format == "WebP":
+                return "WebP %dx%d -> PNG (%s)" % (self.width, self.height, self.conversion_backend or "cached")
+            else:
+                return "%s %dx%d" % (self.format, self.width, self.height)
+
+        return self.error

nomadnet/ui/textui/images/_imagestore.py

@@ -0,0 +1,90 @@
+# Registry of terminal-side image data
+# 
+# Tracks which image payloads have been transmitted to the terminal (Kitty
+# image ids), de-duplicates identical image data across widgets (one payload
+# transmitted once, placed many times), and counts live widget references so
+# that orphaned data can be evicted from the terminal's buffers.
+# 
+# This module only tracks *state*; all protocol output is performed by
+# ImageScreen, which reads and updates the store.
+# 
+# Eviction:
+# 
+# - Automatic, widgets unregister when closed/destroyed; the screen evicts
+#   (deletes from the terminal) any registered data with no live references.
+# - Explicit, the browser can call ImageScreen.purge_images() on page
+#   navigation to drop everything at once.
+
+DEFAULT_IMAGE_ID_MAX = 2 ** 32 - 1  # Protocol limit for image ids
+
+class ImageStore(object):
+    def __init__(self):
+        self._entries = {}  # content key -> {"image_id", "data", "refs", "transmitted"}
+        self._next_image_id = 1
+
+    #############
+    # Lifecycle #
+    #############
+
+    # Registers image data and returns (key, is_new).
+    # If identical data is already registered, the existing entry is
+    # reused (de-duplication) and its reference count is incremented.
+    def register(self, key, data):
+        entry = self._entries.get(key)
+        if entry is None:
+            image_id = self._next_image_id
+            self._next_image_id += 1
+            if image_id > DEFAULT_IMAGE_ID_MAX: raise OverflowError("image id space exhausted")
+            entry = { "image_id": image_id, "data": data,
+                      "refs": 0, "transmitted": False }
+            self._entries[key] = entry
+            is_new = True
+        else: is_new = False
+
+        entry["refs"] += 1
+        return key, is_new
+
+    # Decrements the reference count of registered data
+    def unregister(self, key):
+        entry = self._entries.get(key)
+        if entry is None: return
+        entry["refs"] = max(0, entry["refs"] - 1)
+
+    ###########
+    # Queries #
+    ###########
+
+    def entry_for(self, key): return self._entries.get(key)
+
+    def transmitted(self): return [key for key, entry in self._entries.items() if entry["transmitted"]]
+
+    # Returns [(key, image_id), ...] for entries with no live
+    # widget references that are still present in the terminal.
+    def evict_orphans(self):
+        return [ (key, entry["image_id"])
+                 for key, entry in self._entries.items()
+                 if entry["refs"] == 0 and entry["transmitted"] ]
+
+    ############
+    # Mutation #
+    ############
+
+    def mark_transmitted(self, key):
+        entry = self._entries.get(key)
+        if entry is not None:
+            entry["transmitted"] = True
+
+    def mark_all_untransmitted(self):
+        for entry in self._entries.values():
+            entry["transmitted"] = False
+
+    # Drops the entry for evicted data (frees the payload bytes)
+    def evict(self, key): self._entries.pop(key, None)
+
+    # Forgets all state (used by tests)
+    def reset(self):
+        self._entries.clear()
+        self._next_image_id = 1
+
+# Module-level singleton shared by widgets and the screen.
+image_store = ImageStore()

nomadnet/ui/textui/images/_selftest.py

@@ -0,0 +1,718 @@
+#!/usr/bin/env python3
+"""
+.. Images - headless self test
+
+Exercises the widget + screen machinery without a terminal: image header
+parsing, display sizing, shard scanning, APC construction/chunking and the
+transmit/delete state machine at various scroll positions.
+
+Run::
+
+    python3 -m nomadnet.ui.textui.images._selftest [image-file]
+"""
+
+import base64
+import io
+import os
+import shutil
+import struct
+import sys
+import tempfile
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))
+
+import urwid
+
+from nomadnet.ui.textui.images import _ctlseqs
+from nomadnet.ui.textui.images import _imagestore
+from nomadnet.ui.textui.images import _termlib
+from nomadnet.ui.textui.images._imagedata import ImageData, parse_image_header
+from nomadnet.ui.textui.images.screen import ImageScreen
+from nomadnet.ui.textui.images.widget import ImageWidget
+
+DEFAULT_IMAGE = os.path.expanduser("~/Scratchpad/test.png")
+VIEWPORT = (80, 24)
+CELL_SIZE = (1, 2)  # forced fallback for deterministic layout
+
+PASS = 0
+FAIL = 0
+
+
+def check(cond, label):
+    global PASS, FAIL
+    if cond:
+        PASS += 1
+        print("  ok   - %s" % label)
+    else:
+        FAIL += 1
+        print("  FAIL - %s" % label)
+
+
+def build_pile(image_path, wrap_attrmap=False):
+    widgets = []
+    for i in range(12):
+        widgets.append(urwid.Text("[%02d]\nline one\nline two\n" % i))  # 3 rows each
+    img = ImageWidget(image_path)
+    if wrap_attrmap:
+        img = urwid.AttrMap(img, "body")
+    widgets.append(urwid.Divider(" "))
+    widgets.append(img)
+    widgets.append(urwid.Divider(" "))
+    for i in range(12):
+        widgets.append(urwid.Text("[%02d]\nline one\nline two\n" % (13 + i)))
+    return urwid.Pile(widgets)
+
+
+class RecordingScreen(ImageScreen):
+    def __init__(self):
+        super().__init__()
+        self.log = []
+
+    def write(self, data):
+        if isinstance(data, bytes):
+            data = data.decode("ascii")
+        self.log.append(data)
+        return data
+
+    def flush(self):
+        pass
+
+
+def extract_transmit(log):
+    """Returns (image_id, joined_b64_payload) of the a=t transmission, or
+    None if no transmission is present."""
+    image_id = None
+    payloads = []
+    for item in log:
+        if item.startswith("\x1b_Ga=t"):
+            for piece in item.split("\x1b_G")[1:]:
+                body = piece[:-2]  # strip the trailing ST (ESC \)
+                head, sep, payload = body.partition(";")
+                if not sep:
+                    continue
+                if image_id is None:
+                    image_id = int(head.split("i=")[1].split(",")[0])
+                payloads.append(payload)
+    if image_id is None:
+        return None
+    return image_id, "".join(payloads)
+
+
+def extract_placements(log):
+    """Returns the control data of every a=p placement command."""
+    return [
+        item[3:-2]
+        for item in log
+        if item.startswith("\x1b_Ga=p")
+    ]
+
+
+def extract_deletes(log):
+    """Returns the control data of every a=d delete command."""
+    return [
+        item[3:-2]
+        for item in log
+        if item.startswith("\x1b_Ga=d")
+    ]
+
+
+def screen_probe_placement_key(widget, row, col, canvas_cols, trim_top, vis_rows):
+    """Computes an ImageScreen placement key without a screen instance."""
+    return ImageScreen._placement_key(
+        None, widget, row, col, canvas_cols, trim_top, vis_rows)
+
+
+def test_header_parse(image_path):
+    print("== header parsing ==")
+    with open(image_path, "rb") as f:
+        data = f.read()
+    fmt, w, h = parse_image_header(data)
+    check(fmt == "PNG", "detected PNG")
+    check(w == 1280 and h == 960, "dimensions %dx%d" % (w, h))
+    imgdata = ImageData(image_path)
+    check(imgdata.ok, "ImageData loads without error")
+    check(imgdata.size == (w, h), "ImageData size matches")
+
+
+def test_transmission_encoding(image_path):
+    print("== APC encoding/chunking ==")
+    small = b"\x89PNG\r\n\x1a\n" + b"x" * 100
+    single = _ctlseqs.kitty_transmission(b"a=T,f=100,c=80,r=30,z=1", small)
+    check(single.count(b"\x1b_G") == 1, "single-chunk transmission has one APC")
+    check(single.startswith(b"\x1b_Ga=T,f=100,c=80,r=30,z=1;"), "control data correct")
+    check(single.endswith(b"\x1b\\"), "ends with ST")
+    payload = single.split(b";", 1)[1][:-2]
+    check(base64.b64decode(payload) == small, "payload round-trips to original file")
+
+    with open(image_path, "rb") as f:
+        data = f.read()
+    chunks = _ctlseqs.kitty_transmission(b"a=T,f=100", data, chunk_size=16)
+    apcs = chunks.split(b"\x1b\\")[:-1]
+    check(len(apcs) > 1, "multi-chunk transmission splits into %d APCs" % len(apcs))
+    first, last = apcs[0], apcs[-1]
+    heads = [a.split(b";", 1)[0][3:] for a in apcs]
+    check(b",m=1;" in first, "first chunk carries control + m=1")
+    check(last.startswith(b"\x1b_Gm=0;"), "last chunk carries m=0")
+    check(sum(b"a=T" in h for h in heads) == 1, "control data appears only once")
+    check(all(h == b"m=1" or h == b"m=0" for h in heads[1:]), "continuation chunks carry only the chunk marker")
+    joined_b64 = b"".join(a.split(b";", 1)[1] for a in apcs)
+    check(base64.b64decode(joined_b64) == data, "multi-chunk payload reassembles")
+
+
+_TINY_WEBP = base64.b64decode(
+    "UklGRiQAAABXRUJQVlA4IBgAAAAwAQCdASoBAAEAAUAmJaQAA3AA/vuUAAA=")
+
+
+def _webp_fixture():
+    """Returns (webp_bytes, (width, height)): a 4x4 WebP generated with
+    Pillow when available (more representative), else a 1x1 constant."""
+    try:
+        from PIL import Image
+        if _webp_module_available():
+            image = Image.new("RGBA", (4, 4))
+            for y in range(4):
+                for x in range(4):
+                    image.putpixel((x, y), (x * 60, y * 60, x * y * 10, 255))
+            buf = io.BytesIO()
+            image.save(buf, format="WEBP")
+            data = buf.getvalue()
+            if parse_image_header(data)[0] == "WebP":
+                return data, (4, 4)
+    except Exception:
+        pass
+    return _TINY_WEBP, (1, 1)
+
+
+def _webp_module_available():
+    try:
+        from PIL import features
+        return bool(features.check("webp"))
+    except Exception:
+        return False
+
+
+def test_webp_conversion():
+    print("== webp conversion ==")
+    from nomadnet.ui.textui.images import _webp
+    from nomadnet.ui.textui.images._imagedata import MAX_CONVERTED_PAYLOAD_BYTES
+
+    fmt, w, h = parse_image_header(_TINY_WEBP)
+    check((fmt, w, h) == ("WebP", 1, 1), "embedded fixture is a 1x1 WebP")
+
+    fixture, dims = _webp_fixture()
+    check(parse_image_header(fixture)[1:] == dims,
+          "webp fixture has expected dimensions")
+
+    cache_tmp = tempfile.mkdtemp(prefix="imgw-test-cache-")
+    prev_cache_env = os.environ.get("NOMADNET_IMAGE_CACHE")
+    os.environ["NOMADNET_IMAGE_CACHE"] = cache_tmp
+    try:
+        # Each available backend must convert the fixture to a valid PNG
+        # with matching dimensions. Backends are invoked directly, since
+        # the on-disk cache would otherwise mask all but the first one.
+        _webp.reset()
+        present = [name for name, avail in _webp.available_backends() if avail]
+        converted = 0
+        for name in present:
+            png = _webp._BACKENDS[name](fixture, dims)
+            ok = (png is not None and png.startswith(b"\x89PNG")
+                  and _webp._png_dims(png) == dims)
+            check(ok, "%s backend converts to a valid PNG" % name)
+            if ok:
+                converted += 1
+        check(converted >= 1, "at least one backend converts webp (got %d)" % converted)
+
+        # Repeat conversion is served from the on-disk cache: the backend
+        # table is disabled and the conversion still succeeds.
+        _webp.reset()
+        png1 = _webp.convert_webp_to_png(fixture)
+        check(png1 is not None, "first conversion succeeds")
+        orig_backends = dict(_webp._BACKENDS)
+        _webp._BACKENDS = {name: (lambda *a, **k: None) for name in _webp.BACKENDS}
+        png2 = _webp.convert_webp_to_png(fixture)
+        check(png1 == png2 and _webp.stats()["cache_hits"] >= 1,
+              "repeat conversion served from the on-disk cache")
+        _webp._BACKENDS = orig_backends
+
+        # ImageData-level transparency: a webp file loads as a PNG.
+        _webp.reset()
+        fd, webp_path = tempfile.mkstemp(suffix=".webp")
+        with os.fdopen(fd, "wb") as f:
+            f.write(fixture)
+        d = ImageData(webp_path)
+        check(d.ok and d.format == "PNG" and d.size == dims,
+              "ImageData converts webp transparently to PNG")
+        check(d.data.startswith(b"\x89PNG\r\n\x1a\n") and d.key is not None,
+              "converted payload is a keyed PNG")
+        check("-> PNG" in d.description(), "description notes the conversion")
+        os.unlink(webp_path)
+
+        # Converted-payload cap.
+        _webp.reset()
+        orig_convert = _webp.convert_webp_to_png
+        _webp.convert_webp_to_png = lambda data: b"x" * (MAX_CONVERTED_PAYLOAD_BYTES + 1)
+        fd, webp_path = tempfile.mkstemp(suffix=".webp")
+        with os.fdopen(fd, "wb") as f:
+            f.write(fixture)
+        d = ImageData(webp_path)
+        check(not d.ok and "Converted image too large" in d.error,
+              "converted payload capped at %d MiB"
+              % (MAX_CONVERTED_PAYLOAD_BYTES // (1024 * 1024)))
+        os.unlink(webp_path)
+        _webp.convert_webp_to_png = orig_convert
+
+        # Undecodable webp (valid header/dimensions, garbage bitstream).
+        _webp.reset()
+        junk = (b"RIFF" + struct.pack("<I", 30) + b"WEBP" + b"VP8 "
+                + b"\x00" * 10 + b"\x10\x00\x10\x00" + b"\xff" * 12)
+        check(parse_image_header(junk)[1:] == (16, 16),
+              "junk fixture parses as a 16x16 WebP")
+        fd, webp_path = tempfile.mkstemp(suffix=".webp")
+        with os.fdopen(fd, "wb") as f:
+            f.write(junk)
+        d = ImageData(webp_path)
+        check(not d.ok and "conversion failed" in d.error,
+              "undecodable webp results in the error state")
+        os.unlink(webp_path)
+    finally:
+        if prev_cache_env is not None:
+            os.environ["NOMADNET_IMAGE_CACHE"] = prev_cache_env
+        else:
+            os.environ.pop("NOMADNET_IMAGE_CACHE", None)
+        shutil.rmtree(cache_tmp, ignore_errors=True)
+        _webp.reset()
+
+
+def test_winsize_parsing():
+    print("== TIOCGWINSZ parsing ==")
+    from nomadnet.ui.textui.images import _termlib
+    # struct order is (rows, cols, xpixel, ypixel)
+    check(_termlib._cell_size_from_winsize([40, 100, 1000, 800]) == (10, 20),
+          "correct winsize -> (10, 20)")
+    check(_termlib._cell_size_from_winsize([24, 80, 960, 480]) == (12, 20),
+          "correct winsize -> (12, 20)")
+    check(_termlib._cell_size_from_winsize([24, 80, 0, 0]) is None,
+          "zero pixel fields -> None")
+    check(_termlib._cell_size_from_winsize([40, 100, 800, 1000]) == (8, 25),
+          "portrait window cell keeps valid aspect")
+    check(_termlib._cell_size_from_winsize([30, 80, 1200, 240]) is None,
+          "implausible aspect (broken reporting) -> None")
+    check(_termlib._cell_size_from_winsize([0, 0, 0, 0]) is None,
+          "empty winsize -> None")
+
+
+def test_widget_layout(image_path):
+    print("== widget layout ==")
+    _termlib.kitty_supported = True
+    _termlib.cell_size = CELL_SIZE
+    # Force a deterministic terminal size for the height-percent cases.
+    _orig_terminal_size = _termlib.get_terminal_size
+    _termlib.get_terminal_size = lambda: (80, 24)
+    _orig_ui_margin = None
+    from nomadnet.ui.textui.images import widget as _widget_mod
+    _orig_ui_margin = _widget_mod.UI_MARGIN
+    _widget_mod.UI_MARGIN = 8
+    try:
+        imgdata = ImageData(image_path)
+        w = ImageWidget(image_path)
+        iw, ih = imgdata.size
+        expected_rows = max(1, round(80 * CELL_SIZE[0] * ih / (CELL_SIZE[1] * iw)))
+        rows = w.rows((80,))
+        check(rows == expected_rows,
+              "default: rows((80,)) == %d, got %d" % (expected_rows, rows))
+        canvas = w.render((80,))
+        check(isinstance(canvas, urwid.SolidCanvas), "render returns a solid canvas")
+        check(canvas.rows() == rows and canvas.cols() == 80, "canvas sized 80x%d" % rows)
+        check(w._ti_rows == rows and w._ti_cols == 80, "widget stores display size")
+        w.close()
+
+        # width as absolute columns (aspect preserved); the canvas fills the
+        # allocated width while the image rect is the resolved size.
+        w = ImageWidget(image_path, width=40)
+        c, r = w._display_size(80)
+        check((c, r) == (40, 15), "width=40 -> (40, 15), got (%d, %d)" % (c, r))
+        canvas = w.render((80,))
+        check((canvas.cols(), canvas.rows()) == (80, 15),
+              "width=40 canvas fills the row (80, 15), got (%d, %d)"
+              % (canvas.cols(), canvas.rows()))
+        check((w._ti_cols, w._ti_rows) == (40, 15),
+              "width=40 image rect stored (40, 15)")
+        w.close()
+
+        # width as percent of the layout width
+        w = ImageWidget(image_path, width="50%")
+        c, r = w._display_size(80)
+        check((c, r) == (40, 15), "width=50%% -> (40, 15), got (%d, %d)" % (c, r))
+        w.close()
+
+        # height as absolute rows (width derived, aspect preserved)
+        w = ImageWidget(image_path, height=20)
+        c, r = w._display_size(80)
+        check(r == 20 and c == 53, "height=20 -> (53, 20), got (%d, %d)" % (c, r))
+        w.close()
+
+        # height as percent of the visible page height
+        # page = terminal lines (24) - UI_MARGIN (8) = 16; 50% -> 8 rows
+        w = ImageWidget(image_path, height="50%")
+        c, r = w._display_size(80)
+        check(r == 8 and c == 21, "height=50%% -> (21, 8), got (%d, %d)" % (c, r))
+        w.close()
+
+        # both given: stretch to the exact rectangle
+        w = ImageWidget(image_path, width=40, height=30)
+        c, r = w._display_size(80)
+        check((c, r) == (40, 30), "width=40,height=30 stretches -> (40, 30), got (%d, %d)" % (c, r))
+        w.close()
+
+        # stretching is clamped uniformly when overflowing the space
+        w = ImageWidget(image_path, width=400, height=200)
+        c, r = w._display_size(80)
+        check((c, r) == (80, 40), "overflowing stretch clamped -> (80, 40), got (%d, %d)" % (c, r))
+        w.close()
+
+        # BOX layout: the canvas fills the box; the image rect is clamped
+        # and re-derived (width from the clamped height).
+        w = ImageWidget(image_path, name="box")
+        canvas = w.render((80, 10))
+        check((canvas.cols(), canvas.rows()) == (80, 10),
+              "BOX render((80,10)) canvas fills the box, got (%d, %d)"
+              % (canvas.cols(), canvas.rows()))
+        check((w._ti_cols, w._ti_rows) == (27, 10),
+              "BOX image rect clamped -> (27, 10), got (%d, %d)"
+              % (w._ti_cols, w._ti_rows))
+        w.close()
+
+        # invalid specifications
+        def raises(ctor, want):
+            try:
+                ctor()
+            except want:
+                return True
+            except Exception:
+                return False
+            return False
+        check(raises(lambda: ImageWidget(image_path, width="xx"), ValueError),
+              "width='xx' raises ValueError")
+        check(raises(lambda: ImageWidget(image_path, height=0), ValueError),
+              "height=0 raises ValueError")
+        check(raises(lambda: ImageWidget(image_path, width=1.5), TypeError),
+              "width=1.5 raises TypeError")
+        check(raises(lambda: ImageWidget(image_path, height="50"), ValueError),
+              "height='50' raises ValueError")
+
+        # settable properties re-resolve and invalidate
+        w = ImageWidget(image_path)
+        w.width = "50%"
+        c, r = w._display_size(80)
+        check((c, r) == (40, 15), "set width=50%% property -> (40, 15)")
+        w.height = 10
+        c, r = w._display_size(80)
+        check((c, r) == (40, 10), "width+height set -> stretch (40, 10), got (%d, %d)" % (c, r))
+        w.width = None
+        w.height = 10
+        c, r = w._display_size(80)
+        check((c, r) == (27, 10), "height-only after reset -> (27, 10), got (%d, %d)" % (c, r))
+        w.close()
+
+        # native resolution specification ("n"): display at the image's own
+        # pixel size, quantized to the terminal cell grid. Cell = (1, 2),
+        # small fixture 64x32 -> native 64 cols, 16 rows.
+        from nomadnet.ui.textui.images import _webp
+        small_png = _webp._encode_png_rgba(bytes(64 * 32 * 4), 64, 32)
+        fd, small_path = tempfile.mkstemp(suffix=".png")
+        with os.fdopen(fd, "wb") as f:
+            f.write(small_png)
+        try:
+            wn = ImageWidget(small_path, width="n")
+            wn.render((80,))  # sets _ti_cols/_ti_rows
+            check((wn._ti_cols, wn._ti_rows) == (64, 16),
+                  "width='n' displays native size (64, 16), got (%d, %d)"
+                  % (wn._ti_cols, wn._ti_rows))
+            wn.close()
+            hn = ImageWidget(small_path, height="n")
+            hn.render((80,))
+            check((hn._ti_cols, hn._ti_rows) == (64, 16),
+                  "height='n' displays native size (64, 16), got (%d, %d)"
+                  % (hn._ti_cols, hn._ti_rows))
+            hn.close()
+            bn = ImageWidget(small_path, width="n", height="n")
+            bn.render((80,))
+            check((bn._ti_cols, bn._ti_rows) == (64, 16),
+                  "width='n' height='n' displays native size (64, 16), got (%d, %d)"
+                  % (bn._ti_cols, bn._ti_rows))
+            bn.close()
+            big = ImageWidget(image_path, width="n")  # 1280x960, overflows
+            c, r = big._display_size(80)
+            check((c, r) == (80, 30),
+                  "native of a large image clamps to available width (80, 30), got (%d, %d)" % (c, r))
+            big.close()
+            check(raises(lambda: ImageWidget(image_path, width="x"), ValueError),
+                  "width='x' raises ValueError")
+        finally:
+            os.unlink(small_path)
+
+        # alignment: placement column of a narrowed image inside its canvas
+        w = ImageWidget(image_path, width=40)  # default center
+        w.render((80,))  # sets _ti_cols/_ti_rows
+        check(w._ti_align == "|", "default alignment is center")
+        key = screen_probe_placement_key(w, 3, 0, 80, 0, 15)
+        check(key == (3, 20, 40, 15, None, None),
+              "centered placement col -> %s" % (key,))
+        w.align = "left"
+        key = screen_probe_placement_key(w, 3, 0, 80, 0, 15)
+        check(key == (3, 0, 40, 15, None, None), "left placement col -> 0")
+        w.align = "right"
+        key = screen_probe_placement_key(w, 3, 0, 80, 0, 15)
+        check(key == (3, 40, 40, 15, None, None), "right placement col -> 40")
+        check(raises(lambda: ImageWidget(image_path, align="top"), ValueError),
+              "align='top' raises ValueError")
+        w.close()
+    finally:
+        _termlib.get_terminal_size = _orig_terminal_size
+        if _orig_ui_margin is not None:
+            _widget_mod.UI_MARGIN = _orig_ui_margin
+
+
+def find_image_widget(pile):
+    for widget, _opts in pile.contents:
+        child = widget
+        if isinstance(child, urwid.AttrMap):
+            child = child.original_widget
+        if isinstance(child, ImageWidget):
+            return child
+    return None
+
+
+def run_screen_test(image_path, wrap_attrmap, label):
+    print("== screen state machine (%s) ==" % label)
+    _termlib.kitty_supported = True
+    _termlib.cell_size = CELL_SIZE
+    _imagestore.image_store.reset()
+
+    pile = build_pile(image_path, wrap_attrmap=wrap_attrmap)
+    scrollable = urwid.Scrollable(pile, force_forward_keypress=True)
+    screen = RecordingScreen()
+
+    img_widget = find_image_widget(pile)
+    check(img_widget is not None, "image widget found in pile")
+    z = img_widget._ti_z
+    entry = _imagestore.image_store.entry_for(img_widget._ti_key)
+    check(entry is not None and entry["refs"] == 1, "widget registered with image store")
+    image_id = entry["image_id"]
+    b64 = base64.b64encode(img_widget._ti_data.data).decode("ascii")
+
+    def entries_at(offset):
+        scrollable.set_scrollpos(offset)
+        canvas = scrollable.render(VIEWPORT)
+        return screen._ti_scan(canvas)
+
+    # Build a timeline of image visibility across all scroll offsets.
+    # state = (trim_top, row, visible_rows)
+    timeline = []
+    max_offset = 200
+    for off in range(max_offset + 1):
+        entries = entries_at(off)
+        if not entries:
+            timeline.append((off, None))
+            continue
+        widget, row, col, canvas_cols, trim_top, vis_rows = entries[0]
+        timeline.append((off, (trim_top, row, vis_rows)))
+    image_rows = img_widget._ti_rows
+    visible = [t for t in timeline if t[1] is not None]
+    check(len(visible) > 0, "image is visible at some scroll offset")
+    if not visible:
+        return
+
+    def first_offset(pred):
+        for off, state in timeline:
+            if state is not None and pred(state):
+                return off
+        return None
+
+    off_enter = first_offset(lambda s: s[0] == 0)       # trim_top == 0
+    off_clip = first_offset(lambda s: s[0] > 0)         # top-clipped
+    check(off_enter is not None, "found offset with image fully entered")
+    check(off_clip is not None, "found offset with image top-clipped")
+    if off_enter is None or off_clip is None:
+        return
+    # State at the entry offset: (trim_top, row, visible_rows)
+    off_enter_vis_rows = timeline[off_enter][1][2]
+
+    last_visible_off = visible[-1][0]
+    off_away = last_visible_off + 1 if last_visible_off < max_offset else None
+    check(off_away is not None, "found offset past the image")
+
+    def update(offset):
+        screen.log = []
+        scrollable.set_scrollpos(offset)
+        canvas = scrollable.render(VIEWPORT)
+        screen._ti_update(canvas)
+        return list(screen.log)
+
+    # 1. before the image is visible: nothing
+    log = update(0)
+    check(not any("a=t" in l or "a=p" in l for l in log),
+          "nothing written while out of view")
+    check(not screen._ti_images, "nothing tracked while out of view")
+
+    # 2. image entered (bottom-clipped): transmit once, place the visible
+    #    slice (y=0 crop) at the bottom.
+    log = update(off_enter)
+    trans = extract_transmit(log)
+    placements = extract_placements(log)
+    check(trans is not None, "image data transmitted on entry")
+    if trans:
+        trans_id, pb64 = trans
+        check(trans_id == image_id, "transmit uses the store's image id")
+        check(pb64 == b64, "transmitted payload is the original file")
+    check(len(placements) == 1, "one placement on entry, got %d" % len(placements))
+    if placements:
+        ctrl = placements[0]
+        check("i=%d" % image_id in ctrl and "p=%d" % z in ctrl,
+              "placement references image id and placement id")
+        check("c=80" in ctrl and "r=%d" % off_enter_vis_rows in ctrl,
+              "placement has display size")
+        check("y=0," in ctrl and "h=" in ctrl, "entry slice uses a y=0 crop")
+        check("z=%d" % z in ctrl and "C=1" in ctrl and "q=1" in ctrl,
+              "placement has z-index, C=1 and quiet mode")
+    check(len(screen._ti_images) == 1 and
+          screen._ti_images[id(img_widget)]["drawn"] is not None,
+          "image tracked as drawn")
+    check(entry["transmitted"], "store entry marked transmitted")
+
+    # 3. move while visible: in-place placement update, no re-transmit,
+    #    no delete.
+    log = update(off_enter + 1)
+    check(extract_transmit(log) is None, "position change does not re-transmit")
+    check(len(extract_placements(log)) == 1, "position change re-places in place")
+    check(not extract_deletes(log), "position change does not delete")
+
+    # 4. top-clipped: placement with a y>0 crop.
+    log = update(off_clip)
+    placements = extract_placements(log)
+    check(len(placements) == 1, "top-clip places a slice, got %d" % len(placements))
+    if placements:
+        ctrl = placements[0]
+        y_px = int(ctrl.split("y=")[1].split(",")[0].split(";")[0])
+        check(",y=" in ctrl and "h=" in ctrl and y_px > 0,
+              "top-clipped placement has y>0 crop (y=%d)" % y_px)
+
+    # 5. scrolled past: placement deleted, image data kept.
+    log = update(off_away)
+    deletes = extract_deletes(log)
+    check(deletes == ["a=d,d=i,i=%d,p=%d" % (image_id, z)],
+          "scroll-away deletes the placement only: %s" % deletes)
+    check(extract_transmit(log) is None, "scroll-away does not transmit")
+    check(id(img_widget) not in screen._ti_images, "widget untracked after scroll-away")
+
+    # 6. scroll back: re-placed from the kept data, no re-transmission.
+    log = update(off_enter)
+    check(extract_transmit(log) is None, "scroll-back does not re-transmit")
+    check(len(extract_placements(log)) == 1, "scroll-back re-places the image")
+
+    # 7. still visible, unchanged position: nothing written.
+    log = update(off_enter)
+    check(not extract_placements(log) and not extract_deletes(log),
+          "unchanged position writes nothing")
+
+    # 8. widget closed -> orphaned data evicted from the terminal.
+    img_widget.close()
+    log = update(off_away)
+    deletes = extract_deletes(log)
+    check("a=d,d=I,i=%d" % image_id in deletes,
+          "orphaned image data evicted with d=I")
+    check(_imagestore.image_store.entry_for(img_widget._ti_key) is None,
+          "evicted entry dropped from the store")
+
+
+def test_eviction_and_purge(image_path):
+    print("== eviction and purge ==")
+    _termlib.kitty_supported = True
+    _termlib.cell_size = CELL_SIZE
+    _imagestore.image_store.reset()
+
+    # Shared source: two widgets with the same file share one store entry.
+    w1 = ImageWidget(image_path)
+    w2 = ImageWidget(image_path)
+    entry = _imagestore.image_store.entry_for(w1._ti_key)
+    check(entry["refs"] == 2, "identical sources share one registry entry (refs=%d)"
+          % entry["refs"])
+    check(w2._ti_key == w1._ti_key, "identical sources have the same content key")
+
+    w1.close()
+    check(entry["refs"] == 1, "closing one widget decrements refcount")
+    check(entry["transmitted"] is False, "no data transmitted yet")
+
+    # Place w2 via a screen update and verify orphan eviction after close.
+    pile = urwid.Pile([urwid.Text("x"), urwid.Divider(" "), w2,
+                       urwid.Divider(" "), urwid.Text("y")])
+    scrollable = urwid.Scrollable(pile, force_forward_keypress=True)
+    screen = RecordingScreen()
+    image_id = entry["image_id"]
+
+    def update(offset):
+        screen.log = []
+        scrollable.set_scrollpos(offset)
+        screen._ti_update(scrollable.render(VIEWPORT))
+        return list(screen.log)
+
+    log = update(2)
+    check(extract_transmit(log) is not None, "image transmitted when widget visible")
+    check(entry["transmitted"], "entry marked transmitted")
+
+    w2.close()
+    log = update(0)
+    deletes = extract_deletes(log)
+    check("a=d,d=I,i=%d" % image_id in deletes, "last reference closed -> d=I eviction")
+    check(_imagestore.image_store.entry_for(w2._ti_key) is None,
+          "entry dropped after eviction")
+
+    # Purge: after transmitting, purge_images() deletes everything and the
+    # next redraw re-transmits for still-visible widgets.
+    _imagestore.image_store.reset()
+    w3 = ImageWidget(image_path)
+    pile = urwid.Pile([urwid.Text("x"), urwid.Divider(" "), w3,
+                       urwid.Divider(" "), urwid.Text("y")])
+    scrollable = urwid.Scrollable(pile, force_forward_keypress=True)
+    screen2 = RecordingScreen()
+
+    def update2(offset):
+        screen2.log = []
+        scrollable.set_scrollpos(offset)
+        screen2._ti_update(scrollable.render(VIEWPORT))
+        return list(screen2.log)
+
+    log = update2(2)
+    check(extract_transmit(log) is not None, "purge-test: image transmitted")
+    screen2.log = []
+    screen2.purge_images()
+    check(any("a=d,d=A" in l for l in screen2.log), "purge deletes all (d=A)")
+    check(_imagestore.image_store.transmitted() == [], "purge marks all untransmitted")
+
+    log = update2(2)
+    check(extract_transmit(log) is not None, "visible image re-transmitted after purge")
+    check(len(extract_placements(log)) == 1, "visible image re-placed after purge")
+    w3.close()
+
+
+def main():
+    image_path = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_IMAGE
+    print("Using image: %s" % image_path)
+    _termlib.reset_caches()
+    print("ambient cell size (env): %s" % (_termlib._query_cell_size(),))
+
+    test_header_parse(image_path)
+    test_transmission_encoding(image_path)
+    test_webp_conversion()
+    test_winsize_parsing()
+    test_widget_layout(image_path)
+    run_screen_test(image_path, wrap_attrmap=False, label="bare widget")
+    run_screen_test(image_path, wrap_attrmap=True, label="attrmap-wrapped")
+    test_eviction_and_purge(image_path)
+
+    print("\n%d passed, %d failed" % (PASS, FAIL))
+    return 1 if FAIL else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

nomadnet/ui/textui/images/_termlib.py

@@ -0,0 +1,199 @@
+# Terminal queries for capability detection and layout metrics.
+# 
+# All values are resolved once and cached; subsequent calls are instant.
+# Queries should be performed *before* starting the urwid main loop, because
+# they read directly from the terminal and would otherwise race with urwid's
+# own input handling. If no terminal is available, or the queries time out,
+# sensible fallbacks are used.
+
+import os
+import select
+import shutil
+import sys
+import time
+
+try: import termios
+except: termios = None
+
+from . import _ctlseqs
+
+QUERY_TIMEOUT = 0.3
+
+kitty_supported = None
+cell_size = None
+
+# Last raw terminal query response (bytes), for diagnostics.
+last_query_response = bytearray()
+last_kitty_response = bytearray()
+
+# Last TIOCGWINSZ buffer (rows, cols, xpixel, ypixel), for diagnostics.
+winsize = [0, 0, 0, 0]
+
+# Number of times a query attempt has been made; used to avoid repeated
+# slow queries in a broken environment.
+_query_failed = False
+
+def _open_tty():
+    # Opens the active terminal for reading and writing.
+    for stream in ("out", "in", "err"):
+        try: return os.open(os.ttyname(getattr(sys, "__std%s__" % stream).fileno()), os.O_RDWR)
+        except (OSError, AttributeError): continue
+    try: return os.open("/dev/tty", os.O_RDWR | os.O_NOCTTY)
+    except OSError: return -1
+
+
+def query_tty(request, timeout=QUERY_TIMEOUT):
+    # Sends a query to the terminal and returns the raw response.
+    global _query_failed
+    if _query_failed or not termios: return b""
+
+    fd = _open_tty()
+    if fd == -1:
+        _query_failed = True
+        return b""
+
+    # Never block indefinitely, even on a broken pseudo-terminal,
+    # select has a deadline and reads are non-blocking.
+    try: os.set_blocking(fd, False)
+    except OSError:  pass
+
+    old_attr = termios.tcgetattr(fd)
+    new_attr = termios.tcgetattr(fd)
+    # Raw-ish mode; disable canonical mode and echo. In canonical mode the
+    # terminal buffers input until a newline, which terminal query responses
+    # (for example. kitty's "i=31;OK") never contain. The read would block until
+    # timeout and the query would always "fail".
+    new_attr[3] &= ~(termios.ECHO | termios.ICANON)
+    new_attr[6][termios.VMIN] = 0
+    new_attr[6][termios.VTIME] = 0
+    try:
+        # TCSANOW (not TCSAFLUSH): TCSAFLUSH drains the output queue first,
+        # which can block indefinitely on some pseudo-terminals.
+        termios.tcsetattr(fd, termios.TCSANOW, new_attr)
+        try: os.write(fd, request)
+        except (BlockingIOError, OSError): return b""
+        response = bytearray()
+        deadline = time.monotonic() + timeout
+        while True:
+            remaining = deadline - time.monotonic()
+            if remaining <= 0: break
+            try: ready, _, _ = select.select([fd], [], [], remaining)
+            except (OSError, ValueError): break
+            if not ready: break
+            try: chunk = os.read(fd, 4096)
+            except (BlockingIOError, OSError): continue
+            if not chunk: break
+            response += chunk
+            if response.endswith(b"\x1b\\") or response.endswith(b"c"): break
+        last_query_response[:] = response
+        return bytes(response)
+    
+    finally:
+        try: termios.tcsetattr(fd, termios.TCSANOW, old_attr)
+        except termios.error: pass
+        finally: os.close(fd)
+
+def is_kitty_supported():
+    global kitty_supported
+    if kitty_supported is None:
+        response = query_tty(_ctlseqs.KITTY_QUERY)
+        last_kitty_response[:] = response
+        kitty_supported = bool(response and _ctlseqs.KITTY_QUERY_RESPONSE_RE.search(response))
+    return kitty_supported
+
+def force_kitty_supported(status):
+    # Forces the support status (used by tests).
+    global kitty_supported
+    kitty_supported = bool(status)
+
+def _cell_size_from_winsize(buf):
+    # Interprets a TIOCGWINSZ buffer as (cell_width_px, cell_height_px).
+    # The struct layout is (rows, cols, xpixel, ypixel). Returns None if the
+    # terminal does not report pixel dimensions, or if the result is
+    # implausible (e.g. several times wider than tall - a sign of broken
+    # winsize reporting).
+    rows, cols, xpix, ypix = buf
+    if not (cols and rows and xpix and ypix): return None
+    cell_w, cell_h = max(1, xpix // cols), max(1, ypix // rows)
+    if not (0.1 <= cell_w / cell_h <= 1.5):
+        # Real terminal cells should be as wide as, or narrower than,
+        # they are tall; anything far outside that range is bogus.
+        return None
+    return (cell_w, cell_h)
+
+def _query_cell_size():
+    # Determines the current cell size in pixels, or None if unknown.
+    if not termios: return None
+    fd = _open_tty()
+    if fd != -1:
+        # First, try the TIOCGWINSZ pixel fields (fast, no query)
+        import array
+        import fcntl
+        try:
+            buf = array.array("H", [0, 0, 0, 0])
+            fcntl.ioctl(fd, termios.TIOCGWINSZ, buf)
+            winsize[:] = buf
+            cell = _cell_size_from_winsize(buf)
+            if cell is not None: return cell
+        
+        except OSError: pass
+        finally: os.close(fd)
+
+    # Fall back to XTWINOPS 16 (cell size in pixels); DA1 is appended to
+    # speed up the response (terminals treat queries as FIFO).
+    response = query_tty(_ctlseqs.CELL_SIZE_PX + _ctlseqs.DA1)
+    if response:
+        match = _ctlseqs.CELL_SIZE_RESPONSE_RE.search(response)
+        if match: return (int(match.group("w")), int(match.group("h")))
+    return None
+
+def get_cell_size():
+    # Returns the terminal cell size in pixels as (width, height), or None
+    # if it could not be determined.
+    global cell_size
+    if cell_size is None: cell_size = _query_cell_size()
+    return cell_size
+
+
+def get_terminal_size():
+    # Returns the active terminal size as (columns, lines).
+    # Uses the tty directly when available (correct even with redirected
+    # output), falling back to shutil.get_terminal_size.
+    
+    fd = _open_tty()
+    if fd != -1:
+        try:
+            size = os.get_terminal_size(fd)
+            return (size.columns, size.lines)
+        except OSError: pass
+        finally: os.close(fd)
+    size = shutil.get_terminal_size(fallback=(80, 24))
+    return (size.columns, size.lines)
+
+def reset_caches():
+    # Clears all cached capability/metric values (used by tests).
+    global kitty_supported, cell_size, _query_failed
+    kitty_supported = None
+    cell_size = None
+    _query_failed = False
+    last_query_response[:] = b""
+    last_kitty_response[:] = b""
+    winsize[:] = (0, 0, 0, 0)
+
+def diagnostics():
+    # Returns a list of diagnostic strings about the terminal environment,
+    # for display/debugging.
+    import os
+    lines = [ "TERM=%r" % os.environ.get("TERM"),
+              "TERM_PROGRAM=%r TERM_PROGRAM_VERSION=%r" % (os.environ.get("TERM_PROGRAM"), os.environ.get("TERM_PROGRAM_VERSION")),
+              "kitty graphics protocol supported: %s" % is_kitty_supported(),
+              "kitty query response: %r" % bytes(last_kitty_response),
+              "cell size (px): %s" % (get_cell_size(),),
+              "cell size query response: %r" % bytes(last_query_response),
+              "winsize (rows, cols, xpix, ypix): %r" % (tuple(winsize),) ]
+    try:
+        from . import _webp
+        lines.append("webp conversion: %s" % _webp.describe())
+    except Exception: pass
+
+    return lines

nomadnet/ui/textui/images/_webp.py

@@ -0,0 +1,367 @@
+# Multi-backend WebP decoding
+# 
+# Kitty's graphics protocol decodes PNG natively but not WebP, so WebP
+# sources are converted transparently at the ImageData layer. This module
+# provides a conversion chain that *should* work almost anywhere with
+# zero extra dependencies:
+# 
+#  - Pillow/PIL, if installed with WebP support
+#  - libwebp via ctypes, decoding to RGBA, paired with a pure-Python PNG
+#    encoder. The system libwebp is present on most Linux/macOS systems
+#    as a transitive dependency of common graphics stacks and browsers
+#  - sips - the macOS built-in image tool
+#  - dwebp - from the system PATH (the "official" WebP tool)
+#  - ImageMagick / GraphicsMagick (magick, convert, gm convert)
+#  - ffmpeg / avconv
+#  - gdk-pixbuf-thumbnailer (GTK/GNOME systems)
+# 
+# Every backend's output is validated (it must be a valid PNG with the same
+# dimensions as the source WebP), so missing tools, or tools built without
+# WebP support, are simply skipped. The first backend that produces a valid
+# conversion is remembered and preferred for subsequent conversions, but the
+# preference is replaced whenever a different backend wins for a particular
+# image.
+# 
+# Environment overrides:
+# 
+#     NOMADNET_IMAGE_BACKEND   force a specific backend (strict)
+
+import ctypes
+import ctypes.util
+import hashlib
+import io
+import os
+import shutil
+import struct
+import subprocess
+import tempfile
+import zlib
+
+MAX_CACHE_BYTES = 192 * 1024 * 1024
+MAX_CACHE_FILES = 512
+CONVERSION_TIMEOUT = 5
+
+# libwebp soname candidates, in order of preference.
+_LIBWEBP_NAMES = [ "libwebp.so.7",
+                   "libwebp.so.6",
+                   "libwebp.so.5",
+                   "libwebp.so",
+                   "libwebp.dylib",
+                   "libwebp-7.dll",
+                   "libwebp.dll" ]
+
+# The built-in backend chain, in preference order.
+BACKENDS = ( "pil",
+             "libwebp-ctypes",
+             "sips",
+             "dwebp",
+             "magick",
+             "convert",
+             "gm",
+             "ffmpeg",
+             "avconv",
+             "gdk-pixbuf" )
+
+_ENV_BACKEND = os.environ.get("NOMADNET_IMAGE_BACKEND")
+_winner = None        # name of the currently preferred backend
+_last_backend = None  # backend used for the most recent conversion
+_convert_count = 0
+_cache_hits = 0
+
+_libwebp_cache = {"lib": None, "probed": False}
+
+###############################
+# PNG validation and encoding #
+###############################
+
+# Returns (width, height) if png is a valid PNG, else None
+def _png_dims(png):
+    if not png.startswith(b"\x89PNG\r\n\x1a\n") or len(png) < 24:
+        return None
+    if png[12:16] != b"IHDR":
+        return None
+    width = int.from_bytes(png[16:20], "big")
+    height = int.from_bytes(png[20:24], "big")
+    if not (width > 0 and height > 0):
+        return None
+    return (width, height)
+
+
+# Encodes RGBA pixel data as a PNG (8-bit RGBA, no filtering)
+def _encode_png_rgba(rgba, width, height):
+    def chunk(tag, payload):
+        return (struct.pack(">I", len(payload)) + tag + payload +
+                struct.pack(">I", zlib.crc32(tag + payload) & 0xFFFFFFFF))
+
+    ihdr = struct.pack(">IIBBBBB", width, height, 8, 6, 0, 0, 0)
+    stride = width * 4
+    raw = bytearray(height * (stride + 1))
+    pos = 0
+    for row in range(height):
+        raw[pos] = 0  # filter type: none
+        pos += 1
+        start = row * stride
+        raw[pos:pos + stride] = rgba[start:start + stride]
+        pos += stride
+    
+    return (b"\x89PNG\r\n\x1a\n" +
+            chunk(b"IHDR", ihdr) +
+            chunk(b"IDAT", zlib.compress(bytes(raw), 9)) +
+            chunk(b"IEND", b""))
+
+
+####################
+# Conversion cache #
+####################
+
+CONV_CACHE_BASE = None
+def cache_dir():
+    global CONV_CACHE_BASE
+    if not CONV_CACHE_BASE:
+        from nomadnet.NomadNetworkApp import NomadNetworkApp
+        CONV_CACHE_BASE = NomadNetworkApp._shared_instance.dispcachepath
+    return CONV_CACHE_BASE
+
+def _cache_path(source_data):
+    return os.path.join(cache_dir(), hashlib.sha256(source_data).hexdigest() + ".png")
+
+def _cache_get(source_data):
+    try:
+        with open(_cache_path(source_data), "rb") as f: png = f.read()
+        if _png_dims(png) is not None: return png
+    except OSError: pass
+    return None
+
+def _cache_put(source_data, png):
+    try:
+        directory = cache_dir()
+        os.makedirs(directory, exist_ok=True)
+        fd, tmp_path = tempfile.mkstemp(prefix=".tmp-", dir=directory)
+        try:
+            with os.fdopen(fd, "wb") as f: f.write(png)
+            os.replace(tmp_path, _cache_path(source_data))
+        except OSError:
+            try: os.unlink(tmp_path)
+            except OSError: pass
+        _prune_cache()
+    except OSError: pass
+
+# Evicts least-recently-used entries beyond the size/file limits
+def _prune_cache():
+    try:
+        entries = []
+        for name in os.listdir(cache_dir()):
+            path = os.path.join(cache_dir(), name)
+            try:
+                st = os.stat(path)
+                if st.st_size > 0:
+                    entries.append((st.st_mtime, st.st_size, path))
+            except OSError: pass
+
+        total = sum(e[1] for e in entries)
+        entries.sort()
+        while len(entries) > MAX_CACHE_FILES or total > MAX_CACHE_BYTES:
+            _, size, path = entries.pop(0)
+            total -= size
+            try: os.unlink(path)
+            except OSError: pass
+    except OSError: pass
+
+#######################
+# Individual backends #
+#######################
+
+# Runs a conversion CLI over temp files
+def _run_cli(argv, data):
+    if shutil.which(argv[0]) is None: return None
+    try:
+        with tempfile.TemporaryDirectory(prefix="nomadnet-img-") as td:
+            src = os.path.join(td, "source.webp")
+            dst = os.path.join(td, "output.png")
+            with open(src, "wb") as f: f.write(data)
+            args = [a.replace("{src}", src).replace("{dst}", dst) for a in argv]
+            subprocess.run(
+                args,
+                timeout=CONVERSION_TIMEOUT,
+                # stdin must be closed: some tools (e.g. ffmpeg) block
+                # on inherited stdin otherwise.
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+            with open(dst, "rb") as f: return f.read()
+    except (OSError, subprocess.TimeoutExpired, ValueError): return None
+
+def _backend_pil(data, dims):
+    try:
+        from PIL import Image
+        from PIL import features
+        if not features.check("webp"): return None
+    except ImportError: return None
+    try:
+        image = Image.open(io.BytesIO(data))
+        image.load()
+        buf = io.BytesIO()
+        image.save(buf, format="PNG")
+        return buf.getvalue()
+    except Exception: return None
+
+# Loads the system libwebp via ctypes, caching the result.
+def _get_libwebp():
+    if _libwebp_cache["probed"]: return _libwebp_cache["lib"]
+    _libwebp_cache["probed"] = True
+    lib = None
+    names = list(_LIBWEBP_NAMES)
+    found = ctypes.util.find_library("webp")
+    if found and found not in names: names.insert(0, found)
+    for name in names:
+        try:
+            lib = ctypes.CDLL(name)
+            break
+        except OSError: continue
+    
+    if lib is not None:
+        lib.WebPGetInfo.restype = ctypes.c_int
+        lib.WebPGetInfo.argtypes = [ ctypes.c_char_p, ctypes.c_size_t,
+                                     ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int) ]
+
+        lib.WebPDecodeRGBA.restype = ctypes.c_void_p
+        lib.WebPDecodeRGBA.argtypes = [ ctypes.c_char_p, ctypes.c_size_t,
+                                        ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int) ]
+        lib.WebPFree.restype = None
+        lib.WebPFree.argtypes = [ctypes.c_void_p]
+
+    _libwebp_cache["lib"] = lib
+    return lib
+
+def _backend_ctypes(data, dims):
+    lib = _get_libwebp()
+    if lib is None: return None
+    try:
+        buf = ctypes.create_string_buffer(data, len(data))
+        width = ctypes.c_int(0)
+        height = ctypes.c_int(0)
+        if not lib.WebPGetInfo(buf, len(data), ctypes.byref(width), ctypes.byref(height)): return None
+        if width.value <= 0 or height.value <= 0: return None
+        ptr = lib.WebPDecodeRGBA(buf, len(data), ctypes.byref(width), ctypes.byref(height))
+        if not ptr: return None
+        try: rgba = ctypes.string_at(ptr, width.value * height.value * 4)
+        finally: lib.WebPFree(ptr)
+        return _encode_png_rgba(rgba, width.value, height.value)
+    except Exception: return None
+
+def _backend_gdkpixbuf(data, dims):
+    if dims is None: return None
+    width, height = dims
+    return _run_cli(["gdk-pixbuf-thumbnailer", "-s", "%dx%d" % (width, height), "{src}", "{dst}"], data)
+
+def _backend_sips(data, dims):    return _run_cli(["sips", "-s", "format", "png", "{src}", "--out", "{dst}"], data)
+def _backend_dwebp(data, dims):   return _run_cli(["dwebp", "{src}", "-o", "{dst}"], data)
+def _backend_magick(data, dims):  return _run_cli(["magick", "{src}", "{dst}"], data)
+def _backend_convert(data, dims): return _run_cli(["convert", "{src}", "{dst}"], data)
+def _backend_gm(data, dims):      return _run_cli(["gm", "convert", "{src}", "{dst}"], data)
+def _backend_ffmpeg(data, dims):  return _run_cli(["ffmpeg", "-y", "-loglevel", "error", "-i", "{src}", "-frames:v", "1", "{dst}"], data)
+def _backend_avconv(data, dims):  return _run_cli(["avconv", "-y", "-loglevel", "error", "-i", "{src}", "-frames:v", "1", "{dst}"], data)
+
+_BACKENDS = { "pil": _backend_pil,
+              "libwebp-ctypes": _backend_ctypes,
+              "sips": _backend_sips,
+              "dwebp": _backend_dwebp,
+              "magick": _backend_magick,
+              "convert": _backend_convert,
+              "gm": _backend_gm,
+              "ffmpeg": _backend_ffmpeg,
+              "avconv": _backend_avconv,
+              "gdk-pixbuf": _backend_gdkpixbuf }
+
+#####################
+# Conversion driver #
+#####################
+
+def _webp_dims(data):
+    from ._imagedata import parse_image_header
+    try: fmt, width, height = parse_image_header(data)
+    except ValueError: return None
+    if fmt != "WebP": return None
+    return (width, height)
+
+def _backend_order():
+    if _ENV_BACKEND: return [_ENV_BACKEND] if _ENV_BACKEND in BACKENDS else []
+    order = list(BACKENDS)
+    if _winner in order:
+        order.remove(_winner)
+        order.insert(0, _winner)
+    return order
+
+# Converts WebP data to PNG bytes, or None on failure
+def convert_webp_to_png(data):
+    global _cache_hits
+    png = _cache_get(data)
+    if png is not None:
+        _cache_hits += 1
+        return png
+    return _convert_uncached(data)
+
+def _convert_uncached(data):
+    global _convert_count, _winner, _last_backend
+    dims = _webp_dims(data)
+    if dims is None: return None
+    for name in _backend_order():
+        try: png = _BACKENDS[name](data, dims)
+        except Exception: png = None
+        if png is not None and _png_dims(png) == dims:
+            _convert_count += 1
+            _last_backend = name
+            if not _ENV_BACKEND: _winner = name
+            _cache_put(data, png)
+            return png
+
+    return None
+
+##################################
+# Introspection and test support #
+##################################
+
+# Returns [(name, available), ...] for the built-in chain
+def available_backends():
+    out = []
+    for name in BACKENDS:
+        if name == "pil":
+            try:
+                from PIL import features
+                out.append((name, bool(features.check("webp"))))
+            except Exception: out.append((name, False))
+        elif name == "libwebp-ctypes": out.append((name, _get_libwebp() is not None))
+        elif name == "gdk-pixbuf":     out.append((name, shutil.which("gdk-pixbuf-thumbnailer") is not None))
+        else:                          out.append((name, shutil.which(name) is not None))
+
+    return out
+
+# A diagnostic string describing the conversion setup
+def describe():
+    if _ENV_BACKEND: return "backend forced via NOMADNET_IMAGE_BACKEND: %s" % _ENV_BACKEND
+    present = [name for name, avail in available_backends() if avail]
+    if not present: return "no conversion backend available (WebP images will not render)"
+    return "backend(s) available: %s; preferred: %s" % (", ".join(present), _last_backend or _winner or "none yet")
+
+# Usage statistics for diagnostics
+def stats():
+    return { "backend": _last_backend, "conversions": _convert_count,
+             "cache_hits": _cache_hits, "cache_dir": cache_dir() }
+
+def last_backend(): return _last_backend
+
+# Forces a specific backend name (testing/debugging); None to clear.
+def force_backend(name):
+    global _ENV_BACKEND
+    _ENV_BACKEND = name if name in BACKENDS else None
+
+# Clears dynamic state (winner, counters); used by tests.
+def reset():
+    global _winner, _last_backend, _convert_count, _cache_hits
+    _winner = None
+    _last_backend = None
+    _convert_count = 0
+    _cache_hits = 0
+    _libwebp_cache["lib"] = None
+    _libwebp_cache["probed"] = False

nomadnet/ui/textui/images/screen.py

@@ -0,0 +1,210 @@
+import urwid
+
+from . import _ctlseqs
+from ._imagestore import image_store
+from ._termlib import is_kitty_supported
+from .widget import ImageWidget
+
+# A raw display screen subclass with Kitty TGP side-channel support
+class ImageScreen(urwid.raw_display.Screen):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._ti_images = {}
+
+    ##########
+    # Output #
+    ##########
+
+    def write(self, data):
+        if isinstance(data, bytes): data = data.decode("ascii")
+        return super().write(data)
+
+    #############
+    # Lifecycle #
+    #############
+
+    def clear(self):
+        # Clears all images and forces a full repaint
+        if is_kitty_supported():
+            self.write(_ctlseqs.KITTY_DELETE_ALL)
+            self._ti_images = {}
+            image_store.mark_all_untransmitted()
+        return super().clear()
+
+    def _start(self, *args, **kwargs):
+        ret = super()._start(*args, **kwargs)
+        if is_kitty_supported():
+            # Clear any images left over from a previous run/instance.
+            self.write(_ctlseqs.KITTY_DELETE_ALL)
+            self._ti_images = {}
+            image_store.mark_all_untransmitted()
+        return ret
+
+    def _stop(self):
+        self.purge_images()
+        try: return super()._stop()
+        finally: self._ti_images = {}
+
+    def purge_images(self):
+        # Purges all image data from the terminal buffer.
+        if not is_kitty_supported(): return
+        if self._ti_images or image_store.transmitted():
+            self.write(_ctlseqs.KITTY_DELETE_ALL)
+        self._ti_images = {}
+        image_store.mark_all_untransmitted()
+
+    ###########
+    # Drawing #
+    ###########
+
+    def draw_screen(self, size, canvas):
+        # Draws the canvas, then updates on-screen images within a
+        # synchronized-update region to prevent flicker.        
+        self.write(_ctlseqs.BEGIN_SYNCED_UPDATE)
+        try:
+            super().draw_screen(size, canvas)
+            self._ti_update(canvas)
+        finally:
+            self.write(_ctlseqs.END_SYNCED_UPDATE)
+            self.flush()
+
+    def _ti_update(self, canvas):
+        # Places/updates/deletes image placements to match the current
+        # canvas, and evicts orphaned image data.
+        if not is_kitty_supported(): return
+
+        entries = self._ti_scan(canvas)
+        current = {}
+        for widget, row, col, canvas_cols, trim_top, vis_rows in entries:
+            current[id(widget)] = (widget, row, col, canvas_cols, trim_top, vis_rows)
+
+        # For widgets that scrolled completely out of view, remove their
+        # placement only. The image data is kept in the terminal so that
+        # scrolling back just re-places it.
+        for wid, info in list(self._ti_images.items()):
+            if wid in current: continue
+            self._ti_unplace(info["widget"])
+            del self._ti_images[wid]
+
+        # For visible widgets, place/update the visible slice in place.
+        for wid, (widget, row, col, canvas_cols, trim_top, vis_rows) in current.items():
+            info = self._ti_images.setdefault(wid, {"widget": widget, "drawn": None})
+            if vis_rows <= 0 or trim_top >= widget._ti_rows:
+                if info["drawn"] is not None:
+                    self._ti_unplace(widget)
+                    info["drawn"] = None
+                continue
+
+            key = self._placement_key(widget, row, col, canvas_cols, trim_top, vis_rows)
+            if key is None:
+                if info["drawn"] is not None:
+                    self._ti_unplace(widget)
+                    info["drawn"] = None
+                continue
+
+            if info["drawn"] == key: continue
+            self._ti_place(widget, key)
+            info["drawn"] = key
+
+        # Evict image data that no live widget references any more
+        for key, image_id in image_store.evict_orphans():
+            self.write(_ctlseqs.KITTY_DELETE_IMAGE % image_id)
+            image_store.evict(key)
+
+    #############
+    # Internals #
+    #############
+
+    def _ti_scan(self, canvas):
+        # Walks the canvas shards and returns a list of
+        # (widget, row, col, canvas_cols, trim_top, visible_rows)
+        # for every visible image canvas.
+        found = []
+        if not isinstance(canvas, urwid.CompositeCanvas): return found
+
+        def process_shard_tails():
+            nonlocal col
+            while col in shard_tails:
+                *trim, cols, rows, canv = shard_tails[col]
+                if rows > n_rows: shard_tails[col] = (*trim, cols, rows - n_rows, canv)
+                else:             del shard_tails[col]
+                col += cols
+
+        shard_tails = {}
+        row = 0
+        for n_rows, cviews in canvas.shards:
+            col = 0
+            for cview in cviews:
+                process_shard_tails()
+                *trim, cols, rows, _, canv = cview
+                widget = self._ti_widget_of(canv)
+                if widget is not None: found.append((widget, row, col, cols, trim[1], rows))
+                if rows > n_rows: shard_tails[col] = (*trim, cols, rows - n_rows, canv)
+                col += cols
+
+            process_shard_tails()
+            row += n_rows
+
+        return found
+
+    @staticmethod
+    def _ti_widget_of(canv):
+        # Returns the ImageWidget for a leaf canvas, or None
+        widget_info = getattr(canv, "_widget_info", None)
+        if not widget_info: return None
+        widget = widget_info[0]
+        if isinstance(widget, ImageWidget) and widget._ti_data.ok and not widget._ti_closed: return widget
+        return None
+
+    def _placement_key(self, widget, row, col, canvas_cols, trim_top, vis_rows):
+        # Compute placement parameters for the visible slice of
+        # an image, including the alignment offset.
+
+        cols = widget._ti_cols
+        rows = widget._ti_rows
+        if not (cols and rows): return None
+        place_col = col
+        if cols < canvas_cols:
+            align = widget._ti_align
+            spare = canvas_cols - cols
+            if align == ">":   place_col += spare
+            elif align == "|": place_col += spare // 2
+
+        vis_rows = max(1, min(vis_rows, rows))
+        y_px = h_px = None
+        if not (trim_top == 0 and vis_rows == rows):
+            # Crop to the visible slice of the image, in image pixels
+            ih = widget._ti_data.height
+            y_px = min(ih - 1, round(trim_top * ih / rows))
+            h_px = max(1, min(ih - y_px, round(vis_rows * ih / rows)))
+
+        return (row, place_col, cols, vis_rows, y_px, h_px)
+
+    def _ti_place(self, widget, key):
+        # Place (or in-place update) the visible slice of an image
+        row, place_col, cols, vis_rows, y_px, h_px = key
+        entry = image_store.entry_for(widget._ti_key)
+        if entry is None:
+            # This should not happen, but if the image data was evicted
+            # while the image was still alive, re-register it so the
+            # data can be re-transmitted to the terminal buffer.
+            image_store.register(widget._ti_key, widget._ti_data.data)
+            entry = image_store.entry_for(widget._ti_key)
+        image_id = entry["image_id"]
+
+        if not entry["transmitted"]:
+            payload = widget._ti_data.data
+            b64_len = (len(payload) * 4 + 2) // 3
+            n_chunks = max(1, (b64_len + _ctlseqs.CHUNK_SIZE - 1) // _ctlseqs.CHUNK_SIZE)
+            self.write(_ctlseqs.kitty_transmission(_ctlseqs.kitty_transmit_control(image_id), payload))
+            entry["transmitted"] = True
+
+        self.write(_ctlseqs.CURSOR_POSITION % (row + 1, place_col + 1))
+        self.write(_ctlseqs.kitty_placement(image_id, widget._ti_z, cols, vis_rows, widget._ti_z, y=y_px, h=h_px))
+
+    def _ti_unplace(self, widget):
+        # Delete a widget's placement, but keep the
+        # image data in the terminal buffer.
+        entry = image_store.entry_for(widget._ti_key)
+        if entry is None: return
+        self.write(_ctlseqs.KITTY_DELETE_PLACEMENT % (entry["image_id"], widget._ti_z))

nomadnet/ui/textui/images/widget.py

@@ -0,0 +1,283 @@
+# An urwid widget that occupies the space an image should fill in a layout,
+# without embedding any escape sequences in its canvas. The actual terminal
+# graphics are transmitted by the ImageScreen class which tracks the widget's
+# on-screen position after every draw.
+# 
+# This separation keeps the canvas trivially simple (urwid sees only blank
+# rows) and lets the screen handle the hard parts; positioning, deletion when
+# scrolled away, terminal resize and redraws - all of which are driven by
+# urwid's own shard bookkeeping rather than by fragile escape-sequence
+# manipulation inside canvas rows.
+# 
+# When the Kitty graphics protocol is unavailable (or the image cannot be
+# loaded), the widget renders a simple text placeholder instead.
+
+import os
+import re
+import urwid
+
+from . import _termlib
+from ._imagestore import image_store
+
+DEFAULT_CELL_SIZE = (1, 2)  # Fallback, assume a 1:2 cell aspect ratio
+MAX_Z_INDEX       = 2 ** 31 - 1
+UI_MARGIN         = 8
+_SIZE_SPEC_RE     = re.compile(r"(\d+(?:\.\d+)?)%")
+_ALIGNS           = { "left": "<", "<": "<",
+                      "center": "|", "|": "|",
+                      "right": ">", ">": ">" }
+
+def _check_align(value):
+    if value not in _ALIGNS: raise ValueError("invalid alignment %r (expected 'left', 'center' or 'right')" % (value,))
+    return _ALIGNS[value]
+
+
+def _check_size_spec(value, name):
+    # None = auto, int = absolute cells/rows, "NN%" = percent, "n" = native resolution
+    if value is None: return None
+    
+    if isinstance(value, int):
+        if value <= 0: raise ValueError("%s must be a positive integer, got %r" % (name, value))
+        return value
+    
+    if isinstance(value, str):
+        if value.lower() == "n": return "n"
+        match = _SIZE_SPEC_RE.fullmatch(value)
+        if not match or float(match.group(1)) <= 0:
+            raise ValueError("invalid %s specification %r (expected a positive integer, a percent string like '50%%' or 'n' for native resolution)" % (name, value))
+        return value
+    
+    raise TypeError("invalid type for %s (got %s)" % (name, type(value).__name__))
+
+
+class ImageWidget(urwid.Widget):
+    # A widget that displays an image via the Kitty graphics protocol.
+    # The widget is a plain FLOW/BOX widget; its canvas is blank. The
+    # ImageScreen class performs the actual drawing over the Kitty
+    # TGP side-channel.
+
+    placeholder_glyph = ""
+
+    _sizing = frozenset((urwid.FLOW, urwid.BOX))
+    ignore_focus = True
+
+    # Z-index allocation. Each widget instance gets a unique z-index so that
+    # the screen can delete exactly one image when it scrolls away or moves.
+    _ti_next_z = 1
+    _ti_free_z = set()
+
+    def __init__(self, path, name=None, width=None, height=None, align="center"):
+        if path and not isinstance(path, (str, os.PathLike)): raise TypeError("path must be a string or path-like object")
+        self.path = path
+        self._notice = None
+        self._ti_path = None
+        self._ti_data = None
+        self._ti_width = _check_size_spec(width, "width")
+        self._ti_height = _check_size_spec(height, "height")
+        self._ti_align = _check_align(align)
+        self._ti_name = name if name is not None else ""
+        self._ti_z = self._ti_alloc_z()
+        self.image_id = self._ti_z
+        
+        # Display size (cols, rows) from the most recent render.
+        # Used by the screen to place the image.
+        self._ti_cols = None
+        self._ti_rows = None
+        self._ti_closed = False
+        self._ti_key = None
+        super().__init__()
+        if self.path: self.load()
+
+    def load(self, path=None, remote_source=False):
+        if path: self.path = path
+        if self.path:
+            from ._imagedata import ImageData
+            self._ti_path = os.path.abspath(os.fspath(self.path))
+            self._ti_data = ImageData(self._ti_path, remote_source=remote_source)
+
+            if self._ti_data.ok:
+                # Register with the image store (de-duplicates identical data
+                # and allocates the terminal-side image id on first use).
+                self._ti_key = self._ti_data.key
+                image_store.register(self._ti_key, self._ti_data.data)
+                self._invalidate()
+            else:
+                error_msg = self._ti_data.error or "Unknown error"
+                self.notice(f"Error: {error_msg}")
+
+    def notice(self, msg=""):
+        if not msg: self._notice = None
+        else:       self._notice = str(msg)
+        self._invalidate()
+
+    def __del__(self):
+        try: type(self)._ti_free_z.add(self._ti_z)
+        except Exception: pass
+        try: self.close()
+        except Exception: pass
+
+    def close(self):
+        # Releases the widget's reference to its image data.
+        # The data is evicted from the terminal by the screen as soon as no
+        # live widget references it any more. Safe to call multiple times and
+        # from __del__.
+        try:
+            if not self._ti_closed:
+                self._ti_closed = True
+                if self._ti_key is not None: image_store.unregister(self._ti_key)
+        except Exception: pass
+
+    @classmethod
+    def _ti_alloc_z(cls):
+        if cls._ti_free_z: return cls._ti_free_z.pop()
+        z = cls._ti_next_z
+        if z >= MAX_Z_INDEX: raise RuntimeError("Too many image widgets")
+        cls._ti_next_z += 1
+        return z
+
+    ##########
+    # Sizing #
+    ##########
+
+    width = property(
+        lambda self: self._ti_width,
+        doc="""Display width specification: None (full width), a positive
+        int (columns), a percent string ("NN%") of the available layout
+        width, or "n" for native resolution.""",
+    )
+
+    @width.setter
+    def width(self, value):
+        self._ti_width = _check_size_spec(value, "width")
+        self._invalidate()
+
+    height = property(
+        lambda self: self._ti_height,
+        doc="""Display height specification: None (derived, aspect
+        preserving), a positive int (rows), a percent string ("NN%") of
+        the visible page height, or "n" for native resolution.""",
+    )
+
+    @height.setter
+    def height(self, value):
+        self._ti_height = _check_size_spec(value, "height")
+        self._invalidate()
+
+    align = property(
+        lambda self: self._ti_align,
+        doc="""Horizontal alignment of the image within its allocated
+        space: ``"<"`` (left), ``"|"`` (center) or ``">"`` (right).""",
+    )
+
+    @align.setter
+    def align(self, value):
+        self._ti_align = _check_align(value)
+        self._invalidate()
+
+    def _short(self): return "img#%x" % (id(self) & 0xFFFFF)
+
+    def _display_size(self, maxcol, maxrow=None):
+        if not self._ti_data.ok: return (min(maxcol, 1), 1)
+        iw, ih = self._ti_data.size
+        cell_w, cell_h = _termlib.get_cell_size() or DEFAULT_CELL_SIZE
+        w_spec, h_spec = self._ti_width, self._ti_height
+
+        def _pct(value): return float(value[:-1]) / 100.0
+
+        def _raw_width():
+            if w_spec is None: return float(maxcol)
+            if isinstance(w_spec, int): return float(w_spec)
+            if w_spec == "n": return iw / cell_w
+            return maxcol * _pct(w_spec)
+
+        def _raw_height():
+            if h_spec is None: return None
+            if isinstance(h_spec, int): return float(h_spec)
+            if h_spec == "n": return ih / cell_h
+            columns, lines = _termlib.get_terminal_size()
+            page = max(1, lines - UI_MARGIN)
+            return page * _pct(h_spec)
+
+        def _height_for_width(cols): return cols * cell_w * ih / (cell_h * iw)
+        def _width_for_height(rows): return rows * cell_h * iw / (cell_w * ih)
+
+        if w_spec is not None and h_spec is not None:
+            # Stretch mode: Target rectangle, uniformly scaled down only if
+            # it would overflow the available space.
+            target_w = max(1.0, _raw_width())
+            target_h = max(1.0, _raw_height())
+            scale = min(1.0, maxcol / target_w)
+            if maxrow is not None: scale = min(scale, maxrow / target_h)
+            cols = max(1, round(target_w * scale))
+            rows = max(1, round(target_h * scale))
+        elif w_spec is not None:
+            # Width given: Height derived, aspect preserving.
+            cols = max(1, min(maxcol, round(_raw_width())))
+            rows = max(1, round(_height_for_width(cols)))
+            if maxrow is not None and rows > maxrow:
+                rows = maxrow
+                cols = max(1, min(maxcol, round(_width_for_height(rows))))
+        elif h_spec is not None:
+            # Height given: Width derived, aspect preserving.
+            rows = max(1, round(_raw_height()))
+            cols = max(1, min(maxcol, round(_width_for_height(rows))))
+            if maxrow is not None and rows > maxrow:
+                rows = maxrow
+                cols = max(1, min(maxcol, round(_width_for_height(rows))))
+        else:
+            # Default: Full width, height derived.
+            cols = maxcol
+            rows = max(1, round(_height_for_width(cols)))
+            if maxrow is not None and rows > maxrow:
+                rows = maxrow
+                cols = max(1, min(maxcol, round(_width_for_height(rows))))
+        
+        return (cols, rows)
+
+    def rows(self, size, focus=False):
+        maxcol = size[0]
+        if not (self._ti_data and self._ti_data.ok and _termlib.is_kitty_supported()):
+            return len(self._placeholder_lines())
+        n = self._display_size(maxcol)[1]
+        return n
+
+    def render(self, size, focus=False):
+        if len(size) == 2:
+            maxcol, maxrow = size
+            is_box = True
+        else:
+            maxcol, maxrow = size[0], None
+            is_box = False
+
+        if not (self._ti_data and self._ti_data.ok and _termlib.is_kitty_supported()):
+            return self._placeholder_canvas(maxcol, maxrow, focus)
+
+        cols, rows = self._display_size(maxcol, maxrow)
+        self._ti_cols, self._ti_rows = cols, rows
+        # urwid requires canvases to fill the allocated space exactly
+        # (width == maxcol for FLOW, (maxcol, maxrow) for BOX). The image
+        # itself is placed by the screen at its resolved size inside the
+        # canvas, left-aligned; the rest of the canvas stays blank.
+        if is_box: return ImageCanvas(maxcol, maxrow)
+        else:      return ImageCanvas(maxcol, rows)
+
+    #########################
+    # Placeholder rendering #
+    #########################
+
+    def _placeholder_lines(self):
+        return [f"{self.placeholder_glyph}{self._notice or self._ti_name}"]
+
+    def _placeholder_canvas(self, maxcol, maxrow, focus):
+        align = {"<": "left", "|": "center", ">": "right"}.get(self._ti_align, "center")
+        text = urwid.Text("\n".join(self._placeholder_lines()), align=align)
+        if maxrow is not None: return urwid.Filler(text).render((maxcol, maxrow), focus)
+        else:                  return text.render((maxcol,), focus)
+
+
+class ImageCanvas(urwid.SolidCanvas):
+    # A blank canvas marking the space occupied by an image.
+    # Urwid sets _widget_info on this canvas to the owning
+    # ImageWidget. ImageScreen finds image canvases through
+    # that reference during its shard scan.
+    def __init__(self, cols, rows): super().__init__(" ", cols, rows)

nomadnet/util.py

@@ -0,0 +1,222 @@
+import os
+import re
+import unicodedata
+
+
+def write_config_atomic(config):
+    path = getattr(config, "filename", None)
+    if not path:
+        config.write()
+        return
+
+    directory = os.path.dirname(path) or "."
+    tmp = os.path.join(directory, "."+os.path.basename(path)+".tmp")
+    try:
+        try:
+            existing_mode = os.stat(path).st_mode
+        except OSError:
+            existing_mode = None
+
+        with open(tmp, "wb") as tmp_file:
+            config.write(tmp_file)
+            tmp_file.flush()
+            try: os.fsync(tmp_file.fileno())
+            except Exception: pass
+
+        if existing_mode is not None:
+            try: os.chmod(tmp, existing_mode)
+            except Exception: pass
+
+        os.replace(tmp, path)
+    except Exception:
+        try: os.remove(tmp)
+        except Exception: pass
+        raise
+
+
+invalid_rendering = ["🕵️", "☝"]
+
+# Unicode blocks to strip (symbols, dingbats, emoji ranges)
+STRIP_BLOCKS_RE = re.compile(
+    '['
+    '\U0001F600-\U0001F64F'  # Emoticons
+    '\U0001F300-\U0001F5FF'  # Misc Symbols & Pictographs
+    '\U0001F680-\U0001F6FF'  # Transport & Map Symbols
+    '\U0001F700-\U0001F77F'  # Alchemical Symbols
+    '\U0001F780-\U0001F7FF'  # Geometric Shapes Extended
+    '\U0001F800-\U0001F8FF'  # Supplemental Arrows-C
+    '\U0001F900-\U0001F9FF'  # Supplemental Symbols & Pictographs
+    '\U0001FA00-\U0001FA6F'  # Chess Symbols
+    '\U0001FA70-\U0001FAFF'  # Symbols & Pictographs Extended-A
+    '\U0001F1E0-\U0001F1FF'  # Flags (iOS/regional indicators)
+    '\u2600-\u26FF'          # Misc Symbols (☀, ☁, ☂, etc.)
+    '\u2700-\u27BF'          # Dingbats (✂, ✈, ✉, ✌, etc.)
+    '\uFE00-\uFE0F'          # Variation Selectors
+    '\U000E0100-\U000E01EF'  # Variation Selectors Supplement
+    '\U0001F3FB-\U0001F3FF'  # Emoji modifiers (skin tones)
+    ']+',
+    flags=re.UNICODE
+)
+
+# Control characters and zero-width characters to strip
+STRIP_CONTROL_RE = re.compile(
+    '['
+    '\x00-\x08'      # C0 controls (NUL-BS)
+    '\x0B\x0C'       # VT, FF
+    '\x0E-\x1F'      # C0 controls (SO-US)
+    '\x7F-\x9F'      # DEL and C1 controls
+    '\u200B-\u200F'  # Zero-width chars, LRM, RLM, etc.
+    '\u202A-\u202E'  # Bidi embedding controls
+    '\u2060-\u206F'  # Format chars (word joiner, etc.)
+    '\uFEFF'         # BOM / Zero Width NBSP
+    '\uFFF0-\uFFF8'  # Specials
+    ']+',
+    flags=re.UNICODE
+)
+
+# Surrogates and private use areas
+# Shouldn't appear in valid UTF-8, but strip just in case
+STRIP_PRIVATE_RE = re.compile(
+    '['
+    '\uD800-\uDFFF'         # Surrogates
+    '\uE000-\uF8FF'         # Private Use Area
+    '\uF900-\uFAFF'         # CJK Compatibility Ideographs (keep? strip for safety)
+    '\uFE10-\uFE1F'         # Vertical Forms
+    '\uFE20-\uFE2F'         # Combining Half Marks
+    '\U000F0000-\U000FFFFF' # Supplementary Private Use Area-A
+    '\U00100000-\U0010FFFF' # Supplementary Private Use Area-B
+    ']+',
+    flags=re.UNICODE
+)
+
+def strip_modifiers(text):
+    def process_characters(text):
+        result = []
+        i = 0
+        while i < len(text):
+            char = text[i]
+            category = unicodedata.category(char)
+            
+            if category.startswith(('L', 'N', 'P', 'S')):
+                result.append(char)
+                i += 1
+
+            elif category.startswith(('M', 'Sk', 'Cf')) or char in '\u200d\u200c':
+                i += 1
+
+            else:
+                result.append(char)
+                i += 1
+                
+        return ''.join(result)
+
+    if text == None: return None
+
+    for char in invalid_rendering:
+        text = text.replace(char, " ")
+    
+    stripped = process_characters(text)
+    stripped = re.sub(r'[\uFE00-\uFE0F]', '', stripped)
+    stripped = re.sub(r'[\U000E0100-\U000E01EF]', '', stripped, flags=re.UNICODE)
+    stripped = re.sub(r'[\U0001F3FB-\U0001F3FF]', '', stripped, flags=re.UNICODE)
+    stripped = re.sub(r'[\u200D\u200C]', '', stripped)
+    stripped = re.sub(r'\r\n?', '\n', stripped)
+    stripped = STRIP_CONTROL_RE.sub('', stripped)
+
+    return stripped.strip()
+
+def sanitize_name(name):
+    if name is None: return None
+    
+    # Convert to string and normalize to NFKC
+    # NFKC: Compatibility decomposition followed by canonical composition
+    # This handles: ① to 1, Ⅰ to I, etc., while keeping composed forms
+    name = str(name)
+    name = unicodedata.normalize('NFKC', name)
+    
+    # Build result using category-based filtering
+    result = []
+    for char in name:
+        cat = unicodedata.category(char)
+        cat_prefix = cat[0] if cat else 'C'
+        
+        # Allow letters (L*), numbers (N*), and punctuation (P*)
+        if cat_prefix in ('L', 'N', 'P'): result.append(char)
+        
+        # Allow space separator, normalize to regular space
+        elif cat == 'Zs': result.append(' ')
+
+        # Convert line/paragraph separators to space
+        elif cat in ('Zl', 'Zp'): result.append(' ')
+        
+        # Allow spacing combining marks (Mc) for Indic, Hebrew, etc.
+        elif cat == 'Mc': result.append(char)
+        
+        # Allow modifier letters (Lm) - e.g., ʰ, ʱ, ː
+        elif cat == 'Lm': result.append(char)
+        
+        # Strip everything else:
+        # - Mn (Nonspacing Mark): diacritics, combining marks (Zalgo)
+        # - Me (Enclosing Mark): enclosing combining marks
+        # - C* (Controls, Format, Surrogates, Private Use, Unassigned)
+        # - S* (Symbols: currency, math, modifiers, other)
+    
+    name = ''.join(result)
+    
+    # Additional block-based stripping for symbols that categories missed
+    name = STRIP_BLOCKS_RE.sub('', name)
+    name = STRIP_CONTROL_RE.sub('', name)
+    name = STRIP_PRIVATE_RE.sub('', name)
+    
+    # Collapse multiple whitespace characters
+    name = re.sub(r'\s+', ' ', name)
+    
+    # Strip leading/trailing whitespace
+    name = name.strip()
+    
+    return name
+
+def strip_micron(text):
+    text = re.sub(r'`[FB][0-9a-fA-F]{3}', '', text)
+    text = re.sub(r'`[FB]T[0-9a-fA-F]{6}', '', text)
+    text = re.sub(r'`[!*_=]', '', text)
+    text = re.sub(r'`f`b', '', text)
+    text = re.sub(r'`f', '', text)
+    text = re.sub(r'`b', '', text)
+    text = re.sub(r'`<', '', text)
+    text = re.sub(r'`>', '', text)
+    text = re.sub(r'`{', '', text)
+    return text
+
+def strip_escaped_micron(text):
+    text = re.sub(r'¦[FB][0-9a-fA-F]{3}', '', text)
+    text = re.sub(r'¦[FB]T[0-9a-fA-F]{6}', '', text)
+    text = re.sub(r'¦[!*_=]', '', text)
+    text = re.sub(r'¦f`b', '', text)
+    text = re.sub(r'¦f', '', text)
+    text = re.sub(r'¦b', '', text)
+    text = re.sub(r'¦<', '', text)
+    text = re.sub(r'¦>', '', text)
+    text = re.sub(r'¦{', '', text)
+    return text
+
+def unescape_micron(text):
+    text = re.sub(r'¦([FB][0-9a-fA-F]{3})', r'`\1', text)
+    text = re.sub(r'¦([FB]T[0-9a-fA-F]{6})', r'`\1', text)
+    text = re.sub(r'¦([!*_=])', r'`\1', text)
+    text = re.sub(r'¦(f`b)', r'`\1', text)
+    text = re.sub(r'¦(f)', r'`\1', text)
+    text = re.sub(r'¦(b)', r'`\1', text)
+    text = re.sub(r'¦(<)', r'`\1', text)
+    text = re.sub(r'¦(>)', r'`\1', text)
+    text = re.sub(r'¦({)', r'`\1', text)
+    return text
+
+def strip_non_formatting_tags(text):
+    text = re.sub(r'`<', '', text)
+    text = re.sub(r'`>', '', text)
+    text = re.sub(r'`{', '', text)
+    text = re.sub(r'`r', '', text)
+    text = re.sub(r'`c', '', text)
+    text = re.sub(r'`l', '', text)
+    return text

nomadnet/vendor/AsciiChart.py

@@ -0,0 +1,91 @@
+from __future__ import division
+from math import ceil, floor, isnan
+# Derived from asciichartpy |  https://github.com/kroitor/asciichart/blob/master/asciichartpy/__init__.py
+class AsciiChart:
+    def __init__(self, glyphset="unicode"):
+        self.symbols = ['┼', '┤', '╶', '╴', '─', '╰', '╭', '╮', '╯', '│']
+        if glyphset == "plain":
+            self.symbols = ['+', '|', '-', '-', '-', '\'', ',', '.', '`', '|']
+    def plot(self, series, cfg=None):
+        if len(series) == 0:
+            return ''
+        if not isinstance(series[0], list):
+            if all(isnan(n) for n in series):
+                return ''
+            else:
+                series = [series]
+        cfg = cfg or {}
+        minimum = cfg.get('min', min(filter(lambda n: not isnan(n), [j for i in series for j in i])))
+        maximum = cfg.get('max', max(filter(lambda n: not isnan(n), [j for i in series for j in i])))
+        symbols = cfg.get('symbols', self.symbols)
+        if minimum > maximum:
+            raise ValueError('The min value cannot exceed the max value.')
+        interval = maximum - minimum
+        offset = cfg.get('offset', 3)
+        height = cfg.get('height', interval)
+        ratio = height / interval if interval > 0 else 1
+
+        min2 = int(floor(minimum * ratio))
+        max2 = int(ceil(maximum * ratio))
+
+        def clamp(n):
+            return min(max(n, minimum), maximum)
+
+        def scaled(y):
+            return int(round(clamp(y) * ratio) - min2)
+
+        rows = max2 - min2
+
+        width = 0
+        for i in range(0, len(series)):
+            width = max(width, len(series[i]))
+        width += offset
+
+        placeholder = cfg.get('format', '{:8.2f} ')
+
+        result = [[' '] * width for i in range(rows + 1)]
+
+        for y in range(min2, max2 + 1):
+            if callable(placeholder):
+                label = placeholder(maximum - ((y - min2) * interval / (rows if rows else 1))).rjust(12)
+            else:
+                label = placeholder.format(maximum - ((y - min2) * interval / (rows if rows else 1)))
+
+            result[y - min2][max(offset - len(label), 0)] = label
+            result[y - min2][offset - 1] = symbols[0] if y == 0 else symbols[1]
+
+        d0 = series[0][0]
+        if not isnan(d0):
+            result[rows - scaled(d0)][offset - 1] = symbols[0]
+
+        for i in range(0, len(series)):
+            for x in range(0, len(series[i]) - 1):
+                d0 = series[i][x + 0]
+                d1 = series[i][x + 1]
+
+                if isnan(d0) and isnan(d1):
+                    continue
+
+                if isnan(d0) and not isnan(d1):
+                    result[rows - scaled(d1)][x + offset] = symbols[2]
+                    continue
+
+                if not isnan(d0) and isnan(d1):
+                    result[rows - scaled(d0)][x + offset] = symbols[3]
+                    continue
+
+                y0 = scaled(d0)
+                y1 = scaled(d1)
+                if y0 == y1:
+                    result[rows - y0][x + offset] = symbols[4]
+                    continue
+
+                result[rows - y1][x + offset] = symbols[5] if y0 > y1 else symbols[6]
+                result[rows - y0][x + offset] = symbols[7] if y0 > y1 else symbols[8]
+
+                start = min(y0, y1) + 1
+                end = max(y0, y1)
+                for y in range(start, end):
+                    result[rows - y][x + offset] = symbols[9]
+
+        return '\n'.join([''.join(row).rstrip() for row in result])

nomadnet/vendor/Scrollable.py

@@ -0,0 +1,583 @@
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details
+# http://www.gnu.org/licenses/gpl-3.0.txt
+
+import RNS
+import time
+import urwid
+from urwid.widget import BOX, FIXED, FLOW
+from collections import deque
+
+# Scroll actions
+SCROLL_LINE_UP        = 'line up'
+SCROLL_LINE_DOWN      = 'line down'
+SCROLL_PAGE_UP        = 'page up'
+SCROLL_PAGE_DOWN      = 'page down'
+SCROLL_TO_TOP         = 'to top'
+SCROLL_TO_END         = 'to end'
+
+# Scrollbar positions
+SCROLLBAR_LEFT  = 'left'
+SCROLLBAR_RIGHT = 'right'
+
+class Scrollable(urwid.WidgetDecoration):
+    def sizing(self):
+        return frozenset([BOX,])
+
+    def selectable(self):
+        return True
+
+    def __init__(self, widget, force_forward_keypress = False):
+        """Box widget that makes a fixed or flow widget vertically scrollable
+
+        TODO: Focusable widgets are handled, including switching focus, but
+        possibly not intuitively, depending on the arrangement of widgets.  When
+        switching focus to a widget that is ouside of the visible part of the
+        original widget, the canvas scrolls up/down to the focused widget.  It
+        would be better to scroll until the next focusable widget is in sight
+        first.  But for that to work we must somehow obtain a list of focusable
+        rows in the original canvas.
+        """
+        if not any(s in widget.sizing() for s in (FIXED, FLOW)):
+            raise ValueError('Not a fixed or flow widget: %r' % widget)
+        self._trim_top = 0
+        self._scroll_action = None
+        self._forward_keypress = None
+        self._old_cursor_coords = None
+        self._rows_max_cached = 0
+        self.force_forward_keypress = force_forward_keypress
+        self.force_cursor_update = False
+        self.anchor_cursor_update = False
+        self.cursor_update_forced_at = 0
+        super().__init__(widget)
+
+    def render(self, size, focus=False):
+        maxcol, maxrow = size
+
+        # Render complete original widget
+        ow = self._original_widget
+        ow_size = self._get_original_widget_size(size)
+        canv_full = ow.render(ow_size, focus)
+
+        # Make full canvas editable
+        canv = urwid.CompositeCanvas(canv_full)
+        canv_cols, canv_rows = canv.cols(), canv.rows()
+
+        if canv_cols <= maxcol:
+            pad_width = maxcol - canv_cols
+            if pad_width > 0:
+                # Canvas is narrower than available horizontal space
+                canv.pad_trim_left_right(0, pad_width)
+
+        if canv_rows <= maxrow:
+            fill_height = maxrow - canv_rows
+            if fill_height > 0:
+                # Canvas is lower than available vertical space
+                canv.pad_trim_top_bottom(0, fill_height)
+
+        if canv_cols <= maxcol and canv_rows <= maxrow:
+            # Canvas is small enough to fit without trimming
+            if self._trim_top != 0 or self._scroll_action is not None:
+                # Reset potentially stale _trim_top
+                self._trim_top = 0
+                self._scroll_action = None
+            return canv
+
+        self._adjust_trim_top(canv, size)
+
+        # Trim canvas if necessary
+        trim_top = self._trim_top
+        trim_end = canv_rows - maxrow - trim_top
+        trim_right = canv_cols - maxcol
+        if trim_top > 0:
+            canv.trim(trim_top)
+        if trim_end > 0:
+            canv.trim_end(trim_end)
+        if trim_right > 0:
+            canv.pad_trim_left_right(0, -trim_right)
+
+        # Disable cursor display if cursor is outside of visible canvas parts
+        if canv.cursor is not None:
+            curscol, cursrow = canv.cursor
+            if cursrow >= maxrow or cursrow < 0:
+                canv.cursor = None
+
+        # Figure out whether we should forward keypresses to original widget
+        if canv.cursor is not None:
+            # Trimmed canvas contains the cursor, e.g. in an Edit widget
+            self._forward_keypress = True
+        else:
+            force_check  = self.force_cursor_update and (time.time()-self.cursor_update_forced_at < 0.25)
+            force_check |= self.anchor_cursor_update
+            if canv_full.cursor is not None or force_check:
+                self.force_cursor_update = False
+                self.anchor_cursor_update = False
+                # Full canvas contains the cursor, but scrolled out of view
+                self._forward_keypress = False
+                
+                # Reset cursor position on page/up down scrolling
+                try:
+                    if hasattr(ow, "automove_cursor_on_scroll") and ow.automove_cursor_on_scroll:
+                        pwi = 0
+                        ch = 0
+                        last_hidden = False
+                        first_visible = False
+                        for w,o in ow.contents:
+                            wcanv = w.render((maxcol,))
+                            wh = wcanv.rows()
+                            if wh:
+                                ch += wh
+
+                            if not last_hidden and ch >= self._trim_top:
+                                last_hidden = True
+
+                            elif last_hidden:
+                                if not first_visible:
+                                    first_visible = True
+
+                                if w.selectable():
+                                    ow.focus = pwi
+
+                                    st = None
+                                    nf = ow.get_focus()
+                                    if hasattr(nf, "key_timeout"):
+                                        st = nf
+                                    elif hasattr(nf, "original_widget"):
+                                        no = nf.original_widget
+                                        if hasattr(no, "original_widget"):
+                                            st = no.original_widget
+                                        else:
+                                            if hasattr(no, "key_timeout"):
+                                                st = no
+                                    
+                                    if st and hasattr(st, "key_timeout") and hasattr(st, "keypress") and callable(st.keypress):
+                                        st.keypress(None, None)
+                                    
+                                    break
+
+                            pwi += 1
+                except Exception as e:
+                    pass
+
+            else:
+                # Original widget does not have a cursor, but may be selectable
+
+                # FIXME: Using ow.selectable() is bad because the original
+                # widget may be selectable because it's a container widget with
+                # a key-grabbing widget that is scrolled out of view.
+                # ow.selectable() returns True anyway because it doesn't know
+                # how we trimmed our canvas.
+                #
+                # To fix this, we need to resolve ow.focus and somehow
+                # ask canv whether it contains bits of the focused widget.  I
+                # can't see a way to do that.
+                if ow.selectable():
+                    self._forward_keypress = True
+                else:
+                    self._forward_keypress = False
+
+        return canv
+
+    def keypress(self, size, key):
+        # Maybe offer key to original widget
+        if self._forward_keypress or self.force_forward_keypress:
+            ow = self._original_widget
+            ow_size = self._get_original_widget_size(size)
+
+            # Remember previous cursor position if possible
+            if hasattr(ow, 'get_cursor_coords'):
+                self._old_cursor_coords = ow.get_cursor_coords(ow_size)
+
+            key = ow.keypress(ow_size, key)
+            if key is None:
+                return None
+
+        # Handle up/down, page up/down, etc
+        command_map = self._command_map
+        if command_map[key] == urwid.CURSOR_UP:
+            self._scroll_action = SCROLL_LINE_UP
+        elif command_map[key] == urwid.CURSOR_DOWN:
+            self._scroll_action = SCROLL_LINE_DOWN
+
+        elif command_map[key] == urwid.CURSOR_PAGE_UP:
+            self._scroll_action = SCROLL_PAGE_UP
+        elif command_map[key] == urwid.CURSOR_PAGE_DOWN:
+            self._scroll_action = SCROLL_PAGE_DOWN
+
+        elif command_map[key] == urwid.CURSOR_MAX_LEFT:   # 'home'
+            self._scroll_action = SCROLL_TO_TOP
+        elif command_map[key] == urwid.CURSOR_MAX_RIGHT:  # 'end'
+            self._scroll_action = SCROLL_TO_END
+
+        else:
+            return key
+
+        self._invalidate()
+
+    def mouse_event(self, size, event, button, col, row, focus):
+        ow = self._original_widget
+        if hasattr(ow, 'mouse_event'):
+            ow_size = self._get_original_widget_size(size)
+            row += self._trim_top
+            return ow.mouse_event(ow_size, event, button, col, row, focus)
+        else:
+            return False
+
+    def _adjust_trim_top(self, canv, size):
+        """Adjust self._trim_top according to self._scroll_action"""
+        action = self._scroll_action
+        self._scroll_action = None
+
+        maxcol, maxrow = size
+        trim_top = self._trim_top
+        canv_rows = canv.rows()
+
+        if trim_top < 0:
+            # Negative trim_top values use bottom of canvas as reference
+            trim_top = canv_rows - maxrow + trim_top + 1
+
+        if canv_rows <= maxrow:
+            self._trim_top = 0  # Reset scroll position
+            return
+
+        def ensure_bounds(new_trim_top):
+            return max(0, min(canv_rows - maxrow, new_trim_top))
+
+        if action == SCROLL_LINE_UP:
+            self._trim_top = ensure_bounds(trim_top - 1)
+        elif action == SCROLL_LINE_DOWN:
+            self._trim_top = ensure_bounds(trim_top + 1)
+
+        elif action == SCROLL_PAGE_UP:
+            self._trim_top = ensure_bounds(trim_top - maxrow + 1)
+        elif action == SCROLL_PAGE_DOWN:
+            self._trim_top = ensure_bounds(trim_top + maxrow - 1)
+
+        elif action == SCROLL_TO_TOP:
+            self._trim_top = 0
+        elif action == SCROLL_TO_END:
+            self._trim_top = canv_rows - maxrow
+
+        else:
+            self._trim_top = ensure_bounds(trim_top)
+
+        # If the cursor was moved by the most recent keypress, adjust trim_top
+        # so that the new cursor position is within the displayed canvas part.
+        # But don't do this if the cursor is at the top/bottom edge so we can still scroll out
+        if self._old_cursor_coords is not None and self._old_cursor_coords != canv.cursor and canv.cursor != None:
+            self._old_cursor_coords = None
+            curscol, cursrow = canv.cursor
+            if cursrow < self._trim_top:
+                self._trim_top = cursrow
+            elif cursrow >= self._trim_top + maxrow:
+                self._trim_top = max(0, cursrow - maxrow + 1)
+
+    def _get_original_widget_size(self, size):
+        ow = self._original_widget
+        sizing = ow.sizing()
+        if FLOW in sizing:
+            return (size[0],)
+        elif FIXED in sizing:
+            return ()
+
+    def get_scrollpos(self, size=None, focus=False):
+        """Current scrolling position
+
+        Lower limit is 0, upper limit is the maximum number of rows with the
+        given maxcol minus maxrow.
+
+        NOTE: The returned value may be too low or too high if the position has
+        changed but the widget wasn't rendered yet.
+        """
+        return self._trim_top
+
+    def set_scrollpos(self, position):
+        """Set scrolling position
+
+        If `position` is positive it is interpreted as lines from the top.
+        If `position` is negative it is interpreted as lines from the bottom.
+
+        Values that are too high or too low values are automatically adjusted
+        during rendering.
+        """
+        self._trim_top = int(position)
+        self._invalidate()
+
+    def rows_max(self, size=None, focus=False):
+        """Return the number of rows for `size`
+
+        If `size` is not given, the currently rendered number of rows is returned.
+        """
+        if size is not None:
+            ow = self._original_widget
+            ow_size = self._get_original_widget_size(size)
+            sizing = ow.sizing()
+            if FIXED in sizing:
+                self._rows_max_cached = ow.pack(ow_size, focus)[1]
+            elif FLOW in sizing:
+                self._rows_max_cached = ow.rows(ow_size, focus)
+            else:
+                raise RuntimeError('Not a flow/box widget: %r' % self._original_widget)
+        return self._rows_max_cached
+
+
+class ScrollBar(urwid.WidgetDecoration):
+    def sizing(self):
+        return frozenset((BOX,))
+
+    def selectable(self):
+        return True
+
+    def __init__(self, widget, thumb_char=u'\u2588', trough_char=' ',
+                 side=SCROLLBAR_RIGHT, width=1):
+        """Box widget that adds a scrollbar to `widget`
+
+        `widget` must be a box widget with the following methods:
+          - `get_scrollpos` takes the arguments `size` and `focus` and returns
+            the index of the first visible row.
+          - `set_scrollpos` (optional; needed for mouse click support) takes the
+            index of the first visible row.
+          - `rows_max` takes `size` and `focus` and returns the total number of
+            rows `widget` can render.
+
+        `thumb_char` is the character used for the scrollbar handle.
+        `trough_char` is used for the space above and below the handle.
+        `side` must be 'left' or 'right'.
+        `width` specifies the number of columns the scrollbar uses.
+        """
+        if BOX not in widget.sizing():
+            raise ValueError('Not a box widget: %r' % widget)
+        super().__init__(widget)
+        self._thumb_char = thumb_char
+        self._trough_char = trough_char
+        self.scrollbar_side = side
+        self.scrollbar_width = max(1, width)
+        self._original_widget_size = (0, 0)
+        self._sb_visible = False
+        self._sb_col_start = 0
+        self._sb_col_end = 0
+        self._sb_thumb_height = 1
+        self._sb_ow_rows_max = 0
+        self._sb_maxrow = 0
+        self._sb_dragging = False
+        self._sb_drag_offset = 0
+        self._accel = 1
+        self._sevt_interval = 0
+        self._sevt_start = 0
+        self._sfreq_deque = deque(maxlen=5)
+        
+        self._debug_hz = 0
+        self._debug_hz_smoothed = 0
+        self._debug_scroll_ms = 0
+        self._debug_scroll_started = 0
+
+    def render(self, size, focus=False):
+        maxcol, maxrow = size
+
+        sb_width = self._scrollbar_width
+        ow_size = (max(0, maxcol - sb_width), maxrow)
+        sb_width = maxcol - ow_size[0]
+
+        ow = self._original_widget
+        ow_base = self.scrolling_base_widget
+        ow_rows_max = ow_base.rows_max(size, focus)
+        if ow_rows_max <= maxrow:
+            # Canvas fits without scrolling - no scrollbar needed
+            self._original_widget_size = size
+            self._sb_visible = False
+            return ow.render(size, focus)
+        ow_rows_max = ow_base.rows_max(ow_size, focus)
+
+        ow_canv = ow.render(ow_size, focus)
+        
+        if self._debug_scroll_started:
+            total_ms = (time.time() - self._debug_scroll_started) * 1000
+            # RNS.log(f" accel={self._accel} , hz={RNS.prettyfrequency(self._debug_hz)} smoothed={RNS.prettyfrequency(self._debug_hz_smoothed)} scroll={self._debug_scroll_ms:.1f}ms total={total_ms:.1f}ms", RNS.LOG_DEBUG)
+            self._debug_scroll_started = 0
+        
+        self._original_widget_size = ow_size
+
+        pos = ow_base.get_scrollpos(ow_size, focus)
+        posmax = ow_rows_max - maxrow
+
+        # Clamp position to valid scroll range. The position
+        # reported by the underlying widget can transiently
+        # fall outside [0, posmax] if its layout state is
+        # inconsistent.
+        if pos < 0: pos = 0
+        elif pos > posmax: pos = posmax
+
+        # Thumb shrinks/grows according to the ratio of
+        # <number of visible lines> / <number of total lines>
+        thumb_weight = min(1, maxrow / max(1, ow_rows_max))
+        thumb_height = max(1, round(thumb_weight * maxrow))
+
+        # Thumb may only touch top/bottom if the first/last row is visible
+        top_weight = float(pos) / max(1, posmax)
+        top_height = int((maxrow - thumb_height) * top_weight)
+        if top_height == 0 and top_weight > 0:
+            top_height = 1
+
+        # Bottom part is remaining space. Guard against
+        # negative values so the scrollbar canvas can
+        # never be taller than maxrow.
+        bottom_height = maxrow - thumb_height - top_height
+        if bottom_height < 0: bottom_height = 0
+        assert thumb_height + top_height + bottom_height <= maxrow
+
+        # Create scrollbar canvas
+        # Creating SolidCanvases of correct height may result in "cviews do not
+        # fill gaps in shard_tail!" or "cviews overflow gaps in shard_tail!"
+        # exceptions. Stacking the same SolidCanvas is a workaround.
+        # https://github.com/urwid/urwid/issues/226#issuecomment-437176837
+        top = urwid.SolidCanvas(self._trough_char, sb_width, 1)
+        thumb = urwid.SolidCanvas(self._thumb_char, sb_width, 1)
+        bottom = urwid.SolidCanvas(self._trough_char, sb_width, 1)
+        sb_canv = urwid.CanvasCombine(
+            [(top, None, False)] * top_height +
+            [(thumb, None, False)] * thumb_height +
+            [(bottom, None, False)] * bottom_height,
+        )
+
+        combinelist = [(ow_canv, None, True, ow_size[0]),
+                       (sb_canv, None, False, sb_width)]
+
+        self._sb_visible = True
+        self._sb_thumb_height = thumb_height
+        self._sb_ow_rows_max = ow_rows_max
+        self._sb_maxrow = maxrow
+        if self._scrollbar_side != SCROLLBAR_LEFT:
+            self._sb_col_start = ow_size[0]
+            self._sb_col_end = ow_size[0] + sb_width
+            return urwid.CanvasJoin(combinelist)
+        else:
+            self._sb_col_start = 0
+            self._sb_col_end = sb_width
+            return urwid.CanvasJoin(reversed(combinelist))
+
+    @property
+    def scrollbar_width(self):
+        """Columns the scrollbar uses"""
+        return max(1, self._scrollbar_width)
+
+    @scrollbar_width.setter
+    def scrollbar_width(self, width):
+        self._scrollbar_width = max(1, int(width))
+        self._invalidate()
+
+    @property
+    def scrollbar_side(self):
+        """Where to display the scrollbar; must be 'left' or 'right'"""
+        return self._scrollbar_side
+
+    @scrollbar_side.setter
+    def scrollbar_side(self, side):
+        if side not in (SCROLLBAR_LEFT, SCROLLBAR_RIGHT):
+            raise ValueError('scrollbar_side must be "left" or "right", not %r' % side)
+        self._scrollbar_side = side
+        self._invalidate()
+
+    @property
+    def scrolling_base_widget(self):
+        """Nearest `original_widget` that is compatible with the scrolling API"""
+        def orig_iter(w):
+            while hasattr(w, 'original_widget'):
+                w = w.original_widget
+                yield w
+            yield w
+
+        def is_scrolling_widget(w):
+            return hasattr(w, 'get_scrollpos') and hasattr(w, 'rows_max')
+
+        for w in orig_iter(self):
+            if is_scrolling_widget(w):
+                return w
+        raise ValueError('Not compatible to be wrapped by ScrollBar: %r' % w)
+
+    def keypress(self, size, key):
+        return self._original_widget.keypress(self._original_widget_size, key)
+
+    def mouse_event(self, size, event, button, col, row, focus):
+        ow = self._original_widget
+        ow_size = self._original_widget_size
+        ow_base = self.scrolling_base_widget if hasattr(self, 'scrolling_base_widget') else ow
+        on_sb = (self._sb_visible
+                 and self._sb_col_start <= col < self._sb_col_end)
+
+        if self._sb_dragging:
+            if 'drag' in event:
+                self._set_scrollpos_from_row(row)
+                return True
+            if 'release' in event or 'press' in event:
+                self._sb_dragging = False
+                if 'release' in event:
+                    return True
+
+        # Begin a click/drag
+        if on_sb and 'press' in event and button == 1 and hasattr(ow_base, 'set_scrollpos'):
+            pos = ow_base.get_scrollpos(ow_size)
+            posmax = max(1, self._sb_ow_rows_max - self._sb_maxrow)
+            track_len = max(1, self._sb_maxrow - self._sb_thumb_height)
+            top_height = int(round(track_len * pos / posmax))
+            if top_height <= row < top_height + self._sb_thumb_height:
+                self._sb_drag_offset = row - top_height
+            else:
+                # jump
+                self._sb_drag_offset = self._sb_thumb_height // 2
+                self._set_scrollpos_from_row(row)
+            self._sb_dragging = True
+            return True
+
+        handled = False
+        if hasattr(ow, 'mouse_event'):
+            handled = ow.mouse_event(ow_size, event, button, col, row, focus)
+
+        if not handled and hasattr(ow, 'set_scrollpos'):
+            ow.force_cursor_update = True
+            ow.cursor_update_forced_at = time.time()
+            self._sevt_interval = time.time()-self._sevt_start
+            if self._sevt_interval > 0.3: self._sevt_start = 0; self._accel = 1; self._sfreq_deque.clear()
+            if self._sevt_start == 0: self._sevt_start = time.time()
+            if self._sevt_interval > 0:
+                sevt_freq = 1/self._sevt_interval
+                self._sfreq_deque.append(sevt_freq)
+                freq_smth = sum(self._sfreq_deque)/len(self._sfreq_deque)
+                self._debug_hz          = sevt_freq
+                self._debug_hz_smoothed = freq_smth
+                if len(self._sfreq_deque) >= self._sfreq_deque.maxlen:
+                    if freq_smth > 25.0:  self._accel = max(self._accel, 3)
+                    elif freq_smth > 12.0: self._accel = max(self._accel, 2)
+                    else:                 self._accel = max(self._accel, 1)
+                
+            step = self._accel
+            if button in (4, 5):
+                scroll_started = time.time()
+                pos = ow.get_scrollpos(ow_size)
+                if button == 4: newpos = max(0, pos - step)
+                else:           newpos = pos + step
+                ow.set_scrollpos(newpos)
+                self._debug_scroll_started = scroll_started
+                self._debug_scroll_ms      = (time.time() - scroll_started) * 1000
+                return True
+
+        return False
+
+    def _set_scrollpos_from_row(self, row):
+        ow_base = self.scrolling_base_widget
+        if not hasattr(ow_base, 'set_scrollpos'):
+            return
+        posmax = self._sb_ow_rows_max - self._sb_maxrow
+        if posmax <= 0:
+            return
+        track_len = max(1, self._sb_maxrow - self._sb_thumb_height)
+        top = max(0, min(track_len, row - self._sb_drag_offset))
+        new_pos = int(round(top * posmax / track_len))
+        new_pos = max(0, min(posmax, new_pos))
+        ow_base.set_scrollpos(new_pos)

nomadnet/vendor/__init__.py

@@ -1,5 +1,7 @@
 import os
 import glob
 
-modules = glob.glob(os.path.dirname(__file__)+"/*.py")
-__all__ = [ os.path.basename(f)[:-3] for f in modules if not f.endswith('__init__.py')]
+py_modules  = glob.glob(os.path.dirname(__file__)+"/*.py")
+pyc_modules = glob.glob(os.path.dirname(__file__)+"/*.pyc")
+modules     = py_modules+pyc_modules
+__all__ = list(set([os.path.basename(f).replace(".pyc", "").replace(".py", "") for f in modules if not (f.endswith("__init__.py") or f.endswith("__init__.pyc"))]))

nomadnet/vendor/additional_urwid_widgets/FormWidgets.py

@@ -0,0 +1,575 @@
+import urwid
+import nomadnet
+
+from nomadnet.ui.textui.ReadlineEdit import ReadlineMixin, ReadlineEdit
+
+def dropdown_indicator():
+    try:
+        return nomadnet.NomadNetworkApp.get_shared_instance().ui.glyphs["dropdown"]
+    except Exception:
+        return " ▾"
+
+class DialogLineBox(urwid.LineBox):
+    def __init__(self, body, parent=None, title="?"):
+        super().__init__(body, title=title)
+        self.parent = parent
+
+    def keypress(self, size, key):
+        if key == "esc":
+            if self.parent and hasattr(self.parent, "dismiss_dialog"):
+                self.parent.dismiss_dialog()
+            return None
+        return super().keypress(size, key)
+
+class Placeholder(ReadlineMixin, urwid.Edit):
+    def __init__(self, caption="", edit_text="", placeholder="", **kwargs):
+        super().__init__(caption, edit_text, **kwargs)
+        self.placeholder = placeholder
+
+    def render(self, size, focus=False):
+        if not self.edit_text and not focus:
+            placeholder_widget = urwid.Text(("placeholder", self.placeholder))
+            return placeholder_widget.render(size, focus)
+        else:
+            return super().render(size, focus)
+
+class Dropdown(urwid.WidgetWrap):
+    signals = ['change'] # emit for urwid.connect_signal fn
+
+    def __init__(self, label, options, default=None):
+        self.label = label
+        self.options = options
+        self.selected = default if default is not None else options[0]
+
+        self.main_text = f"{self.selected}"
+        self.main_button = urwid.SelectableIcon(self.main_text, 0)
+        self.main_button = urwid.AttrMap(self.main_button, "button_normal", "button_focus")
+
+        self.indicator_widget = urwid.Text(dropdown_indicator())
+        self.main_row = urwid.Columns([
+            ('pack', self.main_button),
+            ('pack', self.indicator_widget),
+            ('weight', 1, urwid.Text("")),
+        ])
+
+        self.option_widgets = []
+        for opt in options:
+            icon = urwid.SelectableIcon(opt, 0)
+            icon = urwid.AttrMap(icon, "list_normal", "list_focus")
+            self.option_widgets.append(icon)
+
+        self.options_walker = urwid.SimpleFocusListWalker(self.option_widgets)
+        self.options_listbox = urwid.ListBox(self.options_walker)
+        self.dropdown_box = None  # will be created on open_dropdown
+
+        self.pile = urwid.Pile([self.main_row])
+        self.dropdown_visible = False
+
+        super().__init__(self.pile)
+
+    def open_dropdown(self):
+        if not self.dropdown_visible:
+            height = len(self.options)
+            self.dropdown_box = urwid.BoxAdapter(self.options_listbox, height)
+            self.pile.contents.append((self.dropdown_box, self.pile.options()))
+            self.dropdown_visible = True
+            self.pile.focus_position = 1
+            self.options_walker.set_focus(0)
+
+    def close_dropdown(self):
+        if self.dropdown_visible:
+            self.pile.contents.pop()  # remove the dropdown_box
+            self.dropdown_visible = False
+            self.pile.focus_position = 0
+            self.dropdown_box = None
+
+    def keypress(self, size, key):
+        if not self.dropdown_visible:
+            if key == "enter":
+                self.open_dropdown()
+                return None
+            return self.main_button.keypress(size, key)
+        else:
+            if key == "enter":
+                focus_result = self.options_walker.get_focus()
+                if focus_result is not None:
+                    focus_widget = focus_result[0]
+                    new_val = focus_widget.base_widget.text
+                    old_val = self.selected
+                    self.selected = new_val
+                    self.main_button.base_widget.set_text(f"{self.selected}")
+
+                    if old_val != new_val:
+                        self._emit('change', new_val)
+
+                self.close_dropdown()
+                return None
+            return self.dropdown_box.keypress(size, key)
+
+    def get_value(self):
+        return self.selected
+
+class ValidationError(urwid.Text):
+    def __init__(self, message=""):
+        super().__init__(("error", message))
+
+class FormField:
+    def __init__(self, config_key, transform=None):
+        self.config_key = config_key
+        self.transform = transform or (lambda x: x)
+
+class FormEdit(Placeholder, FormField):
+    def __init__(self, config_key, caption="", edit_text="", placeholder="", validation_types=None, transform=None, **kwargs):
+        Placeholder.__init__(self, caption, edit_text, placeholder, **kwargs)
+        FormField.__init__(self, config_key, transform)
+        self.validation_types = validation_types or []
+        self.error_widget = urwid.Text("")
+        self.error = None
+
+    def get_value(self):
+        return self.transform(self.edit_text.strip())
+
+    def validate(self):
+        value = self.edit_text.strip()
+        self.error = None
+
+        for validation in self.validation_types:
+            if validation == "required":
+                if not value:
+                    self.error = "This field is required"
+                    break
+            elif validation == "number":
+                try:
+                    if value:
+                        int(value)
+                except ValueError:
+                    self.error = "This field must be a whole number"
+                    break
+            elif validation == "stamp_cost":
+                if value and value.lower() != "none":
+                    try:
+                        if not 1 <= int(value) <= 255:
+                            raise ValueError
+                    except ValueError:
+                        self.error = "This field must be 1-255 or None"
+                        break
+            elif validation == "float":
+                try:
+                    if value:
+                        float(value)
+                except ValueError:
+                    self.error = "This field must be decimal number"
+                    break
+
+        self.error_widget.set_text(("error", self.error or ""))
+        return self.error is None
+
+class FormCheckbox(urwid.CheckBox, FormField):
+    def __init__(self, config_key, label="", state=False, validation_types=None, transform=None, **kwargs):
+        urwid.CheckBox.__init__(self, label, state, **kwargs)
+        FormField.__init__(self, config_key, transform)
+        self.validation_types = validation_types or []
+        self.error_widget = urwid.Text("")
+        self.error = None
+
+    def get_value(self):
+        return self.transform(self.get_state())
+
+    def validate(self):
+
+        value = self.get_state()
+        self.error = None
+
+        for validation in self.validation_types:
+            if validation == "required":
+                if not value:
+                    self.error = "This field is required"
+                    break
+
+        self.error_widget.set_text(("error", self.error or ""))
+        return self.error is None
+
+class FormDropdown(Dropdown, FormField):
+    signals = ['change']
+
+    def __init__(self, config_key, label, options, default=None, validation_types=None, transform=None):
+        self.options = [str(opt) for opt in options]
+
+        if default is not None:
+            default_str = str(default)
+            if default_str in self.options:
+                default = default_str
+            elif transform:
+                try:
+                    default_transformed = transform(default_str)
+                    for opt in self.options:
+                        if transform(opt) == default_transformed:
+                            default = opt
+                            break
+                except:
+                    default = self.options[0]
+            else:
+                default = self.options[0]
+        else:
+            default = self.options[0]
+
+        Dropdown.__init__(self, label, self.options, default)
+        FormField.__init__(self, config_key, transform)
+
+        self.validation_types = validation_types or []
+        self.error_widget = urwid.Text("")
+        self.error = None
+
+        if hasattr(self, 'main_button'):
+            self.main_button.base_widget.set_text(str(default))
+
+    def get_value(self):
+        return self.transform(self.selected)
+
+    def validate(self):
+        value = self.get_value()
+        self.error = None
+
+        for validation in self.validation_types:
+            if validation == "required":
+                if not value:
+                    self.error = "This field is required"
+                    break
+
+        self.error_widget.set_text(("error", self.error or ""))
+        return self.error is None
+
+    def open_dropdown(self):
+        if not self.dropdown_visible:
+            super().open_dropdown()
+            try:
+                current_index = self.options.index(self.selected)
+                self.options_walker.set_focus(current_index)
+            except ValueError:
+                pass
+
+class FormMultiList(urwid.Pile, FormField):
+    def __init__(self, config_key, placeholder="", validation_types=None, transform=None, **kwargs):
+        self.entries = []
+        self.error_widget = urwid.Text("")
+        self.error = None
+        self.placeholder = placeholder
+        self.validation_types = validation_types or []
+
+        first_entry = self.create_entry_row()
+        self.entries.append(first_entry)
+
+        self.add_button = urwid.Button("+ Add Another", on_press=self.add_entry)
+        add_button_padded = urwid.Padding(self.add_button, left=2, right=2)
+
+        pile_widgets = [first_entry, add_button_padded]
+        urwid.Pile.__init__(self, pile_widgets)
+        FormField.__init__(self, config_key, transform)
+
+    def create_entry_row(self, text=""):
+        edit = ReadlineEdit("", text)
+        entry_row = urwid.Columns([
+            ('weight', 1, edit),
+            (3, urwid.Button("×", on_press=lambda button: self.remove_entry(button, entry_row))),
+        ])
+        return entry_row
+
+    def set_value(self, value):
+        self.entries = []
+        if not value:
+            self.entries.append(self.create_entry_row())
+        else:
+            if isinstance(value, str):
+                value = [value]
+            for item in value:
+                self.entries.append(self.create_entry_row(str(item)))
+        self.contents = [(w, self.options()) for w in self.get_pile_widgets()]
+
+    def remove_entry(self, button, entry_row):
+        if len(self.entries) > 1:
+            self.entries.remove(entry_row)
+            self.contents = [(w, self.options()) for w in self.get_pile_widgets()]
+
+    def add_entry(self, button):
+        new_entry = self.create_entry_row()
+        self.entries.append(new_entry)
+
+        self.contents = [(w, self.options()) for w in self.get_pile_widgets()]
+
+    def get_pile_widgets(self):
+        return self.entries + [urwid.Padding(self.add_button, left=2, right=2)]
+
+    def get_value(self):
+        values = []
+        for entry in self.entries:
+            edit_widget = entry.contents[0][0]
+            value = edit_widget.edit_text.strip()
+            if value:
+                values.append(value)
+        return self.transform(values)
+
+    def validate(self):
+        values = self.get_value()
+        self.error = None
+
+        for validation in self.validation_types:
+            if validation == "required" and not values:
+                self.error = "At least one entry is required"
+                break
+
+        self.error_widget.set_text(("error", self.error or ""))
+        return self.error is None
+
+
+class FormMultiTable(urwid.Pile, FormField):
+    def __init__(self, config_key, fields, validation_types=None, transform=None, **kwargs):
+        self.entries = []
+        self.fields = fields
+        self.error_widget = urwid.Text("")
+        self.error = None
+        self.validation_types = validation_types or []
+
+        header_columns = [('weight', 3, urwid.Text(("list_focus", "Name")))]
+        for field_key, field_config in self.fields.items():
+            header_columns.append(('weight', 2, urwid.Text(("list_focus", field_config.get("label", field_key)))))
+        header_columns.append((4, urwid.Text(("list_focus", ""))))
+
+        self.header_row = urwid.Columns(header_columns)
+
+        first_entry = self.create_entry_row()
+        self.entries.append(first_entry)
+
+        self.add_button = urwid.Button("+ Add ", on_press=self.add_entry)
+        add_button_padded = urwid.Padding(self.add_button, left=2, right=2)
+
+        pile_widgets = [
+            self.header_row,
+            urwid.Divider("-"),
+            first_entry,
+            add_button_padded
+        ]
+
+        urwid.Pile.__init__(self, pile_widgets)
+        FormField.__init__(self, config_key, transform)
+
+    def create_entry_row(self, name="", values=None):
+        if values is None:
+            values = {}
+
+        name_edit = ReadlineEdit("", name)
+
+        columns = [('weight', 3, name_edit)]
+
+        field_widgets = {}
+        for field_key, field_config in self.fields.items():
+            field_value = values.get(field_key, "")
+
+            if field_config.get("type") == "checkbox":
+                widget = urwid.CheckBox("", state=bool(field_value))
+            elif field_config.get("type") == "dropdown":
+                # TODO: dropdown in MultiTable
+                widget = ReadlineEdit("", str(field_value))
+            else:
+                widget = ReadlineEdit("", str(field_value))
+
+            field_widgets[field_key] = widget
+            columns.append(('weight', 2, widget))
+
+        remove_button = urwid.Button("×", on_press=lambda button: self.remove_entry(button, entry_row))
+        columns.append((4, remove_button))
+
+        entry_row = urwid.Columns(columns)
+        entry_row.name_edit = name_edit
+        entry_row.field_widgets = field_widgets
+
+        return entry_row
+
+    def remove_entry(self, button, entry_row):
+        if len(self.entries) > 1:
+            self.entries.remove(entry_row)
+            self.contents = [(w, self.options()) for w in self.get_pile_widgets()]
+
+    def add_entry(self, button):
+        new_entry = self.create_entry_row()
+        self.entries.append(new_entry)
+
+        self.contents = [(w, self.options()) for w in self.get_pile_widgets()]
+
+    def get_pile_widgets(self):
+        return [
+            self.header_row,
+            urwid.Divider("-")
+        ] + self.entries + [
+            urwid.Padding(self.add_button, left=2, right=2)
+        ]
+
+    def get_value(self):
+        values = {}
+        for entry in self.entries:
+            name = entry.name_edit.edit_text.strip()
+            if name:
+                subinterface = {}
+                subinterface["interface_enabled"] = True
+
+                for field_key, widget in entry.field_widgets.items():
+                    field_config = self.fields.get(field_key, {})
+
+                    if hasattr(widget, "get_state"):
+                        value = widget.get_state()
+                    elif hasattr(widget, "edit_text"):
+                        value = widget.edit_text.strip()
+
+                        transform = field_config.get("transform")
+                        if transform and value:
+                            try:
+                                value = transform(value)
+                            except (ValueError, TypeError):
+                                value = ""
+
+                    if value:
+                        subinterface[field_key] = value
+
+                values[name] = subinterface
+
+        return self.transform(values) if self.transform else values
+
+    def set_value(self, value):
+        self.entries = []
+
+        if not value:
+            self.entries.append(self.create_entry_row())
+        else:
+            for name, config in value.items():
+                self.entries.append(self.create_entry_row(name=name, values=config))
+
+        self.contents = [(w, self.options()) for w in self.get_pile_widgets()]
+
+    def validate(self):
+        values = self.get_value()
+        self.error = None
+
+        for validation in self.validation_types:
+            if validation == "required" and not values:
+                self.error = "At least one subinterface is required"
+                break
+
+        self.error_widget.set_text(("error", self.error or ""))
+        return self.error is None
+
+
+class FormKeyValuePairs(urwid.Pile, FormField):
+    def __init__(self, config_key, validation_types=None, transform=None, **kwargs):
+        self.entries = []
+        self.error_widget = urwid.Text("")
+        self.error = None
+        self.validation_types = validation_types or []
+
+        header_columns = [
+            ('weight', 1, urwid.AttrMap(urwid.Text("Parameter Key"), "multitable_header")),
+            ('weight', 1, urwid.AttrMap(urwid.Text("Parameter Value"), "multitable_header")),
+            (4, urwid.AttrMap(urwid.Text("Action"), "multitable_header"))
+        ]
+
+        self.header_row = urwid.AttrMap(urwid.Columns(header_columns), "multitable_header")
+
+        first_entry = self.create_entry_row()
+        self.entries.append(first_entry)
+
+        self.add_button = urwid.Button("+ Add Parameter", on_press=self.add_entry)
+        add_button_padded = urwid.Padding(self.add_button, left=2, right=2)
+
+        pile_widgets = [
+            self.header_row,
+            urwid.Divider("-"),
+            first_entry,
+            add_button_padded
+        ]
+
+        urwid.Pile.__init__(self, pile_widgets)
+        FormField.__init__(self, config_key, transform)
+
+    def create_entry_row(self, key="", value=""):
+        key_edit = ReadlineEdit("", key)
+        value_edit = ReadlineEdit("", value)
+
+        remove_button = urwid.Button("×", on_press=lambda button: self.remove_entry(button, entry_row))
+
+        entry_row = urwid.Columns([
+            ('weight', 1, key_edit),
+            ('weight', 1, value_edit),
+            (4, remove_button)
+        ])
+
+        entry_row.key_edit = key_edit
+        entry_row.value_edit = value_edit
+
+        return entry_row
+
+    def remove_entry(self, button, entry_row):
+        if len(self.entries) > 1:
+            self.entries.remove(entry_row)
+            self.contents = [(w, self.options()) for w in self.get_pile_widgets()]
+
+    def add_entry(self, button):
+        new_entry = self.create_entry_row()
+        self.entries.append(new_entry)
+
+        self.contents = [(w, self.options()) for w in self.get_pile_widgets()]
+
+    def get_pile_widgets(self):
+        return [
+            self.header_row,
+            urwid.Divider("-")
+        ] + self.entries + [
+            urwid.Padding(self.add_button, left=2, right=2)
+        ]
+
+    def get_value(self):
+        values = {}
+        for entry in self.entries:
+            key = entry.key_edit.edit_text.strip()
+            value = entry.value_edit.edit_text.strip()
+
+            if key:
+                if value.isdigit():
+                    values[key] = int(value)
+                elif value.replace('.', '', 1).isdigit() and value.count('.') <= 1:
+                    values[key] = float(value)
+                elif value.lower() == 'true':
+                    values[key] = True
+                elif value.lower() == 'false':
+                    values[key] = False
+                else:
+                    values[key] = value
+
+        return self.transform(values) if self.transform else values
+
+    def set_value(self, value):
+        self.entries = []
+
+        if not value or not isinstance(value, dict):
+            self.entries.append(self.create_entry_row())
+        else:
+            for key, val in value.items():
+                self.entries.append(self.create_entry_row(key=key, value=str(val)))
+
+        self.contents = [(w, self.options()) for w in self.get_pile_widgets()]
+
+    def validate(self):
+        values = self.get_value()
+        self.error = None
+
+        keys = [entry.key_edit.edit_text.strip() for entry in self.entries
+                if entry.key_edit.edit_text.strip()]
+        if len(keys) != len(set(keys)):
+            self.error = "Duplicate keys are not allowed"
+            self.error_widget.set_text(("error", self.error))
+            return False
+
+        for validation in self.validation_types:
+            if validation == "required" and not values:
+                self.error = "Atleast one parameter is required"
+                break
+
+        self.error_widget.set_text(("error", self.error or ""))
+        return self.error is None

nomadnet/vendor/additional_urwid_widgets/widgets/indicative_listbox.py

@@ -257,10 +257,15 @@ class IndicativeListBox(urwid.WidgetWrap):
         # An event is changed to a modified one ('mouse press' => 'ctrl mouse press'). This prevents the widget from responding
         # when mouse buttons are also used to navigate between widgets.
         if event == self._modifier_key.prepend_to("mouse press"):
-            # Store the focus position before passing the input to the contained list box. That way, it can be compared with the 
+            # Store the focus position before passing the input to the contained list box. That way, it can be compared with the
             # position after the input is processed. If the list box body is empty, store None.
             focus_position_before_input = self.get_selected_position()
-            
+
+            # Also remember the currently selected item itself. A button 1 press makes the contained list box move the focus, but
+            # the off focus highlighting is only (un)done during 'render()' while the widget has the focus. When the selection is
+            # changed by mouse while the widget is unfocused, the highlight would otherwise stay stuck on the previous item.
+            item_before_input = self.get_selected_item()
+
             # left mouse button, if not top bar or bottom bar.
             if (button == 1.0) and (topBar_rows <= row < (size[1] - bottomBar_rows)):
                 # Because 'row' includes the top bar, the offset must be substracted before passing it to the contained list box.
@@ -269,18 +274,29 @@ class IndicativeListBox(urwid.WidgetWrap):
             
             # mousewheel up
             elif button == 4.0:
-                was_handeled = self._pass_key_to_contained_listbox(modified_size, "page up")
+                # was_handeled = self._pass_key_to_contained_listbox(modified_size, "page up")
+                was_handeled = self._pass_key_to_contained_listbox(modified_size, "up")
                 
             # mousewheel down
             elif button == 5.0:
-                was_handeled = self._pass_key_to_contained_listbox(modified_size, "page down")
+                # was_handeled = self._pass_key_to_contained_listbox(modified_size, "page down")
+                was_handeled = self._pass_key_to_contained_listbox(modified_size, "down")
                 
             focus_position_after_input = self.get_selected_position()
-            
-            # If the focus position has changed, execute the hook (if existing).
-            if (focus_position_before_input != focus_position_after_input) and (self.on_selection_change is not None):
-                self.on_selection_change(focus_position_before_input,
-                                         focus_position_after_input)
+
+            if focus_position_before_input != focus_position_after_input:
+                # The mouse moved the selection. If off focus highlighting is currently applied (the widget is unfocused), restore
+                # the previously selected item and clear the stored state, so the next 'render()' re-applies the highlight to the
+                # newly selected item instead of leaving it stuck on the old one.
+                if (not self._last_focus_state) and (self._original_item_attr_map is not None) and (item_before_input is not None):
+                    item_before_input.set_attr_map(self._original_item_attr_map)
+                    self._original_item_attr_map = None
+                    self._last_focus_state = None
+
+                # If the focus position has changed, execute the hook (if existing).
+                if self.on_selection_change is not None:
+                    self.on_selection_change(focus_position_before_input,
+                                             focus_position_after_input)
         
         return was_handeled
     

nomadnet/vendor/cbor.py

@@ -0,0 +1,430 @@
+# https://github.com/brianolson/cbor_py
+# Copyright 2014-2015 Brian Olson
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import re
+import struct
+from io import BytesIO
+
+
+CBOR_TYPE_MASK = 0xE0  # top 3 bits
+CBOR_INFO_BITS = 0x1F  # low 5 bits
+
+
+CBOR_UINT    = 0x00
+CBOR_NEGINT  = 0x20
+CBOR_BYTES   = 0x40
+CBOR_TEXT    = 0x60
+CBOR_ARRAY   = 0x80
+CBOR_MAP     = 0xA0
+CBOR_TAG     = 0xC0
+CBOR_7       = 0xE0  # float and other types
+
+CBOR_UINT8_FOLLOWS  = 24  # 0x18
+CBOR_UINT16_FOLLOWS = 25  # 0x19
+CBOR_UINT32_FOLLOWS = 26  # 0x1a
+CBOR_UINT64_FOLLOWS = 27  # 0x1b
+CBOR_VAR_FOLLOWS    = 31  # 0x1f
+
+CBOR_BREAK  = 0xFF
+
+CBOR_FALSE  = (CBOR_7 | 20)
+CBOR_TRUE   = (CBOR_7 | 21)
+CBOR_NULL   = (CBOR_7 | 22)
+CBOR_UNDEFINED   = (CBOR_7 | 23)  # js 'undefined' value
+
+CBOR_FLOAT16 = (CBOR_7 | 25)
+CBOR_FLOAT32 = (CBOR_7 | 26)
+CBOR_FLOAT64 = (CBOR_7 | 27)
+
+CBOR_TAG_DATE_STRING = 0  # RFC3339
+CBOR_TAG_DATE_ARRAY = 1   # any number, seconds since 1970-01-01T00:00:00 UTC
+CBOR_TAG_BIGNUM = 2       # big-endian byte string follows
+CBOR_TAG_NEGBIGNUM = 3    # big-endian byte string follows
+CBOR_TAG_DECIMAL = 4
+CBOR_TAG_BIGFLOAT = 5
+CBOR_TAG_BASE64URL = 21
+CBOR_TAG_BASE64 = 22
+CBOR_TAG_BASE16 = 23
+CBOR_TAG_CBOR = 24
+
+CBOR_TAG_URI = 32
+CBOR_TAG_REGEX = 35
+CBOR_TAG_MIME = 36
+CBOR_TAG_CBOR_FILEHEADER = 55799  # 0xd9d9f7
+
+_CBOR_TAG_BIGNUM_BYTES = struct.pack('B', CBOR_TAG | CBOR_TAG_BIGNUM)
+_CBOR_TAG_NEGBIGNUM_BYTES = struct.pack('B', CBOR_TAG | CBOR_TAG_NEGBIGNUM)
+
+
+def _dumps_bignum_to_bytearray(val):
+    out = []
+    while val > 0:
+        out.insert(0, val & 0x0ff)
+        val = val >> 8
+    return bytes(out)
+
+
+def dumps_int(val):
+    "return bytes representing int val in CBOR"
+    if val >= 0:
+        if val <= 23:
+            return struct.pack('B', val)
+        if val <= 0x0ff:
+            return struct.pack('BB', CBOR_UINT8_FOLLOWS, val)
+        if val <= 0x0ffff:
+            return struct.pack('!BH', CBOR_UINT16_FOLLOWS, val)
+        if val <= 0x0ffffffff:
+            return struct.pack('!BI', CBOR_UINT32_FOLLOWS, val)
+        if val <= 0x0ffffffffffffffff:
+            return struct.pack('!BQ', CBOR_UINT64_FOLLOWS, val)
+        outb = _dumps_bignum_to_bytearray(val)
+        return _CBOR_TAG_BIGNUM_BYTES + _encode_type_num(CBOR_BYTES, len(outb)) + outb
+    val = -1 - val
+    return _encode_type_num(CBOR_NEGINT, val)
+
+
+def dumps_float(val):
+    return struct.pack("!Bd", CBOR_FLOAT64, val)
+
+
+def _encode_type_num(cbor_type, val):
+    """For some CBOR primary type [0..7] and an auxiliary unsigned number,
+    return CBOR encoded bytes."""
+    assert val >= 0
+    if val <= 23:
+        return struct.pack('B', cbor_type | val)
+    if val <= 0x0ff:
+        return struct.pack('BB', cbor_type | CBOR_UINT8_FOLLOWS, val)
+    if val <= 0x0ffff:
+        return struct.pack('!BH', cbor_type | CBOR_UINT16_FOLLOWS, val)
+    if val <= 0x0ffffffff:
+        return struct.pack('!BI', cbor_type | CBOR_UINT32_FOLLOWS, val)
+    if (((cbor_type == CBOR_NEGINT) and (val <= 0x07fffffffffffffff)) or
+        ((cbor_type != CBOR_NEGINT) and (val <= 0x0ffffffffffffffff))):
+        return struct.pack('!BQ', cbor_type | CBOR_UINT64_FOLLOWS, val)
+    if cbor_type != CBOR_NEGINT:
+        raise Exception("value too big for CBOR unsigned number: {0!r}".format(val))
+    outb = _dumps_bignum_to_bytearray(val)
+    return _CBOR_TAG_NEGBIGNUM_BYTES + _encode_type_num(CBOR_BYTES, len(outb)) + outb
+
+
+def dumps_string(val, is_text=None, is_bytes=None):
+    if isinstance(val, str):
+        val = val.encode('utf8')
+        is_text = True
+        is_bytes = False
+    if (is_bytes) or not (is_text == True):
+        return _encode_type_num(CBOR_BYTES, len(val)) + val
+    return _encode_type_num(CBOR_TEXT, len(val)) + val
+
+
+def dumps_array(arr, sort_keys=False):
+    head = _encode_type_num(CBOR_ARRAY, len(arr))
+    parts = [dumps(x, sort_keys=sort_keys) for x in arr]
+    return head + b''.join(parts)
+
+
+def dumps_dict(d, sort_keys=False):
+    head = _encode_type_num(CBOR_MAP, len(d))
+    parts = [head]
+    if sort_keys:
+        for k in sorted(d.keys()):
+            v = d[k]
+            parts.append(dumps(k, sort_keys=sort_keys))
+            parts.append(dumps(v, sort_keys=sort_keys))
+    else:
+        for k, v in d.items():
+            parts.append(dumps(k, sort_keys=sort_keys))
+            parts.append(dumps(v, sort_keys=sort_keys))
+    return b''.join(parts)
+
+
+def dumps_bool(b):
+    if b:
+        return struct.pack('B', CBOR_TRUE)
+    return struct.pack('B', CBOR_FALSE)
+
+
+def dumps_tag(t, sort_keys=False):
+    return _encode_type_num(CBOR_TAG, t.tag) + dumps(t.value, sort_keys=sort_keys)
+
+
+def dumps(ob, sort_keys=False):
+    if ob is None:
+        return struct.pack('B', CBOR_NULL)
+    if isinstance(ob, bool):
+        return dumps_bool(ob)
+    if isinstance(ob, (str, bytes, bytearray)):
+        if isinstance(ob, bytearray):
+            ob = bytes(ob)
+        return dumps_string(ob)
+    if isinstance(ob, (list, tuple)):
+        return dumps_array(ob, sort_keys=sort_keys)
+    if isinstance(ob, dict):
+        return dumps_dict(ob, sort_keys=sort_keys)
+    if isinstance(ob, float):
+        return dumps_float(ob)
+    if isinstance(ob, int):
+        return dumps_int(ob)
+    if isinstance(ob, Tag):
+        return dumps_tag(ob, sort_keys=sort_keys)
+    raise Exception("don't know how to cbor serialize object of type %s" % type(ob))
+
+
+def dump(obj, fp, sort_keys=False):
+    """obj: Python object to serialize. fp: file-like object capable of .write(bytes)."""
+    blob = dumps(obj, sort_keys=sort_keys)
+    fp.write(blob)
+
+
+class Tag(object):
+    def __init__(self, tag=None, value=None):
+        self.tag = tag
+        self.value = value
+
+    def __repr__(self):
+        return "Tag({0!r}, {1!r})".format(self.tag, self.value)
+
+    def __eq__(self, other):
+        if not isinstance(other, Tag):
+            return False
+        return (self.tag == other.tag) and (self.value == other.value)
+
+    def __hash__(self):
+        return hash((self.tag, self.value)) if not isinstance(self.value, (list, dict, bytearray)) else id(self)
+
+
+def loads(data):
+    """Parse CBOR bytes and return Python objects."""
+    if data is None:
+        raise ValueError("got None for buffer to decode in loads")
+    if isinstance(data, (bytes, bytearray, memoryview)):
+        fp = BytesIO(bytes(data))
+    else:
+        fp = data
+    return _loads(fp)[0]
+
+
+def load(fp):
+    """Parse and return object from fp, a file-like object supporting .read(n)."""
+    return _loads(fp)[0]
+
+
+_MAX_DEPTH = 100
+
+
+def _tag_aux(fp, tb):
+    bytes_read = 1
+    tag = tb & CBOR_TYPE_MASK
+    tag_aux = tb & CBOR_INFO_BITS
+    if tag_aux <= 23:
+        aux = tag_aux
+    elif tag_aux == CBOR_UINT8_FOLLOWS:
+        data = fp.read(1)
+        aux = struct.unpack_from("!B", data, 0)[0]
+        bytes_read += 1
+    elif tag_aux == CBOR_UINT16_FOLLOWS:
+        data = fp.read(2)
+        aux = struct.unpack_from("!H", data, 0)[0]
+        bytes_read += 2
+    elif tag_aux == CBOR_UINT32_FOLLOWS:
+        data = fp.read(4)
+        aux = struct.unpack_from("!I", data, 0)[0]
+        bytes_read += 4
+    elif tag_aux == CBOR_UINT64_FOLLOWS:
+        data = fp.read(8)
+        aux = struct.unpack_from("!Q", data, 0)[0]
+        bytes_read += 8
+    else:
+        assert tag_aux == CBOR_VAR_FOLLOWS, "bogus tag {0:02x}".format(tb)
+        aux = None
+
+    return tag, tag_aux, aux, bytes_read
+
+
+def _read_byte(fp):
+    tb = fp.read(1)
+    if len(tb) == 0:
+        raise EOFError()
+    return tb[0]
+
+
+def _loads_var_array(fp, limit, depth, returntags, bytes_read):
+    ob = []
+    tb = _read_byte(fp)
+    while tb != CBOR_BREAK:
+        (subob, sub_len) = _loads_tb(fp, tb, limit, depth, returntags)
+        bytes_read += 1 + sub_len
+        ob.append(subob)
+        tb = _read_byte(fp)
+    return (ob, bytes_read + 1)
+
+
+def _loads_var_map(fp, limit, depth, returntags, bytes_read):
+    ob = {}
+    tb = _read_byte(fp)
+    while tb != CBOR_BREAK:
+        (subk, sub_len) = _loads_tb(fp, tb, limit, depth, returntags)
+        bytes_read += 1 + sub_len
+        (subv, sub_len) = _loads(fp, limit, depth, returntags)
+        bytes_read += sub_len
+        ob[subk] = subv
+        tb = _read_byte(fp)
+    return (ob, bytes_read + 1)
+
+
+def _loads_array(fp, limit, depth, returntags, aux, bytes_read):
+    ob = []
+    for _ in range(aux):
+        subob, subpos = _loads(fp, limit, depth, returntags)
+        bytes_read += subpos
+        ob.append(subob)
+    return ob, bytes_read
+
+
+def _loads_map(fp, limit, depth, returntags, aux, bytes_read):
+    ob = {}
+    for _ in range(aux):
+        subk, subpos = _loads(fp, limit, depth, returntags)
+        bytes_read += subpos
+        subv, subpos = _loads(fp, limit, depth, returntags)
+        bytes_read += subpos
+        ob[subk] = subv
+    return ob, bytes_read
+
+
+def _loads(fp, limit=None, depth=0, returntags=False):
+    "return (object, bytes read)"
+    if depth > _MAX_DEPTH:
+        raise Exception("hit CBOR loads recursion depth limit")
+    tb = _read_byte(fp)
+    return _loads_tb(fp, tb, limit, depth, returntags)
+
+
+def _loads_tb(fp, tb, limit=None, depth=0, returntags=False):
+    # Some special cases of CBOR_7 best handled by special struct.unpack logic here
+    if tb == CBOR_FLOAT16:
+        data = fp.read(2)
+        hibyte, lowbyte = struct.unpack_from("BB", data, 0)
+        exp = (hibyte >> 2) & 0x1F
+        mant = ((hibyte & 0x03) << 8) | lowbyte
+        if exp == 0:
+            val = mant * (2.0 ** -24)
+        elif exp == 31:
+            val = float('Inf') if mant == 0 else float('NaN')
+        else:
+            val = (mant + 1024.0) * (2 ** (exp - 25))
+        if hibyte & 0x80:
+            val = -1.0 * val
+        return (val, 3)
+    elif tb == CBOR_FLOAT32:
+        data = fp.read(4)
+        pf = struct.unpack_from("!f", data, 0)
+        return (pf[0], 5)
+    elif tb == CBOR_FLOAT64:
+        data = fp.read(8)
+        pf = struct.unpack_from("!d", data, 0)
+        return (pf[0], 9)
+
+    tag, tag_aux, aux, bytes_read = _tag_aux(fp, tb)
+
+    if tag == CBOR_UINT:
+        return (aux, bytes_read)
+    elif tag == CBOR_NEGINT:
+        return (-1 - aux, bytes_read)
+    elif tag == CBOR_BYTES:
+        ob, subpos = loads_bytes(fp, aux)
+        return (ob, bytes_read + subpos)
+    elif tag == CBOR_TEXT:
+        raw, subpos = loads_bytes(fp, aux, btag=CBOR_TEXT)
+        ob = raw.decode('utf8')
+        return (ob, bytes_read + subpos)
+    elif tag == CBOR_ARRAY:
+        if aux is None:
+            return _loads_var_array(fp, limit, depth, returntags, bytes_read)
+        return _loads_array(fp, limit, depth, returntags, aux, bytes_read)
+    elif tag == CBOR_MAP:
+        if aux is None:
+            return _loads_var_map(fp, limit, depth, returntags, bytes_read)
+        return _loads_map(fp, limit, depth, returntags, aux, bytes_read)
+    elif tag == CBOR_TAG:
+        ob, subpos = _loads(fp, limit, depth + 1, returntags)
+        bytes_read += subpos
+        if returntags:
+            ob = Tag(aux, ob)
+        else:
+            ob = tagify(ob, aux)
+        return ob, bytes_read
+    elif tag == CBOR_7:
+        if tb == CBOR_TRUE:
+            return (True, bytes_read)
+        if tb == CBOR_FALSE:
+            return (False, bytes_read)
+        if tb == CBOR_NULL:
+            return (None, bytes_read)
+        if tb == CBOR_UNDEFINED:
+            return (None, bytes_read)
+        raise ValueError("unknown cbor tag 7 byte: {:02x}".format(tb))
+
+
+def loads_bytes(fp, aux, btag=CBOR_BYTES):
+    if aux is not None:
+        ob = fp.read(aux)
+        return (ob, aux)
+    chunklist = []
+    total_bytes_read = 0
+    while True:
+        tb = fp.read(1)[0]
+        if tb == CBOR_BREAK:
+            total_bytes_read += 1
+            break
+        tag, tag_aux, aux, bytes_read = _tag_aux(fp, tb)
+        assert tag == btag, 'variable length value contains unexpected component'
+        ob = fp.read(aux)
+        chunklist.append(ob)
+        total_bytes_read += bytes_read + aux
+    return (b''.join(chunklist), total_bytes_read)
+
+
+def _bytes_to_biguint(bs):
+    out = 0
+    for ch in bs:
+        out = out << 8
+        out = out | ch
+    return out
+
+
+def tagify(ob, aux):
+    if aux == CBOR_TAG_DATE_STRING:
+        # RFC3339 date string parsing not implemented; return as Tag.
+        return Tag(aux, ob)
+    if aux == CBOR_TAG_DATE_ARRAY:
+        return datetime.datetime.fromtimestamp(ob, tz=datetime.timezone.utc)
+    if aux == CBOR_TAG_BIGNUM:
+        return _bytes_to_biguint(ob)
+    if aux == CBOR_TAG_NEGBIGNUM:
+        return -1 - _bytes_to_biguint(ob)
+    if aux == CBOR_TAG_REGEX:
+        return re.compile(ob)
+    return Tag(aux, ob)
+
+
+def encode(obj, sort_keys=False):
+    return dumps(obj, sort_keys=sort_keys)
+
+
+def decode(data):
+    return loads(data)

nomadnet/vendor/quotes.py

@@ -1,4 +1,7 @@
 quotes = [
     ("I want the wisdom that wise men revere. I want more.", "Faithless"),
-    ("That's enough entropy for you my friend", "Unknown")
+    ("That's enough entropy for you my friend", "Unknown"),
+    ("Any time two people connect online, it's financed by a third person who believes they can manipulate the first two", "Jaron Lanier"),
+    ("The landscape of the future is set, but how one will march across it is not determined", "Terence McKenna")
+    ("Freedom originates in the division of power, despotism in its concentration.", "John Acton")
 ]

setup.py

@@ -1,3 +1,4 @@
+import sys
 import setuptools
 
 exec(open("nomadnet/_version.py", "r").read())
@@ -5,6 +6,17 @@ exec(open("nomadnet/_version.py", "r").read())
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
+if "--getversion" in sys.argv:
+    print(__version__, end="")
+    exit(0)
+
+package_data = {
+"": [
+    "examples/messageboard/*",
+    "ui/textui/images/demo.webp",
+    ]
+}
+
 setuptools.setup(
     name="nomadnet",
     version=__version__,
@@ -15,6 +27,7 @@ setuptools.setup(
     long_description_content_type="text/markdown",
     url="https://github.com/markqvist/nomadnet",
     packages=setuptools.find_packages(),
+    package_data=package_data,
     classifiers=[
         "Programming Language :: Python :: 3",
         "License :: OSI Approved :: MIT License",
@@ -23,6 +36,6 @@ setuptools.setup(
     entry_points= {
         'console_scripts': ['nomadnet=nomadnet.nomadnet:main']
     },
-    install_requires=['rns>=0.2.1', 'lxmf>=0.0.6', 'urwid>=2.1.2'],
-    python_requires='>=3.5',
-)
+    install_requires=["rns>=1.5.5", "lxmf>=1.2.0", "urwid>=3.0.5", "qrcode"],
+    python_requires=">=3.8",
+)

tools/colorgen.py

@@ -0,0 +1,321 @@
+#!/usr/bin/env python3
+"""
+Color Table Generator with Perceptual Normalization
+
+Generates HSL color tables with optional normalization to uniform perceptual
+intensity (CIELAB L*) followed by global lightness scaling.
+"""
+
+import argparse
+import colorsys
+import math
+import sys
+
+
+def hsl_to_rgb(h, s, l):
+    """Convert HSL (0-360, 0-100, 0-100) to RGB (0.0-1.0)."""
+    h_norm = (h % 360) / 360.0
+    s_norm = max(0, min(100, s)) / 100.0
+    l_norm = max(0, min(100, l)) / 100.0
+    r, g, b = colorsys.hls_to_rgb(h_norm, l_norm, s_norm)
+    return (r, g, b)
+
+
+def rgb_to_hex(r, g, b):
+    """Convert RGB 0.0-1.0 to lowercase hex string."""
+    r = max(0, min(1, r))
+    g = max(0, min(1, g))
+    b = max(0, min(1, b))
+    return f"{int(r*255):02x}{int(g*255):02x}{int(b*255):02x}"
+
+
+# CIELAB conversion functions
+def srgb_to_linear(c):
+    if c <= 0.04045:
+        return c / 12.92
+    return ((c + 0.055) / 1.055) ** 2.4
+
+
+def linear_to_srgb(c):
+    if c <= 0.0031308:
+        return c * 12.92
+    return 1.055 * (c ** (1/2.4)) - 0.055
+
+
+def rgb_to_lab(r, g, b):
+    """Convert sRGB (0-1) to CIELAB (L: 0-100, a,b: ~-128 to 127). D65 illuminant."""
+    # sRGB to Linear
+    r_lin = srgb_to_linear(r)
+    g_lin = srgb_to_linear(g)
+    b_lin = srgb_to_linear(b)
+    
+    # Linear to XYZ (D65)
+    X = 0.4124564 * r_lin + 0.3575761 * g_lin + 0.1804375 * b_lin
+    Y = 0.2126729 * r_lin + 0.7151522 * g_lin + 0.0721750 * b_lin
+    Z = 0.0193339 * r_lin + 0.1191920 * g_lin + 0.9503041 * b_lin
+    
+    # XYZ to LAB
+    Xn, Yn, Zn = 95.047, 100.0, 108.883
+    delta = 6/29
+    
+    def f(t):
+        if t > delta**3:
+            return t ** (1/3)
+        return t / (3 * delta**2) + 4/29
+    
+    L = 116 * f(Y / Yn) - 16
+    a = 500 * (f(X / Xn) - f(Y / Yn))
+    b_val = 200 * (f(Y / Yn) - f(Z / Zn))
+    
+    return (L, a, b_val)
+
+
+def lab_to_rgb(L, a, b_val):
+    """Convert CIELAB to sRGB (0-1)."""
+    Xn, Yn, Zn = 95.047, 100.0, 108.883
+    delta = 6/29
+    
+    def inv_f(y):
+        if y > delta:
+            return y ** 3
+        return 3 * delta**2 * (y - 4/29)
+    
+    L_adj = (L + 16) / 116
+    X = Xn * inv_f(L_adj + a / 500)
+    Y = Yn * inv_f(L_adj)
+    Z = Zn * inv_f(L_adj - b_val / 200)
+    
+    # XYZ to Linear RGB
+    r_lin =  3.2404542 * X - 1.5371385 * Y - 0.4985314 * Z
+    g_lin = -0.9692660 * X + 1.8760108 * Y + 0.0415560 * Z
+    b_lin =  0.0556434 * X - 0.2040259 * Y + 1.0572252 * Z
+    
+    r = linear_to_srgb(r_lin)
+    g = linear_to_srgb(g_lin)
+    b = linear_to_srgb(b_lin)
+    
+    return (r, g, b)
+
+
+def generate_colors(
+    hues=None,
+    hue_start=0.0,
+    hue_step=30.0,
+    sat_start=70.0,
+    sat_step=0.0,
+    sat_steps=1,
+    light_start=50.0,
+    light_step=0.0,
+    light_steps=1,
+    perceptual_multiplier=1.0,
+    use_hsl_space=False,
+    normalize=False,
+    normalize_target=None,
+    discards=[]
+):
+    """
+    Generate color table with optional perceptual normalization.
+    
+    Pipeline:
+    1. Generate HSL variations
+    2. If normalize: Convert to LAB, set all L* to target (mean or specified)
+    3. Apply global perceptual multiplier to L*
+    4. Convert to RGB and output hex
+    """
+    # Determine hue list
+    if hues is not None:
+        hue_list = [float(h) % 360 for h in hues]
+    else:
+        hue_list = []
+        current = hue_start % 360
+        if hue_step <= 0:
+            raise ValueError("hue_step must be positive when using start/step mode")
+        while current < 360:
+            hue_list.append(current)
+            current += hue_step
+    
+    # Determine if we need LAB processing
+    needs_lab = normalize or (perceptual_multiplier != 1.0) or not use_hsl_space
+    
+    colors = []
+    lab_colors = []  # Store as (L, a, b) tuples if processing needed
+    
+    for h in hue_list:
+        for i in range(sat_steps):
+            s = sat_start + (i * sat_step)
+            s = max(0, min(100, s))
+            
+            for j in range(light_steps):
+                # l = light_start + (j * light_step)
+                l = light_start + (i * light_step)
+                l = max(0, min(100, l))
+                
+                r, g, b = hsl_to_rgb(h, s, l)
+                
+                if not needs_lab:
+                    colors.append(rgb_to_hex(r, g, b))
+                else:
+                    L, a, b_lab = rgb_to_lab(r, g, b)
+                    lab_colors.append((L, a, b_lab))
+
+    filtered_colors = []
+    filtered_lab_colors = []
+    i = 0
+    for c in colors:
+        i += 1
+        if not str(i) in discards:
+            filtered_colors.append(c)
+
+    i = 0
+    for c in lab_colors:
+        i += 1
+        if not str(i) in discards:
+            filtered_lab_colors.append(c)
+
+    colors = filtered_colors
+    lab_colors = filtered_lab_colors
+
+    if not needs_lab:
+        return colors
+
+    # Step 2: Perceptual Normalization (equalize intensity)
+    if normalize:
+        if normalize_target is not None:
+            target_L = normalize_target
+        else:
+            # Calculate mean perceptual lightness
+            target_L = sum(lab[0] for lab in lab_colors) / len(lab_colors)
+            print(f"Target L*: {target_L}")
+        
+        # Clamp target to valid LAB range
+        target_L = max(0, min(100, target_L))
+        
+        # Normalize all colors to target L, preserving hue (a, b)
+        lab_colors = [(target_L, a, b) for (L, a, b) in lab_colors]
+    
+    # Step 3: Apply final global perceptual multiplier
+    final_colors = []
+    for L, a, b in lab_colors:
+        L_final = L * perceptual_multiplier
+        L_final = max(0, min(100, L_final))
+        r, g, b = lab_to_rgb(L_final, a, b)
+        final_colors.append(rgb_to_hex(r, g, b))
+    
+    return final_colors
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate perceptually normalized color tables",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Pipeline: HSL Generation → [Normalize to uniform L*] → Global Multiplier → Hex
+
+Examples:
+  # 32 colors normalized to same perceptual intensity, then darkened 20%
+  python3 colorgen.py --hue-start 0 --hue-step 11.25 --sat-start 75 --normalize --perceptual-multiplier 0.8
+  
+  # Force all colors to exactly L*=55, then scale by 1.1
+  python3 colorgen.py --hues "0,90,180,270" --normalize --normalize-target 55 --perceptual-multiplier 1.1
+  
+  # Generate with lightness variation, normalize to mean intensity, output brightened
+  python3 colorgen.py --hue-start 15 --hue-step 30 --light-start 40 --light-step 10 --light-steps 3 --normalize --perceptual-multiplier 1.2
+        """
+    )
+    
+    # Hue configuration
+    hue_group = parser.add_mutually_exclusive_group()
+    hue_group.add_argument("--hues", type=str, metavar="LIST",
+                          help='Comma-separated hue values, e.g., "0,60,120"')
+    hue_group.add_argument("--hue-start", type=float, default=0.0,
+                          help="Initial hue offset in degrees")
+    parser.add_argument("--hue-step", type=float, default=30.0,
+                       help="Separation between hues")
+    
+    # Saturation/Lightness configuration
+    parser.add_argument("--sat-start", type=float, default=75.0,
+                       help="Initial saturation (0-100)")
+    parser.add_argument("--sat-step", type=float, default=0.0,
+                       help="Saturation step")
+    parser.add_argument("--sat-steps", type=int, default=1,
+                       help="Number of saturation variations")
+    parser.add_argument("--light-start", type=float, default=50.0,
+                       help="Initial lightness (0-100)")
+    parser.add_argument("--light-step", type=float, default=0.0,
+                       help="Lightness step")
+    parser.add_argument("--light-steps", type=int, default=1,
+                       help="Number of lightness variations")
+    
+    # Perceptual processing
+    parser.add_argument("--normalize", action="store_true",
+                       help="Normalize all colors to same perceptual intensity (L*) before applying multiplier")
+    parser.add_argument("--normalize-target", type=float, default=None, metavar="L",
+                       help="Target L* value (0-100) for normalization. Default: mean of generated colors")
+    parser.add_argument("--perceptual-multiplier", type=float, default=1.0,
+                       help="Final global lightness multiplier (1.0=no change)")
+    parser.add_argument("--hsl-space", action="store_true",
+                       help="Skip LAB conversion if not normalizing (faster, less accurate perceptually)")
+    
+    # Output
+    parser.add_argument("--python-list", action="store_true",
+                       help="Output as Python list")
+    parser.add_argument("--separator", type=str, default="\n",
+                       help="Separator between colors")
+    parser.add_argument("--stats", action="store_true",
+                       help="Print generation stats to stderr")
+    hue_group.add_argument("--discard", type=str, metavar="LIST",
+                          help='Discard output indexes, e.g., "20,2,33"')
+    
+    args = parser.parse_args()
+    
+    if args.hsl_space and args.normalize:
+        print("Warning: --hsl-space ignored because --normalize requires LAB space", file=sys.stderr)
+        args.hsl_space = False
+    
+    try:
+        if args.discard: discards = args.discard.split(",")
+        else: discards = []
+
+        colors = generate_colors(
+            hues=args.hues.split(",") if args.hues else None,
+            hue_start=args.hue_start,
+            hue_step=args.hue_step,
+            sat_start=args.sat_start,
+            sat_step=args.sat_step,
+            sat_steps=args.sat_steps,
+            light_start=args.light_start,
+            light_step=args.light_step,
+            light_steps=args.light_steps,
+            perceptual_multiplier=args.perceptual_multiplier,
+            use_hsl_space=args.hsl_space,
+            normalize=args.normalize,
+            normalize_target=args.normalize_target,
+            discards=discards,
+        )
+        
+        i = 0
+        # Enable to test on light background
+        # print(f"#!bg=fff\n")
+        if args.python_list: print("colors = [", end="")
+        for c in colors:
+            i+=1
+            if args.python_list:
+                print(f"\"{c}\", ", end="")
+
+            else: print(f"`FT{c}colored {i}`f")
+        
+        if args.python_list: print("]")
+
+        if args.stats:
+            total = len(colors)
+            mode = "normalized + " if args.normalize else ""
+            print(f"# Generated {total} colors ({mode}L* × {args.perceptual_multiplier})", 
+                  file=sys.stderr)
+                  
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

tools/rrc-log-dump.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""Dump the contents of rrc on-disk room history logs.
+
+Usage:
+    rrc-log-dump.py <path>
+
+<path> may be:
+  - a single .log file
+  - a hub directory under rrc_history/
+  - the rrc_history/ root (or any ancestor; .log files are found recursively)
+"""
+
+import os
+import sys
+import time
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.join(SCRIPT_DIR, "..", "nomadnet", "vendor"))
+import cbor
+
+
+H_KIND    = "k"
+H_SRC     = "s"
+H_NICK    = "n"
+H_TEXT    = "t"
+H_TS      = "ts"
+H_MENTION = "m"
+
+KIND_COLOR = {
+    "msg":    "",
+    "system": "\033[2m",
+    "notice": "\033[36m",
+    "error":  "\033[31m",
+}
+RESET = "\033[0m"
+
+
+def _fmt_ts(ts_ms):
+    try:
+        return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(ts_ms) // 1000))
+    except Exception:
+        return "????-??-?? ??:??:??"
+
+
+def _fmt_speaker(entry):
+    nick = entry.get(H_NICK)
+    src  = entry.get(H_SRC)
+    src_hex = src.hex()[:12] if isinstance(src, (bytes, bytearray)) else None
+    if nick and src_hex:
+        return "%s (%s)" % (nick, src_hex)
+    if nick:
+        return nick
+    if src_hex:
+        return src_hex
+    return ""
+
+
+def dump_file(path, use_color):
+    print("=== %s ===" % path)
+    count = 0
+    truncated = False
+    with open(path, "rb") as f:
+        while True:
+            try:
+                entry = cbor.load(f)
+            except EOFError:
+                break
+            except Exception as ex:
+                truncated = True
+                print("  [corrupt record after entry %d: %s]" % (count, ex), file=sys.stderr)
+                break
+            if not isinstance(entry, dict):
+                print("  [skipping non-dict entry: %r]" % (entry,), file=sys.stderr)
+                continue
+            kind    = entry.get(H_KIND) or "msg"
+            ts      = _fmt_ts(entry.get(H_TS, 0))
+            speaker = _fmt_speaker(entry)
+            text    = entry.get(H_TEXT) or ""
+            mention = " *" if entry.get(H_MENTION) else "  "
+            color   = KIND_COLOR.get(kind, "") if use_color else ""
+            end     = RESET if (color and use_color) else ""
+            line = "%s  %-6s%s  %-32s  %s" % (ts, kind, mention, speaker, text)
+            print(color + line + end)
+            count += 1
+    suffix = " (truncated)" if truncated else ""
+    print("--- %d entries%s ---" % (count, suffix))
+
+
+def main(argv):
+    if len(argv) != 2 or argv[1] in ("-h", "--help"):
+        print(__doc__, file=sys.stderr)
+        return 2
+    target = argv[1]
+    use_color = sys.stdout.isatty()
+    if os.path.isfile(target):
+        dump_file(target, use_color)
+        return 0
+    if os.path.isdir(target):
+        files = []
+        for root, _dirs, names in os.walk(target):
+            for name in names:
+                if name.endswith(".log"):
+                    files.append(os.path.join(root, name))
+        files.sort()
+        if not files:
+            print("no .log files found under %s" % target, file=sys.stderr)
+            return 1
+        for i, path in enumerate(files):
+            if i > 0:
+                print()
+            dump_file(path, use_color)
+        return 0
+    print("not a file or directory: %s" % target, file=sys.stderr)
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))