From a438dfc2319f50ab4fe42280ec7faed4bc6715f8 Mon Sep 17 00:00:00 2001 From: Rekushi <170186554+RekuNote@users.noreply.github.com> Date: Tue, 15 Sep 2026 23:07:13 +0100 Subject: [PATCH] [RELEASE] RexiMemo: Open Source Edition --- DB.py | 4 + DOCS.md | 492 +++++++++ Hatenatools/License.txt | 661 ++++++++++++ Hatenatools/NTFT.py | 211 ++++ Hatenatools/PPM.py | 990 ++++++++++++++++++ Hatenatools/ReadMe.txt | 34 + Hatenatools/UGO.py | 528 ++++++++++ Hatenatools/__init__.py | 6 + Hatenatools/hatenatools project page.url | 5 + LICENSE | 661 ++++++++++++ README.md | 83 ++ .../000001_OSEOPEN000001_000.ppm | Bin 0 -> 254534 bytes .../000002_OSEOPEN000002_001.ppm | Bin 0 -> 307368 bytes .../000003_OSEOPEN000003_002.ppm | Bin 0 -> 323896 bytes database/__init__.py | 529 ++++++++++ database/reximemo.sqlite3 | Bin 0 -> 90112 bytes docs_page.py | 418 ++++++++ hatena.py | 253 +++++ hatenadir/css/ds/basic.css | 830 +++++++++++++++ hatenadir/css/ds/common.css | 808 ++++++++++++++ hatenadir/css/ds/ose.css | 1 + hatenadir/ds/v2-xx/allflipnotes.ugo.py | 20 + hatenadir/ds/v2-xx/ch.py | 52 + hatenadir/ds/v2-xx/comment.py | 38 + hatenadir/ds/v2-xx/creator.py | 61 ++ hatenadir/ds/v2-xx/eula_list.tsv | 1 + hatenadir/ds/v2-xx/inbox.ugo.py | 8 + hatenadir/ds/v2-xx/index.ugo.py | 30 + hatenadir/ds/v2-xx/movie.py | 185 ++++ hatenadir/ds/v2-xx/sa.py | 58 + hatenadir/images/ds/account.ntft | 1 + hatenadir/images/ds/browse.ntft | 1 + hatenadir/images/ds/placeholder.npf | Bin 0 -> 4144 bytes hatenadir/images/ds/room.ntft | 1 + requirements.txt | 6 +- reximemo.py | 178 ++++ reximemo_image_formats.py | 472 +++++++++ server.py | 252 ++--- tests/test_database.py | 42 + tests/test_release_shape.py | 46 + tests/test_resources.py | 189 ++++ tools/generate_placeholders.py | 52 + web/static/RexiMemo_OSE_Logo.png | Bin 0 -> 58585 bytes web/static/site.css | 83 ++ webapp.py | 395 +++++++ 45 files changed, 8516 insertions(+), 169 deletions(-) create mode 100644 DB.py create mode 100644 DOCS.md create mode 100644 Hatenatools/License.txt create mode 100644 Hatenatools/NTFT.py create mode 100644 Hatenatools/PPM.py create mode 100644 Hatenatools/ReadMe.txt create mode 100644 Hatenatools/UGO.py create mode 100644 Hatenatools/__init__.py create mode 100644 Hatenatools/hatenatools project page.url create mode 100644 LICENSE create mode 100644 README.md create mode 100644 database/Creators/A000000000000001/000001_OSEOPEN000001_000.ppm create mode 100644 database/Creators/A000000000000002/000002_OSEOPEN000002_001.ppm create mode 100644 database/Creators/A000000000000003/000003_OSEOPEN000003_002.ppm create mode 100644 database/__init__.py create mode 100644 database/reximemo.sqlite3 create mode 100644 docs_page.py create mode 100644 hatena.py create mode 100644 hatenadir/css/ds/basic.css create mode 100644 hatenadir/css/ds/common.css create mode 100644 hatenadir/css/ds/ose.css create mode 100644 hatenadir/ds/v2-xx/allflipnotes.ugo.py create mode 100644 hatenadir/ds/v2-xx/ch.py create mode 100644 hatenadir/ds/v2-xx/comment.py create mode 100644 hatenadir/ds/v2-xx/creator.py create mode 100644 hatenadir/ds/v2-xx/eula_list.tsv create mode 100644 hatenadir/ds/v2-xx/inbox.ugo.py create mode 100644 hatenadir/ds/v2-xx/index.ugo.py create mode 100644 hatenadir/ds/v2-xx/movie.py create mode 100644 hatenadir/ds/v2-xx/sa.py create mode 100644 hatenadir/images/ds/account.ntft create mode 100644 hatenadir/images/ds/browse.ntft create mode 100644 hatenadir/images/ds/placeholder.npf create mode 100644 hatenadir/images/ds/room.ntft create mode 100644 reximemo.py create mode 100644 reximemo_image_formats.py create mode 100644 tests/test_database.py create mode 100644 tests/test_release_shape.py create mode 100644 tests/test_resources.py create mode 100644 tools/generate_placeholders.py create mode 100644 web/static/RexiMemo_OSE_Logo.png create mode 100644 web/static/site.css create mode 100644 webapp.py diff --git a/DB.py b/DB.py new file mode 100644 index 0000000..f917bb1 --- /dev/null +++ b/DB.py @@ -0,0 +1,4 @@ +"""SQLite database compatibility import used by the DSi resource modules.""" +from database import Database + +DB_type = "sqlite" diff --git a/DOCS.md b/DOCS.md new file mode 100644 index 0000000..1869702 --- /dev/null +++ b/DOCS.md @@ -0,0 +1,492 @@ +# RexiMemo OSE documentation + +This is the repository copy of the manual served at `/docs` when RexiMemo OSE is started with `python3 server.py docs:enabled`. The web route is optional; this Markdown file is always available in the source tree. + +This manual describes the Open Source Edition as it is shipped in this repository: the HTTP server, web client, Flipnote Studio resources, SQLite layout, media handling, identity model, moderation controls, assets and tests. + +> **About the simplified OSE design.** A number of security, identity, moderation, networking and privacy mechanisms are intentionally much simpler here than on the private RexiMemo service. This is a release-safety boundary: the open-source tree avoids exposing production keys, authentication internals, trust checks, anti-abuse logic and deployment assumptions that could create security problems for the live service if published. The simpler replacements are easier to inspect and run independently, but they should not be mistaken for stronger security. Sections with an important consequence call this out explicitly. + +## Contents + +- [Starting the server](#start) +- [Repository layout](#shape) +- [Request routing](#request-routing) +- [Web client](#web) +- [Flipnote view page](#flipnote-page) +- [Flipnote Studio / DSi side](#dsi) +- [Posting a Flipnote](#posting) +- [Comments](#comments) +- [Accounts and IP login](#accounts) +- [Creator's Rooms](#rooms) +- [SQLite database](#database) +- [On-disk media](#files) +- [PPM, TMB, UGO, NTFT and NPF](#formats) +- [Image conversion code](#images) +- [Internal code reference](#code-reference) +- [Admin and bans](#admin) +- [Configuration](#configuration) +- [Assets and styling](#assets) +- [Tests](#tests) +- [Changing the server](#changing) +- [Deliberately absent systems](#limits) + +## Starting the server + +The entry point is `server.py`. A normal start is: + + python3 -m pip install -r requirements.txt + python3 server.py + +The default TCP port is `8080`. Set `REXIMEMO_PORT` before starting the process if another port is required. The process changes its working directory to the repository root before importing the server modules, so paths such as `database/`, `hatenadir/` and `web/static/` are resolved relative to the checkout rather than the shell's current directory. + +The optional documentation site is enabled with an argument rather than being exposed by default: + + python3 server.py docs:enabled + +When that argument is present, the server sets its documentation flag before constructing the Twisted resource tree. The web navigation gains a Docs item and `/docs` is served. Without the flag, there is no Docs navigation item and the same path follows the ordinary web 404 path. + +`server.py` creates a Twisted `Site` subclass named `ProxyCompatibleSite`. Its only protocol alteration is for old-style absolute HTTP proxy requests. Before Twisted parses incoming bytes, request lines beginning with `GET http://flipnote.hatena.com`, `POST http://flipnote.hatena.com`, `GET http://ugomemo.hatena.ne.jp` or `POST http://ugomemo.hatena.ne.jp` are rewritten into origin-form request lines. This lets the same HTTP listener accept the historical proxy traffic used by Flipnote Studio and ordinary browser requests. + +> **Network/security notice.** OSE deliberately does not reproduce RexiMemo's production NAS, DNS, certificate or authentication gateway. Those pieces are kept out of the public repository to reduce the risk of exposing live-service security logic, secrets or deployment assumptions. As a result, OSE is only the Hatena HTTP server/proxy portion: a DSi must obtain working Nintendo/Flipnote authentication through the official RexiMemo DNS service or another functional DNS/auth/NAS setup, then send Hatena HTTP traffic to this server as its proxy. + +Logs are written to `logs/reximemo.log` and mirrored to standard output unless a call is marked silent. The `logs/` directory is runtime data and is not part of the repository. + +## Repository layout + +| Path | Purpose | +|--------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `.gitignore` | Excludes Python caches/virtual environments, logs, SQLite WAL/SHM sidecars, common key/certificate/token files, `.env` and Finder metadata from commits. | +| `server.py` | Process entry point, port selection, optional feature flags, logging and the proxy-compatible Twisted Site. | +| `hatena.py` | Top-level HTTP resource router. It separates web traffic from DSi traffic, serves static CSS/images, loads the `hatenadir` resource tree and applies IP bans before either surface is reached. | +| `reximemo.py` | Shared DSi helpers: regional URLs, current IP/account lookup, DSi keyboard values, DSi dialog/forward headers, UGO menu construction, pagination, PPM first-frame rendering and mini-Flipnote NPF conversion. | +| `webapp.py` | The normal browser client. It renders Home, Browse, About, optional Docs, Flipnote, Creator's Room, account and admin pages and exposes thumbnails/PPM downloads. | +| `docs_page.py` | The text of this built-in manual. It is imported by the web client only when the documentation route is requested. | +| `DB.py` | Compatibility shim used by the older-style resource modules. It imports the single SQLite `Database` object and advertises `DB_type = "sqlite"`. | +| `database/__init__.py` | SQLite schema, seed data and all database operations used by the OSE server. | +| `database/reximemo.sqlite3` | Seed database supplied with the repository. It includes the default admin account, General channel, sample Creator's Rooms and three sample Flipnote records. | +| `database/Creators/` | PPM storage, grouped by creator FSID. The release contains three anonymised sample PPM files. | +| `database/Comments/` | Created/used for one-frame mini-Flipnote comment PPMs. Text comments stay entirely in SQLite. | +| `hatenadir/ds/v2-xx/` | Flipnote Studio HTTP resources. Python files are loaded into Twisted resources at start-up; the loader also supplies `.uls` aliases for `.ugo` resources. | +| `hatenadir/css/ds/common.css` | Large compatibility stylesheet containing the old DSi HTML class names expected by the retained resource layout. | +| `hatenadir/css/ds/basic.css` | Additional legacy/basic DSi styles. It overlaps a number of old service selectors but remains harmless for the small OSE page set. | +| `hatenadir/css/ds/ose.css` | Short OSE override sheet that gives the DSi HTML pages their purple/green placeholder presentation and readable detail/comment styling. | +| `hatenadir/images/ds/browse.ntft` | 32×32 generated Browse menu icon. | +| `hatenadir/images/ds/account.ntft` | 32×32 generated Account menu icon. | +| `hatenadir/images/ds/room.ntft` | 32×32 generated Creator's Room menu icon. | +| `hatenadir/images/ds/placeholder.npf` | 128×64 generated indexed placeholder banner retained for DSi asset testing/use. | +| `web/static/site.css` | Complete browser stylesheet: colour variables, header/navigation, Flipnote grids, richer watch layout, comments, forms, admin tables, documentation layout and responsive breakpoints. | +| `web/static/RexiMemo_OSE_Logo.png` | White RexiMemo logo with “Open Source Edition” included in the image itself. | +| `Hatenatools/__init__.py` | Convenience exports for `PPM`, `TMB`, `UGO` and `NTFT`. | +| `Hatenatools/PPM.py` | Legacy Flipnote Studio PPM/TMB parser: metadata, thumbnail data, frame decoding, sound decoding and helper dump functions. OSE mainly uses `PPM.Read()`, `PPM.GetFrame()` and `TMB.Read()`. | +| `Hatenatools/UGO.py` | Legacy UGO/UGAR parser/packer. OSE builds DSi menus by appending layout, top-screen text, category, post and button items then calling `Pack()`. | +| `Hatenatools/NTFT.py` | Legacy NTFT reader/writer retained for compatibility and format work. OSE's generated placeholders use the newer encoder in `reximemo_image_formats.py`. | +| `Hatenatools/ReadMe.txt` | Original Hatenatools version notes, format summary and contributor credits. | +| `Hatenatools/License.txt` | The Hatenatools AGPL licence text shipped with that code. | +| `Hatenatools/hatenatools project page.url` | Legacy Internet Shortcut retained with the upstream helper sources. | +| `reximemo_image_formats.py` | Independent image codec helpers for NTFT, NPF and NBF plus standard Pillow-backed image formats. | +| `tools/generate_placeholders.py` | Rebuilds the plain DSi placeholder icons and banner from simple geometric shapes. | +| `tests/test_database.py` | Temporary-database checks for seed admin, IP login/logout, guest room stability and bans. | +| `tests/test_resources.py` | Twisted-stub resource tests covering the DSi tree, UGAR menu generation, web routes, watch-page detail, Docs gating and PNG thumbnail generation. | +| `tests/test_release_shape.py` | Release checks preventing key/certificate/font file types, named removed production components and sample-content drift. | +| `requirements.txt` | Runtime Python dependencies: Twisted, NumPy and Pillow. | +| `README.md` | Short installation/release overview. | +| `DOCS.md` | GitHub-readable copy of this detailed technical reference. The built-in `/docs` page and this file are intended to carry the same operational information. | +| `LICENSE` | GNU Affero General Public License version 3. | + +The three sample PPM paths are `database/Creators/A000000000000001/000001_OSEOPEN000001_000.ppm`, `...0002/000002_OSEOPEN000002_001.ppm` and `...0003/000003_OSEOPEN000003_002.ppm`. Their matching public Flipnote IDs are `f_sample1` through `f_sample3`; their Creator's Room public IDs are `r_sample1` through `r_sample3`. + +## Request routing + +`hatena.Setup()` returns a `Root` resource. Every request first reaches that object. The source IP is checked against `ip_bans` before web or DSi routing. A blocked address receives a plain `403` response. + +The request's `Host` header determines which surface is selected. `flipnote.hatena.com` and `ugomemo.hatena.ne.jp` are treated as DSi hosts. Everything else, including `localhost` and a normal server hostname, is sent to `WebRoot`. + +DSi-host requests must contain `X-DSi-SID`. OSE does not validate or decode the SID; the header is only the transport-side gate expected by this reduced resource tree. A missing header produces the access-denied resource. This is deliberately separate from account state: an IP may have a web/DSi OSE account login row while DSi-host routing still requires the header. + +> **Identity notice.** `X-DSi-SID` is a compatibility marker in OSE, not proof of a particular console or account. Production SID/device verification, FSID ownership checks and related anti-abuse systems were intentionally removed rather than publishing their implementation. This keeps the public DSi path small, but callers must not treat the header as a security credential. + +Static top-level routes are handled before host routing. `/css/` maps to `hatenadir/css/`, `/images/` maps to `hatenadir/images/`, and `/static/` maps to `web/static/`. This mirrors the old server shape while keeping browser assets separate from DSi assets. + +The DSi region root recognises `v2-xx`, `v2-eu`, `v2-us` and `v2-jp`. All four currently use the same loaded `v2-xx` implementation. URL generation switches the Japanese host to `ugomemo.hatena.ne.jp`; the other regions use `flipnote.hatena.com`. + +## Web client + +The browser client is server-rendered HTML in `webapp.py`. It has no JavaScript framework and no browser session cookie. Each request asks the database for the account currently associated with the source IP. + +> **Login notice.** The web client uses the same deliberately simple IP identity model as the rest of OSE. Production session/device management was left out so that live authentication design, keys and account-security machinery are not copied into a public release. Shared addresses, proxies and changing IPs therefore have consequences described in the Accounts section below. + +| Route | Method | Behaviour | +|------------------------------|----------|-------------------------------------------------------------------------------------------------------------------------------------| +| `/` | GET | Home page with the most recent Flipnotes. | +| `/browse` | GET | Browse up to 100 Flipnotes. `?sort=new`, `?sort=popular` and `?sort=top` map to posted time, views/stars and stars/views orderings. | +| `/about` | GET | Open-source credits and acknowledgements only. | +| `/docs` | GET | This manual, only when documentation is enabled at server start. | +| `/watch/` | GET | Flipnote detail page. It increments the view count, loads comments and shows other notes from the same Creator's Room. | +| `/watch//comment` | POST | Adds a text comment using the current account or guest-IP Creator's Room. | +| `/thumb/.png` | GET | Decodes the first PPM frame and returns a cached PNG thumbnail. | +| `/comment-thumb/.png` | GET | Returns a first-frame PNG preview for a mini-Flipnote comment. | +| `/flipnote/.ppm` | GET | Increments downloads and returns the original PPM as an attachment. | +| `/creator/` | GET | Creator's Room with its Flipnote grid and account/guest ownership label. | +| `/login` | GET/POST | Username/password sign-in. Successful POST writes an `ip_logins` row for the source IP. | +| `/register` | GET/POST | Creates an account and immediately associates the source IP with it. | +| `/logout` | GET | Deletes the current source IP from `ip_logins`. | +| `/account` | GET | Shows the signed-in username, current IP and Creator's Room link when one exists. | +| `/admin` | GET | Minimal administrator view for IP bans and Flipnote deletion. | +| `/admin/ban` | POST | Adds or updates an IP ban and clears that IP's login row. | +| `/admin/unban` | POST | Removes an IP ban. | +| `/admin/delete` | POST | Soft-deletes a Flipnote record by setting `deleted=1`. | + +The common page wrapper builds the header, navigation, account/admin actions and footer. Docs is inserted into that navigation only when the `WebRoot` instance was created with documentation enabled. + +## Flipnote view page + +The browser does not attempt to reproduce Flipnote Studio playback. A Flipnote view uses the first decoded frame as a large preview and presents the original `.ppm` as the downloadable playable file. This keeps the web client small and avoids adding a separate animation/audio player implementation. + +Opening the page calls `Database.AddView()` before the Flipnote row is reloaded, so the displayed view count includes the current request. The page then loads up to 100 comments, the channel record, the Creator's Room record and a short list of other non-deleted Flipnotes from the same room. + +The information panel displays stars, views, downloads and comment count, then the post time, channel, creator FSID, PPM filename and public identifier. These values come directly from the OSE SQLite row; there is no secondary profile, device or ownership lookup. + +> **Metadata notice.** Creator metadata shown here is repository data, not a cryptographically verified identity. OSE deliberately avoids the production ownership/device verification chain so those security mechanisms are not exposed in the public codebase. Treat FSIDs, embedded creator names and room ownership as local OSE identifiers. + +Text comments are HTML-escaped. Mini-Flipnote comments are shown as first-frame thumbnails through `/comment-thumb/.png`; the stored PPM is still the authoritative comment file used by the DSi endpoints. + +## Flipnote Studio / DSi side + +### Connecting a DSi + +RexiMemo OSE does not contain the authentication/NAS or DNS service needed to get a DSi through the normal Nintendo/Flipnote connection path. Use the official RexiMemo DNS service, or another working DNS setup that also provides the required authentication/NAS behaviour for Flipnote Studio. + +1. Configure the DSi internet connection to use the official RexiMemo DNS service, or another known-working DNS/auth/NAS service. +2. In the same DSi connection settings, enable the HTTP **Proxy Server**. +3. Set the proxy server address to the IP address of the machine running RexiMemo OSE. +4. Set the proxy port to `8080`, unless `REXIMEMO_PORT` was deliberately changed on the OSE server. +5. Start OSE with `python3 server.py` and then open Flipnote Studio/Flipnote Hatena on the DSi. + +DNS/auth/NAS and the Hatena HTTP proxy are separate jobs in this arrangement. The external DNS/auth/NAS service gets the console through the authentication path; RexiMemo OSE handles the Hatena web requests that the DSi sends to the configured proxy. + +> **Connection/security notice.** The missing auth/NAS stack is deliberate. OSE uses an external working service instead of publishing RexiMemo's private authentication and network-security implementation. Do not copy private keys, production credentials or live authentication code into an OSE checkout to make up for the missing component. Point the DSi at an authorised working DNS/auth/NAS service and keep OSE limited to the proxy/server role it was released for. + +The DSi surface is built from Twisted resources under `hatenadir/ds/v2-xx`. `hatena.LoadHatenadirStructure()` walks that directory at start-up. Ordinary files become static resources. A filename ending in `.py` is imported as a module and its `PyResource()` object is mounted under the filename without `.py`. Any resulting resource whose mounted name ends in `.ugo` is also mounted under the equivalent `.uls` name. + +| Resource | Function | +|-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `index.ugo.py` | Home UGO menu. It shows Browse, Account, the current Creator's Room when one exists, then recent Flipnotes. Menu icons are the generated 32×32 NTFT placeholders. | +| `allflipnotes.ugo.py` | Paginated all-Flipnotes UGO list. It uses 50 Flipnotes per page and adds Previous/Next buttons when required. | +| `ch.py` | Channel browse and posting. `ch/1.uls` lists channel Flipnotes; `ch/1.htm` is a small HTML description; POST to `ch/1.post` accepts a PPM and stores it. | +| `creator.py` | Creator's Room resources. It accepts the numeric internal room ID and serves `profile.htm`, `room.uls` and `movies.uls`. `room.ugo`/`movies.ugo` are accepted as aliases. | +| `movie.py` | Flipnote PPM/TMB/details, star, download and comment endpoints. | +| `comment.py` | Stored mini-Flipnote comment delivery as raw `.ppm` or generated `.npf` preview. | +| `sa.py` | Simple DSi account pages and keyboard-based login/register/logout flow. | +| `inbox.ugo.py` | Empty compatibility page stating that notifications are not included. | +| `eula_list.tsv` | Static compatibility row containing the base64-encoded “English” label and `en` language code. It is retained because older clients/resources may request the file even though OSE has no terms/EULA flow. | + +UGO menu data is assembled by `reximemo.make_menu()` through Hatenatools' `UGO` class. Thumbnail buttons embed the first `0x6A0` bytes of a PPM as TMB data. Icon buttons embed a 32×32 NTFT and are rejected if the byte length is not exactly 2048. + +The DSi HTML helper links `common.css`, `basic.css` and `ose.css` from the Hatena host. These pages deliberately stay small and old-browser friendly: basic HTML, tables, metadata tags and CSS with no JavaScript dependency. + +## Posting a Flipnote + +1. Flipnote Studio POSTs PPM bytes to a channel `.post` resource. +2. `ch.py` reads the request body and asks `current_account()` whether the source IP is signed in. +3. `Database.AddFlipnote()` parses the PPM header through Hatenatools `TMB().Read()`. +4. The editor author ID becomes the stored creator FSID. `CurrentFilename` supplies the base filename. If either value is missing, or the same FSID/filename already exists, the post is refused. +5. The embedded username/editor name is used as the display-name hint. +6. `get_or_create_creator()` obtains the account-owned room or guest-IP room. This is the point at which a Creator's Room is automatically created for a first post. +7. The requested channel is checked; an unknown channel falls back to channel 1. +8. A Flipnote row is created with a new public ID beginning `f_`. +9. The exact submitted PPM bytes are written to `database/Creators//.ppm`. + +No web PPM upload route exists. Browser users can browse/download/comment, but new Flipnotes enter OSE through the Flipnote Studio channel post path. + +> **Posting notice.** OSE reads the creator FSID, username and filename from the submitted PPM, but it does not perform the production FSID ownership or PPM signature-verification steps. Those systems were removed to avoid releasing production trust logic and key-dependent verification code. The simpler OSE path is intended for a small self-hosted/test environment; do not interpret embedded PPM identity fields as independently proven. + +## Comments + +### Text comments + +Web text comments POST a normal form field named `text`. DSi text comments use the keyboard endpoint and read the value from `X-Email-Addr`, matching the old keyboard transport convention. The database collapses whitespace, truncates to 500 characters and rejects an empty result. + +A comment needs a Creator's Room. If the source IP is logged in, the room belongs to that account. Otherwise a room is created or reused for the source IP. The stored comment references only the room ID; display names shown later are joined from `creator_rooms`. + +### Mini-Flipnote comments + +A POST to a Flipnote's `.reply` endpoint is parsed as PPM/TMB. OSE requires exactly one frame. The PPM's author ID and embedded username are passed into the same room-creation function, then the comment row is inserted as type `memo`. Its binary PPM is written to `database/Comments/.ppm` and that filename is stored in the row. + +For the DSi comment preview, `comment_npf()` decodes the first PPM frame, resizes it to the requested dimensions and encodes it as NPF. Width is clamped to 1–256 and height to 1–192. The raw mini-Flipnote remains available through its `.ppm` comment route. + +> **Comment identity notice.** Guest comments are attached to an IP-backed Creator's Room, and mini-Flipnote identity hints come from the submitted PPM. This is intentionally simpler than production moderation/account attribution so the public tree does not contain the production device, session and ownership-verification stack. On shared networks, more than one person can therefore appear under the same guest identity. + +## Accounts and IP login + +Accounts are optional. Usernames are 3–24 characters and may contain ASCII letters, numbers, dots, dashes and underscores. Passwords must contain at least four characters. The OSE database stores a SHA-256 digest in `password_sha256`. + +> **Account security notice.** Accounts are deliberately implemented in a much simpler form than the private service. Production password/session/device security, passkeys, recovery flows, per-device state and authentication-gateway logic are not published here because doing so could expose security-sensitive implementation details or assumptions from the live service. OSE substitutes a small local model so the public code can stand alone. That release choice limits what sensitive production material is exposed; it does not make the replacement stronger. Unsalted SHA-256 is not a modern password-storage scheme and the minimum password rule is intentionally small. + +OSE login state is tied to your IP address. `login_ip()` upserts one `ip_logins` row whose primary key is the source IP. If that address logs into a different account, the row is replaced with the new account ID. `account_for_ip()` joins the row to `accounts` and refreshes `last_seen`. Logout deletes the row. + +There are no cookies or browser session identifiers in this flow. The DSi and browser clients see the same account when they reach the server from the same source IP. If traffic is behind a shared NAT or reverse proxy, the address visible to Twisted is the identity key used by OSE. + +> **IP login notice.** An IP address is only a convenient OSE ownership key. Addresses can be shared, reassigned, hidden behind NAT or represented by a reverse proxy. OSE uses this simpler local mapping so the public release does not contain RexiMemo's production session/device security implementation, reducing the risk of exposing live-service authentication behaviour. It is a release-safety compromise, not a secure substitute for a proper session design. + +The DSi account page uses `username:password` entered through the Flipnote Studio keyboard. The keyboard value arrives in `X-Email-Addr`. Registration creates the account, writes the IP login, and forwards back to `sa/account.htm` using `X-DSi-Forwarder`. + +The seeded administrator is `reximemo` with password `alpine`. It is created only when no account named `reximemo` exists in the opened database. + +> **Default credential notice.** The seed login exists so a fresh checkout can exercise the admin page immediately. It is public repository data, not a secret. Change or remove it before exposing an OSE instance beyond a controlled test network. + +## Creator's Rooms + +A room has an internal integer ID and a public ID beginning `r_`. Account rooms have `account_id` set and normally have no `owner_ip`. Guest rooms have `owner_ip` set and no account ID. SQLite uniqueness rules permit only one room per account and one guest room per stored IP. + +For an account, the displayed room name is the account username. A first observed FSID can fill an empty room FSID. For a guest, the first room name is either the PPM-provided creator name or a deterministic `Guest-XXXXXX` value derived from the IP with SHA-1. If a guest room still has the generated Guest name, a later PPM with a display name can replace it. + +The web uses the room's public ID in URLs. The DSi resource tree primarily uses the internal numeric room ID for Creator's Room paths. `UgoRoot` also recognises a 16-hex-character path component and can resolve a room from its FSID before passing into the creator resource. + +> **Room ownership notice.** Account ownership and guest-IP ownership are convenience rules inside this OSE database. They do not reproduce production creator verification. This design deliberately avoids carrying production account/device linkage and FSID proof into the public repository, so room ownership should be understood as local OSE state rather than an authoritative Nintendo/Flipnote identity claim. + +## SQLite database + +`DatabaseBackend` opens `database/reximemo.sqlite3` unless `REXIMEMO_DB` points elsewhere. `check_same_thread=False` is used because Twisted/resource code may access the shared connection outside SQLite's default creating-thread check. A re-entrant Python lock wraps writes and related grouped operations. Foreign keys are enabled and journal mode is set to WAL. + +| Table | Important fields | Meaning | +|-----------------|-------------------------------------------------------------------------------------|--------------------------------------------------------------| +| `accounts` | `username`, `password_sha256`, `is_admin`, `created_at` | Optional local web/DSi account. | +| `ip_logins` | `ip` primary key, `account_id`, `last_seen` | Current IP-to-account association. | +| `ip_bans` | `ip` primary key, `reason`, `created_at` | Addresses denied before routing. | +| `creator_rooms` | `public_id`, `account_id`, `owner_ip`, `fsid`, `display_name` | Identity/ownership container used by Flipnotes and comments. | +| `channels` | `id`, `title`, `description` | Posting/browse channel. OSE seeds channel 1 as General. | +| `flipnotes` | `public_id`, room, FSID, filename, title, channel, counters, `posted_at`, `deleted` | Metadata for each stored PPM. | +| `comments` | `public_id`, Flipnote, room, `type`, text/memo filename, `posted_at`, `deleted` | Text or one-frame mini-Flipnote comments. | + +`flipnotes` has a uniqueness constraint on `(creator_fsid, filename)`. Public IDs are generated from `secrets.token_urlsafe()`, stripped of `-`/`_`, truncated, and prefixed according to object type. They are routing identifiers rather than authentication tokens. + +> **Database/privacy notice.** OSE stores usernames, password digests, source IP login mappings, guest owner IPs and IP bans directly in SQLite. This is deliberately straightforward so the public release has no hidden account/session service or production datastore dependency. If you run a public instance, decide what IP retention, access control, backups and database-file permissions are appropriate for your environment. + +Flipnote deletion is a soft delete. Listing and lookup queries require `deleted=0`; the PPM file is not erased by `delete_flipnote()`. Comments also carry a `deleted` column, although the OSE web admin does not expose comment moderation controls. + +Sorting rules are defined in `list_flipnotes()`. Newest uses `posted_at DESC, id DESC`; Popular uses views, then stars, then posted time; Stars uses stars, then views, then posted time. + +## On-disk media + +SQLite holds metadata; PPM bytes are kept as files. A Flipnote path is constructed as: + + database/Creators//.ppm + +The PPM filename stored in SQLite does not include `.ppm`. `GetFlipnotePPM()` reads the whole file. `GetFlipnoteTMB()` reads only the first `0x6A0` bytes for menu thumbnails. + +Mini comments use: + + database/Comments/.ppm + +The database stores that basename in `memo_filename`. The numeric filename is assigned after the row insert because the SQLite row ID is used as the stable file name. + +The release's three example PPMs use sample FSIDs and rewritten sample metadata. They are present so the web/DSi browse paths, thumbnail decoder and download routes can be tested immediately after cloning. + +> **Storage notice.** Flipnotes and mini-Flipnote comments are ordinary files on disk and are not encrypted by OSE. A soft-deleted Flipnote row also leaves its PPM file in place. The storage model is intentionally transparent and dependency-free; operators who need stronger deletion, encryption, retention or backup guarantees should add them outside or on top of this repository. + +## PPM, TMB, UGO, NTFT and NPF + +### PPM and TMB + +PPM is the Flipnote Studio animation container handled by `Hatenatools/PPM.py`. OSE uses the full PPM parser when it needs a decoded frame or mini-Flipnote validation. It uses `TMB().Read()` for metadata-oriented work such as FSID, username, filename and frame count. For menu thumbnails the first `0x6A0` bytes are served directly as TMB data. + +> **Parser notice.** Hatenatools is retained primarily for compatibility with these old file formats. OSE deliberately does not wrap it in the full set of production upload validation, signature checks, anti-abuse controls and isolation measures. Treat arbitrary files from an untrusted public upload source as untrusted input and add the controls suitable for that deployment. + +`ppm_first_frame_image()` asks Hatenatools to read frames but not sound, obtains frame zero, converts the returned array in Fortran order and constructs a Pillow RGBA image. Browser thumbnails convert that image to RGB and resize with nearest-neighbour resampling. + +### UGO / UGAR + +Flipnote Studio menu screens are packed with Hatenatools `UGO`. `make_menu()` begins with a layout item, optionally adds top-screen title data and category/dropdown items, then encodes each button. Flipnote buttons embed TMB thumbnails; OSE navigation buttons embed generated NTFT icons. The packed result begins with the UGAR container signature used by the test suite. + +### NTFT + +NTFT assets are raw 16-bit ABGR1555 pixels. Dimensions are external to the file. A 32×32 NTFT therefore occupies exactly 2048 bytes. The OSE UGO helper enforces that size for embedded menu icons. + +### NPF + +NPF is a 4-bit indexed image format with a transparent index and up to 15 opaque colours. OSE uses it for mini-Flipnote comment previews and one generated placeholder banner. Width and height are supplied by the caller because the raw image dimensions are not stored in the NPF payload. + +### NBF + +`reximemo_image_formats.py` also retains NBF encode/decode support. NBF is an 8-bit indexed format with up to 256 opaque colours. There is no current OSE HTTP endpoint that requires NBF; the codec remains useful for working with related DSi assets. + +## Image conversion code + +`reximemo_image_formats.py` is a self-contained converter for Nintendo-oriented and standard still-image formats. The proprietary set is `ntft`, `npf` and `nbf`. Standard outputs are PNG, JPEG, WebP, BMP and GIF. + +Input dimensions are validated before large allocations: each side is limited to 2048 pixels and total area to 4,194,304 pixels. Raw formats whose dimensions are not embedded require width/height from the caller. ABGR1555 packing/unpacking is implemented directly. Indexed formats use Pillow quantisation after conversion to a reduced 5-bit colour space, which better matches the source pixel formats. + +The public encoder returns an `EncodedImage` dataclass containing bytes, format key/label, extension, MIME type, preview image and format note. The decoder returns a `DecodedImage` with the corresponding image and metadata. JPEG/BMP paths flatten transparency onto white; GIF output is one still frame. + +`tools/generate_placeholders.py` uses this encoder. It draws three 32×32 icons and one 128×64 banner with Pillow primitives and writes NTFT/NPF bytes into `hatenadir/images/ds`. + +## Internal code reference + +This section lists the small internal API that joins the repository together. Names are included because most extensions to OSE can be made by following these existing call paths rather than adding another framework layer. + +### `hatena.py` + +| Name | Role | +|---------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `as_text(value)` | Converts bytes to UTF-8 text with replacement for malformed sequences; all other values are stringified. | +| `request_path(request)` | Returns the request path as text. | +| `request_headers(request)` | Builds a lower-case text-keyed dictionary from Twisted's request headers. | +| `request_args(request)` | Converts Twisted's byte query arguments into a text dictionary whose values remain lists. | +| `client_ip(request)` | Reads `getClientIP()` first and falls back to `getClientAddress().host`. The returned string is the key used by OSE login, guest ownership and bans. | +| `Log(request, path=None, silent=...)` | Writes a short source-IP/request-path entry through the process logger when one is attached. | +| `AccessDeniedResource` | Plain 403 used when DSi-host traffic does not carry `X-DSi-SID`. | +| `BannedResource` | Plain 403 used for an IP present in `ip_bans`. | +| `NotFoundResource` | Plain 404 resource which also records the path/query in the log. | +| `Root` | Top-level static/host router. It owns the web root, DSi root and static directory resources. | +| `DSRoot` | Recognises supported `v2-*` region names and forwards each to the shared UgoRoot. | +| `UgoRoot` | Loads `hatenadir/ds/v2-xx`. It also resolves a bare 16-hex FSID path to a Creator's Room when possible. | +| `FileResource` | Static DSi file wrapper. Files whose extension starts with `htm` receive an HTML content type. | +| `FolderResource` | Container for recursively loaded directories. Direct rendering is forbidden with 403. | +| `LoadHatenadirStructure()` | One-level-at-a-time recursive loader. Python resources are imported by absolute path with a SHA-1-derived module name to avoid ordinary module-name collisions. | +| `Setup(docs_enabled=False)` | Constructs the root resource and passes the documentation switch into the browser `WebRoot`. | + +### `reximemo.py` + +| Name | Role | +|---------------------------|------------------------------------------------------------------------------------------------------------------------| +| `region_from_request()` | Finds the first path segment beginning `v2-` and returns its region suffix; defaults to `xx`. | +| `host_for_region()` | Returns the Japanese Ugomemo host for `jp`, otherwise the main Flipnote host. | +| `dsi_url()` | Builds an absolute historical-style DSi URL using the current request region. | +| `current_account()` | Looks up the source IP in `ip_logins`. | +| `current_creator()` | Convenience call into `get_or_create_creator()` using the request IP/current account and optional PPM identity hints. | +| `keyboard_value()` | Returns the stripped `X-Email-Addr` header used by Flipnote Studio keyboard submissions. | +| `dialog()` | Sets an HTTP status, `X-DSi-Dialog-Type: 1`, plain content type and UTF-16LE message body. | +| `forward()` | Sets `X-DSi-Forwarder` to tell the DSi browser which resource to open next. | +| `html_page()` | Produces the minimal DSi HTML wrapper, top-screen title metadata and three DSi stylesheets. | +| `query_int()` | Reads a query argument as an integer and clamps it to caller-supplied minimum/maximum bounds. | +| `make_menu()` | Builds an in-memory UGO menu from layout/title/category/post/button dictionaries, including embedded TMB or NTFT data. | +| `placeholder_icon()` | Reads one generated NTFT file from `hatenadir/images/ds`. | +| `flipnote_buttons()` | Turns database Flipnote rows into UGO thumbnail buttons; unreadable PPM/TMB files are skipped. | +| `pagination_buttons()` | Calculates page count and returns Previous/Next UGO button dictionaries. | +| `ppm_first_frame_image()` | Decodes PPM frame zero into a Pillow RGBA image. | +| `ppm_thumbnail_png()` | Creates a nearest-neighbour RGB PNG thumbnail from frame zero. | +| `comment_npf()` | Loads a stored mini-comment PPM, decodes/resizes its first frame and returns NPF bytes. | + +### `DatabaseBackend` + +> **Implementation notice.** Several database methods below deal with login, ownership and moderation. Their small size is intentional: the OSE implementation replaces production session/device/security services with local SQLite operations so private operational logic is not exposed. Method names such as `verify_account()` describe what they do inside OSE, not a claim that they provide production-grade identity verification. + +| Method/property | Role | +|-------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------| +| `create_account()` | Validates username/password, hashes the password and inserts a non-admin account. | +| `verify_account()` | Hashes the supplied password and queries by username plus digest. | +| `get_account()` / `get_account_by_username()` | Direct account lookup helpers. | +| `login_ip()` | Upserts the one login row for an IP and records `last_seen`. | +| `logout_ip()` | Deletes an IP login row. | +| `account_for_ip()` | Joins login to account and refreshes `last_seen` when found. | +| `is_ip_banned()`, `ban_ip()`, `unban_ip()`, `list_bans()` | Exact-IP moderation operations. Banning also signs that IP out. | +| `get_creator_room*` | Room lookups by numeric ID, public ID or FSID. | +| `room_for_account()` / `room_for_ip()` | Ownership lookups used before automatic room creation. | +| `get_or_create_creator()` | Central account/guest identity resolver. It creates the room on first use and fills previously empty FSID/name fields when suitable. | +| `list_creator_flipnotes()` | Thin room-filtered wrapper around `list_flipnotes()`. | +| `get_channel()` / `list_channels()` | Channel metadata access. | +| `FlipnotePath()` | Returns the PPM filesystem path for an FSID/filename pair. | +| `CreatorExists()` | True when a room with the FSID exists or the corresponding creator directory exists. | +| `FlipnoteExists()` | Requires both a non-deleted database row and an existing PPM file. | +| `AddFlipnote()` | Parses TMB identity/filename, resolves the room/channel, inserts metadata and writes PPM bytes. | +| `get_flipnote()`, `get_flipnote_by_id()`, `get_flipnote_by_public_id()` | Joined non-deleted Flipnote/Creator's Room lookups. | +| `GetFlipnote()` | Compatibility wrapper returning a row or `False`. | +| `GetFlipnotePPM()` / `GetFlipnoteTMB()` | Reads whole PPM bytes or the first `0x6A0` thumbnail/metadata bytes. | +| `list_flipnotes()` | Filtered, sorted, paginated Flipnote query used by both web and DSi lists. | +| `count_flipnotes()` | Counts non-deleted Flipnotes globally or by channel/room. | +| `Newest` | Compatibility property returning `(creator_fsid, filename)` pairs for all current notes. | +| `Views` / `Stars` | Aggregate counters over non-deleted Flipnotes. | +| `AddView()`, `AddDownload()`, `AddStar()` | Atomic counter increments. Star amounts are clamped to 1–65535. | +| `delete_flipnote()` | Sets `deleted=1` for the row. | +| `add_text_comment()` | Normalises/truncates text, resolves a Creator's Room and inserts a text comment. | +| `add_memo_comment()` | Parses PPM metadata, resolves a room, inserts the row, writes `.ppm` and updates `memo_filename`. | +| `get_comment()` / `list_comments()` / `count_comments()` | Non-deleted comment access with Creator's Room display information. | +| `comment_ppm()` | Loads bytes for a memo-type comment when the row and file are present. | +| `recent_ip_logins()` | Returns recent IP/account associations ordered by `last_seen`. The current minimal admin page does not display this helper. | + +### `reximemo_image_formats.py` public surface + +`decode_uploaded_image()` accepts bytes plus a filename/input-format hint and optional raw dimensions. It normalises standard image formats through Pillow and raw Nintendo formats through the local decoders, then returns `DecodedImage`. There is no current OSE web upload route using this function; it remains a reusable format utility. + +`encode_image()` accepts a Pillow image and an output format key. It delegates to the proprietary encoders or Pillow standard writers and returns `EncodedImage`. `image_characteristics()` reports useful source properties such as size/mode/alpha. `ConversionError` is the module's input/format validation exception. + +## Admin and bans + +An account reaches `/admin` only when `is_admin` is 1. OSE's admin page has two jobs: maintain `ip_bans` and soft-delete Flipnotes. + +Banning inserts or updates the exact text IP value and removes any login row for that address. The top-level router checks bans on every request, so a banned address cannot reach the browser pages, DSi resources or static content through the normal root resource. + +Unban deletes the matching row. There are no subnet rules, expiry times, device bans, case queues, strike systems or hidden moderation states. The optional reason is limited to 200 characters and is only shown in the admin table. + +> **Moderation notice.** IP banning and soft deletion are intentionally much simpler than RexiMemo's private moderation system. OSE leaves out device bans, behavioural/bot detection, case tooling and other production anti-abuse logic so those systems are not disclosed in a public release. Exact-IP bans are easy to understand but can affect shared networks and can be bypassed when a user's public address changes. + +## Configuration + +| Setting | Default | Use | +|-----------------|-----------------------------|--------------------------------------------------------------------------------------------------------------------| +| `REXIMEMO_PORT` | `8080` | TCP listen port read by `server.py`. | +| `REXIMEMO_DB` | `database/reximemo.sqlite3` | Alternative SQLite file path read when `DatabaseBackend` is constructed. | +| `docs:enabled` | off | Command-line flag enabling the built-in web documentation route and navigation item. | +| DSi proxy | server IP, port `8080` | Set in the DSi connection settings. OSE is the Hatena HTTP proxy/server; DNS/auth/NAS must be supplied separately. | + +There is no separate configuration file parser in OSE. Channel 1 and the default administrator are database seed data. DSi host names are constants in `reximemo.py`/`hatena.py`. + +> **Configuration notice.** OSE favours explicit, visible defaults over a production secrets/configuration stack. Do not place private keys, service tokens or live production credentials into the repository to compensate for missing configuration features. Keep deployment secrets outside the checkout and change the public seed administrator if the instance is reachable by others. + +## Assets and styling + +The web interface uses `web/static/site.css` and `web/static/RexiMemo_OSE_Logo.png`. The logo is the RexiMemo mark rendered in white with “Open Source Edition” included in the image. The main web accent is `#b13b29`. Fonts are ordinary system/browser fonts; no TTF, OTF, WOFF or WOFF2 files are part of the release. + +The browser layout is intentionally reminiscent of the older RexiMemo site without copying the production interface one-for-one. It uses a coloured header, compact navigation, bordered dark panels, Flipnote cards and responsive grid breakpoints. + +DSi placeholder artwork is generated locally and is not copied from production artwork. Re-run `python3 tools/generate_placeholders.py` after changing the simple icon drawings or encoder. + +## Tests + +The test suite uses the standard library `unittest` runner: + + python3 -m unittest discover -s tests -v + +`test_database.py` creates a temporary SQLite database and checks the default administrator, IP login/logout, stable guest-room ownership and IP bans. + +`test_resources.py` can run even when Twisted is not installed by installing a small in-process Twisted resource/static-file stub. It changes into the repository root, builds the resource tree, checks expected DSi nodes, validates 2048-byte NTFT icons, confirms UGO resources begin with `UGAR`, renders the main web pages and verifies PNG thumbnail output. It also checks that Docs is hidden by default and renderable when a `WebRoot` is explicitly created with documentation enabled. + +`test_release_shape.py` scans the checkout for blocked key/certificate/font extensions, checks that named production-only components are absent, requires exactly three sample PPMs, and checks the OSE web branding and login wording. + +These tests are smoke/shape checks, not a substitute for testing on a physical DSi. The DSi resource tests verify routing and generated binary structures inside Python; they do not emulate the full Flipnote Studio browser/network stack. + +> **Test-scope notice.** Passing this suite does not constitute a security review. In particular it does not prove resistance to hostile traffic, credential attacks, parser abuse, proxy misconfiguration or IP-identity edge cases. The release tests concentrate on keeping OSE small, reproducible, free of private material and compatible with the retained DSi paths. + +## Changing the server + +### Adding a browser page + +Add a path branch to `WebRoot.render_GET()` or `render_POST()`. Use `_page()` for the shared shell. Escape any database/user-provided values with `html.escape()`, using `quote=True` for attribute values. Add CSS to `web/static/site.css`. If the page should be navigable globally, add it to `_page()`'s navigation construction. + +> **Extension notice.** New authenticated, upload or moderation features should not copy assumptions from the simplified OSE identity model without considering their threat model. The repository intentionally omits production security components rather than publishing them, so an extension that needs stronger guarantees should introduce its own suitable authentication, validation, rate limiting and audit behaviour. + +### Adding a DSi endpoint + +For a new top-level DSi resource, place a `.py` file under `hatenadir/ds/v2-xx` and expose a `PyResource` class. The loader mounts an instance automatically at server start. For subpaths, make the resource non-leaf and return child Resource objects from `getChild()`. Keep responses simple and compatible with the DSi browser; use existing helpers for dialogs, forwarding, UGO menus and HTML wrappers. + +### Adding a database field + +Extend the `CREATE TABLE IF NOT EXISTS` schema and provide an upgrade path for existing SQLite files if the field cannot be added transparently. The current schema creation code creates missing tables/indexes but is not a general migration framework. Queries return rows as plain dictionaries through `_row()`. + +### Adding a channel + +Insert a new row in `channels`. The DSi `ch.py` resource accepts any numeric channel ID present in that table. OSE's supplied home menus still post to channel 1 unless their generated URLs are changed. + +### Changing sample data + +Keep SQLite records and files in `database/Creators` consistent. A Flipnote row requires the matching upper-case FSID directory and `.ppm`. The release-shape test currently expects exactly three PPM samples. + +## Deliberately absent systems + +OSE is not the production RexiMemo service. The public repository does not contain Nintendo NAS/DNS/auth gateway code, certificate/private-key material, production session/device management, FSID ownership proof, PPM signature verification, emulator/bot detection, passkeys, profile pictures, Creator's Room themes, experiments, Sudomemo migration, Discord integration, support-centre flows, policy/terms pages, notifications, production API tokens, web PPM upload, advanced moderation cases or the production moderation toolset. + +> **Why the simplification exists.** This is a deliberate release-safety boundary. Security-sensitive production systems, secrets, trust checks and anti-abuse implementation details are excluded so publishing OSE is less likely to disclose or couple itself to RexiMemo's live infrastructure. The public replacements are intentionally smaller and easier to inspect. They exist to reduce the amount of sensitive production material exposed by an open-source release, not because a simpler login, ban or verification mechanism is inherently more secure. + +The remaining code should be read with that boundary in mind. An OSE Creator's Room is an account-or-IP ownership record; a login is an IP-to-account database row; a ban is an exact IP row; a web Flipnote preview is frame zero; and the server accepts the old proxy-style DSi HTTP path rather than reproducing the removed network/authentication stack. + +For public deployment work, the places most likely to need replacement or expansion are the password hashing, IP login identity, proxy/reverse-proxy IP handling, transport/authentication boundary, request-rate controls, database migrations, moderation controls, content limits/backups and operational logging. Those are outside the scope of this repository rather than hidden behind the Docs flag. diff --git a/Hatenatools/License.txt b/Hatenatools/License.txt new file mode 100644 index 0000000..1ce8758 --- /dev/null +++ b/Hatenatools/License.txt @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. \ No newline at end of file diff --git a/Hatenatools/NTFT.py b/Hatenatools/NTFT.py new file mode 100644 index 0000000..2677046 --- /dev/null +++ b/Hatenatools/NTFT.py @@ -0,0 +1,211 @@ +# NTFT.py by pbsds +# AGPL3 licensed +# Python 3 port + +from __future__ import annotations + +import os +import sys +import numpy as np + +try: + from PIL import Image + hasPIL = True +except ImportError: + Image = None + hasPIL = False + + +def AscDec(data, LittleEndian=False): + if isinstance(data, str): + data = data.encode("latin-1") + return int.from_bytes(bytes(data), "little" if LittleEndian else "big") + + +def DecAsc(dec, length=None, LittleEndian=False): + if dec < 0: + raise ValueError("DecAsc does not support negative values") + if length is None: + length = max(1, (dec.bit_length() + 7) // 8) if dec else 0 + if length == 0: + return b"" + dec &= (1 << (length * 8)) - 1 + return dec.to_bytes(length, "little" if LittleEndian else "big") + + +def clamp(value, minimum, maximum): + if value > maximum: + return maximum + if value < minimum: + return minimum + return value + + +class NTFT: + """Reader/writer for Nintendo's raw ARGB1555 NTFT image data.""" + + def __init__(self): + self.Loaded = False + self.Image = None + + def ReadFile(self, path, size): + with open(path, "rb") as f: + return self.Read(f.read(), size) + + def Read(self, data, size): + w, h = size + data = bytes(data) + + psize = [] + for dimension in (w, h): + p = 1 + while 1 << p < dimension: + p += 1 + psize.append(1 << p) + pw, ph = psize + + if pw * ph * 2 != len(data): + print("Invalid sizes") + return False + + self.Image = np.zeros((w, h), dtype=">u4") + for y in range(h): + for x in range(w): + pos = (x + y * pw) * 2 + value = AscDec(data[pos : pos + 2], True) + + a = ((value >> 15) & 0x1) * 0xFF + b = (((value >> 10) & 0x1F) * 0xFF + 15) // 0x1F + g = (((value >> 5) & 0x1F) * 0xFF + 15) // 0x1F + r = ((value & 0x1F) * 0xFF + 15) // 0x1F + self.Image[x, y] = (r << 24) | (g << 16) | (b << 8) | a + + self.Loaded = True + return self + + def WriteFile(self, path): + if not self.Loaded: + return False + with open(path, "wb") as f: + f.write(self.Pack()) + return True + + def Pack(self): + if not self.Loaded or self.Image is None: + return False + + w, h = self.Image.shape + psize = [] + for dimension in (w, h): + p = 1 + while 1 << p < dimension: + p += 1 + psize.append(1 << p) + + out = bytearray() + for y in range(psize[1]): + for x in range(psize[0]): + c = int(self.Image[clamp(x, 0, w - 1), clamp(y, 0, h - 1)]) + r = c >> 24 + g = (c >> 16) & 0xFF + b = (c >> 8) & 0xFF + a = c & 0xFF + + a = 1 if a >= 0x80 else 0 + r = (r * 0x1F + 127) // 0xFF + g = (g * 0x1F + 127) // 0xFF + b = (b * 0x1F + 127) // 0xFF + value = (a << 15) | (b << 10) | (g << 5) | r + out.extend(DecAsc(value, 2, True)) + return bytes(out) + + def SetImage(self, image): + self.Image = image + self.Loaded = True + return self + + +def WriteImage(image, outputPath): + if not hasPIL: + print("Error: PIL not found!") + return False + + out = image.tobytes(order="F") + pil_image = Image.frombytes("RGBA", (len(image), len(image[0])), out) + filetype = outputPath[outputPath.rfind(".") + 1 :] + pil_image.save(outputPath, filetype) + return True + + +def ReadImage(path): + if not hasPIL: + return False + + image = Image.open(path).convert("RGBA") + pixeldata = np.asarray(image, dtype=np.uint8) + w, h = image.size + ret = np.zeros((w, h), dtype=">u4") + for x in range(w): + for y in range(h): + r, g, b, a = (int(value) for value in pixeldata[y, x]) + ret[x, y] = (r << 24) | (g << 16) | (b << 8) | a + return ret + + +def _main(): + print(" == NTFT.py ==") + print(" == by pbsds ==") + print(" == v0.95 ==") + print() + if not hasPIL: + print("PIL not found! Exiting...") + return 1 + + if len(sys.argv) < 2: + print("Usage:") + print(" NTFT.py [ [ ]]") + print() + print("Can convert an NTFT to PNG or the other way around.") + print("32x32 is the normal resolution for button icons in UGO files.") + return 0 + + input_path = sys.argv[1] + encode = not (input_path[-4:].lower() == "ntft" or len(sys.argv) >= 5) + print("Mode: image -> NTFT" if encode else "Mode: NTFT -> image") + + width = height = None + if len(sys.argv) >= 3: + output = sys.argv[2] + if len(sys.argv) >= 5: + if not sys.argv[3].isdigit() or not sys.argv[4].isdigit(): + print("Invalid size input!") + return 1 + width, height = int(sys.argv[3]), int(sys.argv[4]) + if not (width and height) and not encode: + print("Image size not provided!") + return 1 + else: + output = ".".join(input_path.split(".")[:-1]) + (".ntft" if encode else ".png") + + print("Converting...") + try: + if encode: + image = ReadImage(input_path) + if image is False: + return 1 + NTFT().SetImage(image).WriteFile(output) + else: + ntft = NTFT().ReadFile(input_path, (width, height)) + if not ntft: + return 1 + WriteImage(ntft.Image, output) + except OSError as err: + print(err) + return 1 + + print("Done!") + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/Hatenatools/PPM.py b/Hatenatools/PPM.py new file mode 100644 index 0000000..c87f918 --- /dev/null +++ b/Hatenatools/PPM.py @@ -0,0 +1,990 @@ +#!/usr/bin/env python3 +#PPM.py by pbsds; Python 3 port +#AGPL3 licensed +# +#Numpy is required +#PIL is needed to write images to disk +# +#Credits: +# +# - Steven for most of the documentation on DSiBrew and his frame decoding example on his talkpage +# - Remark for help on the 8x8 tiling on the preview image. +# - Jsafive for supplying .tmb files +# - Austin Burk, Midmad on hatena haiku and WDLmaster on hcs64.com for determining the sound codec +# +import sys, wave, re, os, struct # Python 3: audioop was removed in 3.13 +import numpy as np +import subprocess # used for ffmpeg +import tempfile, shutil # temporary directory whilst exporting, shutil to clean up after +import time +try: + from PIL import Image + hasPIL = True +except ImportError: + print("Warning: PIL not found, image extraction won't work!") + hasPIL = False +# Test for ffmpeg by executing ffmpeg -h; an OSError indicates ffmpeg is not installed + +try: + with open(os.devnull,"w") as null: + subprocess.call(["ffmpeg","-h"], + stdout=null, + stderr=null) + hasffmpeg = True +except OSError: + print("Warning: ffmpeg not found, video export unavailable. Please make sure ffmepg is installed and can be accessed on your path.") + hasffmpeg = False + +#helpers: +def AscDec(data, LittleEndian=False):#Converts a byte string into a decimal + if isinstance(data, str): + data = data.encode("latin-1") + return int.from_bytes(bytes(data), "little" if LittleEndian else "big") + +def DecAsc(dec, length=None, LittleEndian=False):#Converts a decimal into bytes of chosen length + if dec < 0: + raise ValueError("DecAsc does not support negative values") + if length is None: + length = max(1, (dec.bit_length() + 7) // 8) if dec else 0 + if length == 0: + return b"" + dec &= (1 << (length * 8)) - 1 + return dec.to_bytes(length, "little" if LittleEndian else "big") + +def AddPadding(i,pad = 0x10):#used mainly for zipaligning offsets + if i % pad != 0: + return i + pad - (i % pad) + return i + +# Python 3.13 removed audioop. This is the IMA/DVI ADPCM decoder used by +# audioop.adpcm2lin(..., 2, None), retaining the original initial state. +_IMA_INDEX_TABLE = (-1, -1, -1, -1, 2, 4, 6, 8) +_IMA_STEP_TABLE = ( + 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34, + 37, 41, 45, 50, 55, 60, 66, 73, 80, 88, 97, 107, 118, 130, 143, 157, + 173, 190, 209, 230, 253, 279, 307, 337, 371, 408, 449, 494, 544, + 598, 658, 724, 796, 876, 963, 1060, 1166, 1282, 1411, 1552, 1707, + 1878, 2066, 2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, + 5358, 5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635, + 13899, 15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, + 32767, +) + +def _decode_ima_adpcm(data): + valpred = 0 + index = 0 + out = bytearray() + for byte in data: + for delta in (byte >> 4, byte & 0x0F): + step = _IMA_STEP_TABLE[index] + vpdiff = step >> 3 + if delta & 4: vpdiff += step + if delta & 2: vpdiff += step >> 1 + if delta & 1: vpdiff += step >> 2 + if delta & 8: + valpred -= vpdiff + else: + valpred += vpdiff + valpred = max(-32768, min(32767, valpred)) + index += _IMA_INDEX_TABLE[delta & 7] + index = max(0, min(88, index)) + out.extend(struct.pack("u4). Will decode it first if not done so already +# - instance.GetFrame(n): Returns the specified frame as a 2D list of uint32 RGBA values (>u4). Only works if ReadFrames was true when reading the PPM file +# - instance.GetSound(n): Returns the decoded sound. Index 0 is the BGM. 1, 2 and 3 are the SFXs. if outputpath is None, the raw mono PCM @ 8184Hz is returned instead as a string +# - instance.Looped: A boolean telling us wether it's looped or not +# - instance.SFXUsage: A list telling us when the different sound effects are used: +# instance.SFXUsage[frame] = [SFX1, SFX2, SFX3] where the SFX's are booleans +# - instance.Framespeed: a value between 1 and 8 telling us the speed +# - instance.BGMFramespeed: The framespeed when the BGM was recorded. used for resampling +# - instance.Frames: A list containing the decoded frames' raw data. Only available if ReadFrames was true when reading the PPM file +# instance.Frames[frame] = [Inverted(boolean), [color #1, color #2], frame] +# color #1 and #2 is a value between 0 and 3, check the palette in instance.GetFrame() to see what they are. +# frame is a 3d list, used like this. frame[layer][x][y] = boolean +# To get a colored image of a frame, use instance.GetFrame() +# - instance.SoundData: A list contaning the raw data of the BGM and the 3 sound effects. Only available if ReadSound was true set to true +class PPM: + def __init__(self,forced_speed=None): + self.Loaded = [False, False, False]#(Meta, Frames, Sound) + self.Frames = None + self.Thumbnail = None + self.RawThumbnail = None + self.SoundData = None + self.forced_speed = forced_speed + def ReadFile(self, path, DecodeThumbnail=False, ReadFrames=True, ReadSound=False):#Load: (thumbnail, frames, sound) + f = open(path, "rb") + ret = self.Read(f.read(), DecodeThumbnail, ReadFrames, ReadSound) + f.close() + return ret + def Read(self, data, DecodeThumbnail=False, ReadFrames=True, ReadSound=False):#Load: (thumbnail, frames, sound) + if data[:4] != b"PARA" or len(data) <= 0x6a0: + return False + + + #Read the header: + AudioOffset = AscDec(data[4:8], True) + 0x6a0 + AudioLenght = AscDec(data[8:12], True) + + self.FrameCount = AscDec(data[12:14], True) + 1 + self.Locked = data[0x10] & 0x01 == 1 + self.ThumbnailFrameIndex = AscDec(data[0x12:0x14], True)#which frame the thumbnnail represents + + self.OriginalAuthorName = data[0x14:0x2A].decode("UTF-16LE").split(u"\0")[0] + self.EditorAuthorName = data[0x2A:0x40].decode("UTF-16LE").split(u"\0")[0] + self.Username = data[0x40:0x56].decode("UTF-16LE").split(u"\0")[0] + + self.OriginalAuthorID = data[0x56:0x5e][::-1].hex().upper() + self.EditorAuthorID = data[0x5E:0x66][::-1].hex().upper()#the last user to save the file + + self.OriginalFilenameC = data[0x66:0x78]#compressed + self.CurrentFilenameC = data[0x78:0x8a]#compressed + self.OriginalFilename = "%s_%s_%s.ppm" % (self.OriginalFilenameC[:3].hex().upper(), self.OriginalFilenameC[3:-2].decode("ascii", "replace"), str(AscDec(self.OriginalFilenameC[-2:], True)).zfill(3)) + self.CurrentFilename = "%s_%s_%s.ppm" % (self.CurrentFilenameC[:3].hex().upper(), self.CurrentFilenameC[3:-2].decode("ascii", "replace"), str(AscDec(self.CurrentFilenameC[-2:], True)).zfill(3)) + + self.PreviousEditAuthorID = data[0x8a:0x92][::-1].hex().upper()#don't know what this really is + + #self.PartialFilenameC = data[0x92:0x9a]#compressed + + self.Date = AscDec(data[0x9a:0x9e], True)#in seconds since midnight 1'st january 2000 + + self.RawThumbnail = data[0xa0:0x6a0] + if DecodeThumbnail: + self.GetThumbnail()#self.Thumbnail[x, y] = uint32 RGBA + + self.Loaded[0] = True + + #read the animation sequence header: + self.Looped = data[0x06A6] >> 1 & 0x01 == 1#Unverified? + AnimationOffset = 0x6a8 + AscDec(data[0x6a0:0x6a4], True) + FrameOffsets = [AnimationOffset + AscDec(data[0x06a8+i*4:0x06a8+i*4+4], True) for i in range(self.FrameCount)] + + #Read the audio header: + self.SFXUsage = [(i&0x1!=0, i&0x2!=0, i&0x4!=0) for i in data[AudioOffset:AudioOffset+self.FrameCount]]#SFXUsage[frame] = (sfx1, sfx2, sfx2) shere sfxX is either 0 or 1 + SoundSize =(AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4) :AddPadding(AudioOffset+self.FrameCount, 4)+ 4], True),#BG music + AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4)+ 4:AddPadding(AudioOffset+self.FrameCount, 4)+ 8], True),#SFX1 + AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4)+ 8:AddPadding(AudioOffset+self.FrameCount, 4)+12], True),#SFX2 + AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4)+12:AddPadding(AudioOffset+self.FrameCount, 4)+16], True))#SFX3 + if self.forced_speed == None: + self.Framespeed = 8 - data[AddPadding(AudioOffset+self.FrameCount, 4) + 16] + else: + self.Framespeed = self.forced_speed + self.BGMFramespeed = 8 - data[AddPadding(AudioOffset+self.FrameCount, 4) + 17]#framespeed when the bgm was recorded + +## self.BGMFramespeed = self.forced_speed + + #Read the Frames: + if ReadFrames: + self.Frames = []#self.Frames[frame] = [inverted(bool), (color1(0-2), color2(0-2)), frame[layer, x, y] = bool] + for i, offset in enumerate(FrameOffsets): + #Read frame header: + Inverted = data[offset] & 0x01 == 0 + + #Reads which color that will be used: + Colors = (data[offset] >> 1 & 0x03, + data[offset] >> 3 & 0x03) + + Frame = self.ExtractFrame(data, offset, self.Frames[i-1][2] if i else None) + + self.Frames.append([Inverted, Colors, Frame]) + + self.Loaded[1] = True + else: + self.Loaded[1] = False + self.Frames = None + + #Read the Audio: + if ReadSound: + self.SoundData = [] + pos = AddPadding(AudioOffset+self.FrameCount+32, 4) + for i in range(4): + self.SoundData.append(data[pos:pos+SoundSize[i]]) + pos += SoundSize[i] + + self.Loaded[2] = True + else: + self.Loaded[2] = False + self.SoundData = None + + #return the results + return self + def WriteFile(self, path):#not implented + pass + #Extracting: + def GetFrame(self, frame):#frame is the index in self.Frames. Only works if ReadFrames was True when reading the PPM file + global FramePalette + if not self.Loaded[1]: return None + + Inverted, Colors, Frame = self.Frames[frame] + + #Defines the palette: + Palette = FramePalette[:] + if Inverted: + Palette[0], Palette[1] = Palette[1], Palette[0] + Color1 = Palette[Colors[0]] + Color2 = Palette[Colors[1]] + + out = np.zeros((256, 192), dtype=">u4") + out[:] = Palette[0] + out[Frame[1]] = Color2 + out[Frame[0]] = Color1 + + return out + def GetThumbnail(self, force=False): + if (self.Thumbnail is None or force):# and self.RawThumbnail: + global ThumbPalette + if not self.RawThumbnail: + return False + + out = np.zeros((64, 48), dtype=">u4") + + #speedup: + palette = ThumbPalette + + #8x8 tiling: + for ty in range(6): + for tx in range(8): + for y in range(8): + for x in range(0,8,2): + #two colors stored in each byte: + byte = self.RawThumbnail[(ty*512+tx*64+y*8+x)//2] + out[x+tx*8 , y+ty*8] = palette[byte & 0xF] + out[x+tx*8+1, y+ty*8] = palette[byte >> 4] + + self.Thumbnail = out + return self.Thumbnail + def GetSound(self, index, outputpath=None):#index 0 is BGM. 1, 2 and 3 is SFX. if outputpath is None, the raw mono PCM @ 8184Hz is returned instead + if self.Loaded[2]: + if self.SoundData[index]: + #reverse nibbles: + data = [] + for i in self.SoundData[index]: + data.append((i&0xF)<< 4 | (i>>4)) + data = bytes(data) + + #4bit ADPCM decode + decoded = _decode_ima_adpcm(data) + + #write to wav: + if outputpath: + f = wave.open(outputpath, "wb") + f.setnchannels(1) + f.setsampwidth(2) + f.setframerate(8192)#possibly 8184, but not a noticable difference anyway + f.writeframes(decoded) + #f.writeframes("".join(out)) + f.close() + + return True + else: + return decoded + else: + return False + #sub functions: (private) + def ExtractFrame(self, data, offset, PrevFrame=None): + #defines line encoding storage: + #Enc1 = [0 for i in range(192)] + #Enc2 = [0 for i in range(192)] + #Defines the frame storage: + #Color1Frame = [[0 for i in range(192)] for i in range(256)] + #Color2Frame = [[0 for i in range(192)] for i in range(256)] + + Encoding = [[], []] + #Frame = [[[False for _ in range(192)] for _ in range(256)] for _ in range(2)] + Frame = np.zeros((2, 256, 192), dtype=np.bool_) + + #Read tags: + NewFrame = data[offset] & 0x80 != 0 + Unknown = data[offset] >> 5 & 0x03 + + offset += 1 + + #WIP - framemove: + FrameMove = [0,0] + if Unknown & 0x2:#doesn't work 100%... + print("FrameMove at offset ",offset-1) + + move_x = AscDec(data[offset+0:offset+1], True) + move_y = AscDec(data[offset+1:offset+2], True) + FrameMove[0] = move_x if move_x <= 127 else move_x-256 + FrameMove[1] = move_y if move_y <= 127 else move_y-256 + offset += 2 + elif Unknown: + print("Unknown tags:",Unknown,"at offset ",offset-1) + + + #read the line encoding of the layers: + for layer in range(2): + for byte in data[offset:offset+48]: + Encoding[layer].append(byte & 0x03) + Encoding[layer].append(byte >> 2 & 0x03) + Encoding[layer].append(byte >> 4 & 0x03) + Encoding[layer].append(byte >> 6 ) + offset += 48 + + #read layers: + for layer in range(2): + for y in range(192): + if Encoding[layer][y] == 0:#Nothing + pass + elif Encoding[layer][y] == 1:#Normal + UseByte = AscDec(data[offset:offset+4]) + offset += 4 + x = 0 + while UseByte & 0xFFFFFFFF: + if UseByte & 0x80000000: + byte = data[offset] + offset += 1 + for _ in range(8): + if byte & 0x01: + Frame[layer][x][y] = True + x += 1 + byte >>= 1 + else: + x += 8 + UseByte <<= 1 + elif Encoding[layer][y] == 2:#Inverted + UseByte = AscDec(data[offset:offset+4]) + offset += 4 + x = 0 + while UseByte&0xFFFFFFFF: + if UseByte & 0x80000000: + byte = data[offset] + offset += 1 + for _ in range(8): + if not byte & 0x01: + Frame[layer, x, y] = True + x += 1 + byte >>= 1 + else: + x += 8 + UseByte <<= 1 + for n in range(256): + #Frame[layer][n][y] = not Frame[layer][n][y] + Frame[layer, n, y] = not Frame[layer, n, y] + elif Encoding[layer][y] == 3:#Raw/full + x = 0 + for _ in range(32): + byte = data[offset] + offset += 1 + for _ in range(8): + if byte & 0x01: + Frame[layer, x, y] = True + x += 1 + byte >>= 1 + + #Merges this frame with the previous frame if NewFrame isn't true: + if not NewFrame and PrevFrame is not None:#maybe optimize this better for numpy... + if FrameMove[0] or FrameMove[1]:#Moves the previous frame if specified: + NewPrevFrame = np.zeros((2, 256, 192), dtype=np.bool_) + + for y in range(192):#this still isn't perfected + for x in range(256): + TempX = x+FrameMove[0] + TempY = y+FrameMove[1] + if 0 <= TempX < 256 and 0 <= TempY < 192: + NewPrevFrame[0, TempX, TempY] = PrevFrame[0, x, y] + NewPrevFrame[1, TempX, TempY] = PrevFrame[1, x, y] + + PrevFrame = NewPrevFrame + + #merge the frames: + Frame = Frame != PrevFrame + + return Frame + +#Class TMB: +# +# With this class you can read a Flipnote TMB file into memory and get its attributes. +# You do so by using either instance.ReadFile() or instance.Read() of your instance of the TMB class. +# If you pass DecodeThumbnail=True to these functions, it will decode thumbnail. +# The TMB file is a file containing the metadata to a corresponding PPM file. +# +# After reading a TMB file, these values are available: +# +# - instance.FrameCount: The number of frames this flipnote consists of +# - instance.Locked: A boolean telling us wether it's locked or not +# - instance.ThumbnailFrameIndex: Which frame the thumbnail is(starting at 0) +# - instance.OriginalAuthorName: The name of the original author in str +# - instance.EditorAuthorName: The name of the (previous?) editing author in str +# - instance.Username: The username in str +# - instance.OriginalAuthorID: The ID of the original author +# - instance.EditorAuthorID: The ID of the last editor, the last user to save the file +# - instance.PreviousEditAuthorID: The ID of the previous editor +# - instance.OriginalFilename: The original filename +# - instance.CurrentFilename: The current filename +# - instance.Date: The date in seconds since January 1'st 2000 +# - instance.GetThumbnail(): The thumbnail stored in a 2D numpy array of uint32 RGBA values (>u4). Will decode it first if not done so already +class TMB: + def __init__(self): + self.Loaded = False + self.Thumbnail = None + self.RawThumbnail = None + def ReadFile(self, path, DecodeThumbnail=False): + f = open(path, "rb") + ret = self.Read(f.read(), DecodeThumbnail) + f.close() + return ret + def Read(self, data, DecodeThumbnail=False): + if data[:4] != b"PARA" or len(data) < 0x6a0: + return False + + #Read the header: + self.AudioOffset = AscDec(data[4:8], True) + 0x6a0#only stored for self.Pack() + self.AudioLenght = AscDec(data[8:12], True)#only stored for self.Pack() + + self.FrameCount = AscDec(data[12:14], True) + 1 + self.Locked = data[0x10] & 0x01 == 1 + self.ThumbnailFrameIndex = AscDec(data[0x12:0x14], True)#which frame is in the thumbnnail + + #self.OriginalAuthorName = u"".join(chr(AscDec(data[0x14+i*2:0x14+i*2+2], True)) for i in range(11)).split(u"\0")[0] + #self.EditorAuthorName = u"".join(chr(AscDec(data[0x2A+i*2:0x2A+i*2+2], True)) for i in range(11)).split(u"\0")[0] + #self.Username = u"".join(chr(AscDec(data[0x40+i*2:0x40+i*2+2], True)) for i in range(11)).split(u"\0")[0] + self.OriginalAuthorName = data[0x14:0x2A].decode("UTF-16LE").split(u"\0")[0] + self.EditorAuthorName = data[0x2A:0x40].decode("UTF-16LE").split(u"\0")[0] + self.Username = data[0x40:0x56].decode("UTF-16LE").split(u"\0")[0] + + self.OriginalAuthorID = data[0x56:0x5e][::-1].hex().upper() + self.EditorAuthorID = data[0x5E:0x66][::-1].hex().upper()#the last user to save the file + + self.OriginalFilenameC = data[0x66:0x78]#compressed + self.CurrentFilenameC = data[0x78:0x8a]#compressed + self.OriginalFilename = "%s_%s_%s.tmb" % (self.OriginalFilenameC[:3].hex().upper(), self.OriginalFilenameC[3:-2].decode("ascii", "replace"), str(AscDec(self.OriginalFilenameC[-2:], True)).zfill(3)) + self.CurrentFilename = "%s_%s_%s.tmb" % (self.CurrentFilenameC[:3].hex().upper(), self.CurrentFilenameC[3:-2].decode("ascii", "replace"), str(AscDec(self.CurrentFilenameC[-2:], True)).zfill(3)) + + self.PreviousEditAuthorID = data[0x8a:0x92][::-1].hex().upper()#don't know what this really is + + self.PartialFilenameC = data[0x92:0x9a]#compressed + + self.Date = AscDec(data[0x9a:0x9e], True)#in seconds since midnight 1'st january 2000 + + self.RawThumbnail = data[0xa0:0x6a0] + if DecodeThumbnail: + self.GetThumbnail()#self.Thumbnail[x, y] = uint32 RGBA + + self.Loaded = True + #return the results + return self + def WriteFile(self, path):#not implented + out = self.Pack() + if out: + f = open(path, "wb") + f.write(out) + f.close() + return True + else: + return False + def Pack(self, ppm=None):#not implented + if not self.Loaded: return False + + #realself = self + #if ppm: self = ppm + + out = [b"PARA",#magic + DecAsc(self.AudioOffset-0x6a0, 4, True),#animation data size + DecAsc(self.AudioLenght, 4, True),#audio data size + DecAsc(self.FrameCount-1, 2, True),#frame count + b"\x24\x00",#unknown + bytes((int(bool(self.Locked)),)), b"\0",#locked + DecAsc(self.ThumbnailFrameIndex, 2, True),#which frame is in the thumbnnail + self.OriginalAuthorName.encode("UTF-16LE") + b"\0\0"*(11-len(self.OriginalAuthorName)),#Original Author Name + self.EditorAuthorName.encode("UTF-16LE") + b"\0\0"*(11-len(self.EditorAuthorName)),#Editor Author Name + self.Username.encode("UTF-16LE") + b"\0\0"*(11-len(self.Username)),#Username + bytes.fromhex(self.OriginalAuthorID)[::-1],#OriginalAuthorID + bytes.fromhex(self.EditorAuthorID)[::-1],#EditorAuthorID + self.OriginalFilenameC,#OriginalFilename + self.CurrentFilenameC,#CurrentFilename + bytes.fromhex(self.PreviousEditAuthorID)[::-1],#EditorAuthorID + self.PartialFilenameC,#PartialFilename + DecAsc(self.Date, 4, True),#Date in seconds + b"\0\0",#padding + self.PackThumbnail()]#thumbnail + + return b"".join(out) + def GetThumbnail(self, force=False): + if (self.Thumbnail is None or force):# and self.RawThumbnail: + global ThumbPalette + if not self.RawThumbnail: + return False + + out = np.zeros((64, 48), dtype=">u4") + + #speedup: + palette = ThumbPalette + + #8x8 tiling: + for ty in range(6): + for tx in range(8): + for y in range(8): + for x in range(0,8,2): + #two colors stored in each byte: + byte = self.RawThumbnail[(ty*512+tx*64+y*8+x)//2] + out[x+tx*8 , y+ty*8] = palette[byte & 0xF] + out[x+tx*8+1, y+ty*8] = palette[byte >> 4] + + self.Thumbnail = out + return self.Thumbnail + def PackThumbnail(self, Exact=True, force=False):#more or less a private function for now + palette = (0xFEFEFEFF,#0 + 0x4F4F4FFF,#1 + 0xFFFFFFFF,#2 + 0x9F9F9FFF,#3 + 0xFF0000FF,#4 + 0x770000FF,#5 + 0xFF7777FF,#6 + 0x00FF00FF,#7- + 0x0000FFFF,#8 + 0x000077FF,#9 + 0x7777FFFF,#A + 0x00FF00FF,#B- + 0xFF00FFFF,#C + 0x00FF00FF,#D- + 0x00FF00FF,#E- + 0x00FF00FF)#F- + + + if self.Thumbnail is None: + return self.RawThumbnail + else: + if Exact: + out = [] + + #8x8 tiling: + for ty in range(6): + for tx in range(8): + for y in range(8): + for x in range(0,8,2): + #two colors stored in each byte: + #pos = 0xa0+(ty*512+tx*64+y*8+x)//2 + p1 = palette.index(int(self.Thumbnail[x+tx*8 , y+ty*8])) + p2 = palette.index(int(self.Thumbnail[x+tx*8+1, y+ty*8])) + out.append(p2<<4 | p1) + + self.RawThumbnail = bytes(out) + return self.RawThumbnail + else: + #not yet implented + return False + +#Function WriteImage: +# +# Writes a 2D numpy array of uint32 RGBA values (>u4) as a image files. +# Designed to work with PPM().Thumbnail or PPM().GetFrame(n) +# +# This function requires the PIl imaging module +def WriteImage(image, outputPath, scale=1): + if not hasPIL: + print("Error: PIL not found!") + return False + #if not image: return False + + # Upscale the image by the scale factor so each original pixel is + # represented by a n-by-n-pixel square in the larger version. + # Any number 1 or smaller is just treated as a scale factor of 1. + if scale > 1: + newImg = np.repeat(np.repeat(image, scale, axis = 0), scale, axis = 1) + else: + newImg = image + + out = newImg.tobytes(order="F") + out = Image.frombytes("RGBA", (len(newImg), len(newImg[0])), out) + + filetype = outputPath[outputPath.rfind(".")+1:] + out.save(outputPath, filetype) + + return True + +def get_metadata(flipnote, filetype=None): + if filetype is None: + filetype = "ppm" if isinstance(flipnote, PPM) else "tmb" + epoch = time.mktime(time.struct_time([2000, 1, 1, 0, 0, 0, 5, 1, -1])) + + meta = { + u"Current filename":flipnote.CurrentFilename[:-3]+filetype, + u"Original filename":flipnote.OriginalFilename[:-3]+filetype, + u"Number of frames":flipnote.FrameCount, + u"Locked":flipnote.Locked, + u"Thumbnail frame index":(flipnote.ThumbnailFrameIndex+1), + u"Original author":flipnote.OriginalAuthorName, + u"Editor author":flipnote.EditorAuthorName, + u"Username":flipnote.Username, + u"Original author ID":flipnote.OriginalAuthorID, + u"Editor author ID":flipnote.EditorAuthorID, + u"Date(seconds since 1'st Jan 2000)":flipnote.Date, + u"Date":time.strftime("%H:%M %d/%m-%Y (faulty)", time.gmtime(epoch+flipnote.Date)), + } + if filetype == "ppm": + meta[u"Frame speed"]=flipnote.Framespeed + meta[u"BGM Frame speed"]=flipnote.BGMFramespeed + meta[u"Looped"]=flipnote.Looped + + return meta + +def DumpFrames(flipnote,directory,scale=1): + for i in range(flipnote.FrameCount): + print("Dumping frame #%i of %i..." % (i+1, flipnote.FrameCount)) + WriteImage(flipnote.GetFrame(i), os.path.join(directory, "frame %s.png" % str(i+1).zfill(3)), scale) + +def DumpSoundFiles(flipnote,directory,raw=False): + for i, data in enumerate(flipnote.SoundData): + if not data: continue + path = os.path.join(directory, ("BGM.wav", "SFX1.wav", "SFX2.wav", "SFX3.wav")[i]) + flipnote.GetSound(i, path) + + if raw: + f = open(path[:-3]+"bin", "wb") + f.write(data) + f.close() + +def DumpSFXUsage(flipnote,directory): + with open(os.path.join(directory, "SFX usage.txt"), "w") as f: + for i, (s1, s2, s3) in enumerate(flipnote.SFXUsage): + f.write("Frame %i:%s%s%s\n" % (i, " SFX1"*s1, " SFX2"*s2, " SFX3"*s3)) + +if __name__ == '__main__': + print(" == PPM.py ==") + print(" == by pbsds ==") + print(" == v1.3 ==") + print() + if len(sys.argv) < 3: + print("Usage:") + print(" PPM.py [] [] [