Port hatena.py to Python 3 and modern Twisted

* Replace Python 2 syntax and string handling with Python 3-compatible code
* Add explicit bytes/text conversion for Twisted requests and responses
* Update request paths, headers, arguments, and child resource names for modern Twisted
* Return byte responses from resource render methods
* Replace deprecated `imp.load_source()` with `importlib.util`
* Generate unique module names for dynamically loaded Hatena endpoints
* Update HTTP response/header handling
* Preserve Nintendo DSi connection-test handling
* Preserve Flipnote Hatena proxy access restrictions
* Preserve region routing for v2-xx, v2-eu, v2-us, and v2-jp
* Preserve UGOXML loading and packing behavior
* Update static file and folder resource handling
* Improve 404 query-string reconstruction
* Add compatibility fallback for modern Twisted client address APIs
* Keep the original dynamic `hatenadir` resource-tree behavior

hatena.py

@@ -1,222 +1,284 @@
-from twisted.web import static, resource
+from __future__ import annotations
-import sys, os, imp
 
-from Hatenatools import *#UGO, PPM, TMB and NTFT
+import hashlib
-#from DB import Database#flipnote handler
+import importlib.util
-#from DB import Database#flipnote handler
+import os
-ServerLog = None#Is set to the log class by server.py
+import sys
+from urllib.parse import urlencode
 
+from twisted.web import resource, static
+
+from Hatenatools import NTFT, PPM, TMB, UGO
+
+ServerLog = None  # Set by server.py
 Silent = False
+
+
+def as_text(value):
+    if isinstance(value, bytes):
+        return value.decode("utf-8", "replace")
+    return str(value)
+
+
+def as_bytes(value):
+    if isinstance(value, bytes):
+        return value
+    return str(value).encode("utf-8")
+
+
+def request_path(request):
+    return as_text(request.path)
+
+
+def request_headers(request):
+    return {as_text(k).lower(): as_text(v) for k, v in request.getAllHeaders().items()}
+
+
+def request_args(request):
+    out = {}
+    for key, values in request.args.items():
+        out[as_text(key)] = [as_text(v) for v in values]
+    return out
+
+
+def client_ip(request):
+    # getClientIP exists on the Twisted versions this project historically used;
+    # getClientAddress is the modern fallback.
+    getter = getattr(request, "getClientIP", None)
+    if getter is not None:
+        value = getter()
+        if value:
+            return as_text(value)
+    address = request.getClientAddress()
+    return as_text(getattr(address, "host", address))
+
+
 def Log(request, path=None, silent=Silent):
-	if not path:
+    if path is None:
-		path = "\"%s\"" % request.path
+        path = '"%s"' % request_path(request)
-	ServerLog.write("%s requested %s" % (request.getClientIP(), path), Silent)
+    if ServerLog is not None:
+        ServerLog.write("%s requested %s" % (client_ip(request), path), silent)
 
-#Responses:
+
-class AccessDenied(resource.Resource):
+class AccessDeniedResource(resource.Resource):
     isLeaf = True
-	def render(self, request):
-		ServerLog.write("%s got 403 when requesting \"%s\"" % (request.getClientIP(), request.path), Silent)
-		print "debug:",request.getAllHeaders()
 
+    def render(self, request):
+        if ServerLog is not None:
+            ServerLog.write(
+                '%s got 403 when requesting "%s"' % (client_ip(request), request_path(request)),
+                Silent,
+            )
         request.setResponseCode(403)
-		return "403 - Access denied\nThis proxy is only allowed to use for Flipnote Hatena for the DSi."
+        request.setHeader(b"content-type", b"text/plain; charset=utf-8")
-AccessDenied = AccessDenied()
+        return b"403 - Access denied\nThis proxy is only allowed to use for Flipnote Hatena for the DSi."
-class NotFound(resource.Resource):
-	isLeaf = True
-	def render(self, request):
-		args = "&".join(("%s=%s" % (i, request.args[i][0]) for i in request.args))
-		path = ("%s?%s" % (request.path, args)) if args else request.path
 
-		ServerLog.write("%s got 404 when requesting \"%s\"" % (request.getClientIP(), path), Silent)
+
+AccessDenied = AccessDeniedResource()
+
+
+class NotFoundResource(resource.Resource):
+    isLeaf = True
+
+    def render(self, request):
+        args = request_args(request)
+        query = urlencode([(k, v) for k, values in args.items() for v in values])
+        path = request_path(request)
+        if query:
+            path += "?" + query
+        if ServerLog is not None:
+            ServerLog.write('%s got 404 when requesting "%s"' % (client_ip(request), path), Silent)
         request.setResponseCode(404)
-		return "404 - Not Found\nThis proxy is only allowed to use for Flipnote Hatena for the DSi."
+        request.setHeader(b"content-type", b"text/plain; charset=utf-8")
-NotFound = NotFound()
+        return b"404 - Not Found\nThis proxy is only allowed to use for Flipnote Hatena for the DSi."
-class ConnectionTest(resource.Resource):#used for people setting up the proxy in their DSi. It's nice to see it's actually working.
+
-	#http://conntest.nintendowifi.net/
+
+NotFound = NotFoundResource()
+
+
+class ConnectionTestResource(resource.Resource):
     isLeaf = True
+
     def render(self, request):
-		ServerLog.write("%s performed a connection test" % request.getClientIP(), True)
+        if ServerLog is not None:
-		request.responseHeaders.setRawHeaders('X-Organization', ['Nintendo'])#i hope this isn't illegal...
+            ServerLog.write("%s performed a connection test" % client_ip(request), True)
-		return '<html><head><title>HTML Page</title></head><body bgcolor="#FFFFFF">This is test.html page</body></html>'
+        request.setHeader(b"X-Organization", b"Nintendo")
-ConnectionTest = ConnectionTest()
+        request.setHeader(b"content-type", b"text/html; charset=utf-8")
+        return b'<html><head><title>HTML Page</title></head><body bgcolor="#FFFFFF">This is test.html page</body></html>'
 
-#Root structure:
+
-class Root(resource.Resource):#filters out non-hantena clients
+ConnectionTest = ConnectionTestResource()
+
+
+class Root(resource.Resource):
     isLeaf = False
-	def __init__(self):
-		resource.Resource.__init__(self)
 
+    def __init__(self):
+        super().__init__()
         self.dsResource = ds()
         self.cssResource = static.File("hatenadir/css/")
         self.imagesResource = static.File("hatenadir/images/")
+
     def getChild(self, name, request):
-		if "x-dsi-sid" not in request.getAllHeaders():
+        name = as_text(name)
-			if "host" in request.getAllHeaders():
+        headers = request_headers(request)
-				if request.getAllHeaders()["host"] == "conntest.nintendowifi.net":
+        if "x-dsi-sid" not in headers:
+            if headers.get("host", "").split(":", 1)[0].lower() == "conntest.nintendowifi.net":
                 return ConnectionTest
             return AccessDenied
 
         if name == "ds":
             return self.dsResource
-		elif name == "css":
+        if name == "css":
             return self.cssResource
-		elif name == "images":
+        if name == "images":
             return self.imagesResource
-		elif name == "":
+        if name == "":
+            return self
+        # The original server intentionally tolerated extra leading path pieces.
         return self
 
-		#return NotFound
-		return self
     def render(self, request):
-		if request.getHost() == "conntest.nintendowifi.net":
+        headers = request_headers(request)
-			return ConnectionTest
+        if headers.get("host", "").split(":", 1)[0].lower() == "conntest.nintendowifi.net":
+            return ConnectionTest.render(request)
         Log(request, "root")
-		return "Welcome to hatena.pbsds.net!\nThis is still in early stages, so please don't expect too much."
+        request.setHeader(b"content-type", b"text/plain; charset=utf-8")
-class ds(resource.Resource):#child of Root
+        return b"Welcome to hatena.pbsds.net!\nThis is still in early stages, so please don't expect too much."
-	isLeaf = False
-	def __init__(self):
-		resource.Resource.__init__(self)
 
+
+class ds(resource.Resource):
+    isLeaf = False
+
+    def __init__(self):
+        super().__init__()
         self.region = UgoRoot()
-		self.regions = ("v2-xx", "v2-eu", "v2-us", "v2-jp")
+        self.regions = {"v2-xx", "v2-eu", "v2-us", "v2-jp"}
+
     def getChild(self, name, request):
+        name = as_text(name)
         if name in self.regions:
             return self.region
-		elif name == "":
+        if name == "":
             return self
         return NotFound
+
     def render(self, request):
         Log(request)
-		return "ds desu"
+        return b"ds desu"
-class UgoRoot(resource.Resource):#child of ds. (v2-xx)
-	isLeaf = False
-	def __init__(self):
-		resource.Resource.__init__(self)
 
+
+class UgoRoot(resource.Resource):
+    isLeaf = False
+
+    def __init__(self):
+        super().__init__()
         LoadHatenadirStructure(self)
+
     def getChild(self, name, request):
-		if name == "":
+        if as_text(name) == "":
             return self
         return NotFound
+
     def render(self, request):
         Log(request, "ugoroot")
-		return "UgoRoot desu"
+        return b"UgoRoot desu"
+
 
-#UgoRoot filestructure:
 class FileResource(resource.Resource):
     isLeaf = True
+
     def __init__(self, filepath, Store=False):
-		resource.Resource.__init__(self)
+        super().__init__()
-		
         self.Store = Store
+        self.file = open(filepath, "rb").read() if Store else filepath
+        self.html = filepath.split(".")[-1][:3].lower() == "htm"
 
-		if Store:
-			f = open(filepath, "rb")
-			self.file = f.read()
-			f.close()
-		else:
-			self.file = filepath
-		
-		self.html = filepath.split(".")[-1][:3] == "htm"
     def render(self, request):
-		path = "/".join(request.path.split("/")[3:])
+        path = "/".join(request_path(request).split("/")[3:])
         Log(request, path)
-		
         if self.html:
-			#request.responseHeaders.setRawHeaders('content-type', ['text/html'])
+            request.setHeader(b"content-type", b"text/html; charset=utf-8")
-			pass
         else:
-			request.responseHeaders.setRawHeaders('content-type', ['text/plain'])
+            request.setHeader(b"content-type", b"text/plain")
-		
         if self.Store:
             return self.file
-		else:
-			#f = open(self.file, "rb")
-			#ret = f.read()
-			#f.close()
-			#return ret
-			
         return static.File(self.file).render(request)
+
+
 class UGOXMLResource(resource.Resource):
     isLeaf = True
+
     def __init__(self, filepath):
-		resource.Resource.__init__(self)
+        super().__init__()
+        ugo = UGO().ReadXML(filepath, False)
+        if not ugo:
+            raise ValueError("Could not parse UGOXML file: %s" % filepath)
+        self.ugofile = ugo.Pack()
 
-		
-		self.ugofile = UGO().ReadXML(filepath, False)
-		self.ugofile = self.ugofile.Pack()
     def render(self, request):
-		path = "/".join(request.path.split("/")[3:])
+        path = "/".join(request_path(request).split("/")[3:])
         Log(request, path)
-		
+        request.setHeader(b"content-type", b"text/plain")
-		#page = int(request.args["page"][0]) if "page" in request.args else 1
-		request.responseHeaders.setRawHeaders('content-type', ['text/plain'])
         return self.ugofile
+
+
 class FolderResource(resource.Resource):
     isLeaf = False
+
     def getChild(self, name, request):
-		if name == "":
+        if as_text(name) == "":
             return self
         return NotFound
+
     def render(self, request):
-		path = "/".join(request.path.split("/")[3:])
+        path = "/".join(request_path(request).split("/")[3:])
         Log(request, path)
-		return "I am a folder, but I'm to lazy to list my contents..."
+        return b"I am a folder, but I'm too lazy to list my contents..."
+
+
+def _load_python_resource(filepath):
+    module_name = "hatena_dynamic_" + hashlib.sha1(os.path.abspath(filepath).encode("utf-8")).hexdigest()
+    spec = importlib.util.spec_from_file_location(module_name, filepath)
+    if spec is None or spec.loader is None:
+        raise ImportError("Could not create import spec for %s" % filepath)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
 def LoadHatenadirStructure(Resource, path=os.path.join("hatenadir", "ds", "v2-xx")):
     for root, dirs, files in os.walk(path):
-		if root <> path: continue#use recursion instead
+        if root != path:
+            continue
+
         for filename in files:
             filetype = filename.split(".")[-1].lower()
-			os.path.join(path, filename)
+            filepath = os.path.join(path, filename)
-			
             if filetype == "ugoxml":
-				Resource.putChild(filename[:-3], UGOXMLResource(os.path.join(path, filename)))
+                Resource.putChild(filename[:-3].encode("utf-8"), UGOXMLResource(filepath))
             elif filetype == "py":
                 try:
-					pyfile = imp.load_source("pyfile", os.path.abspath(os.path.join(path, filename)))
+                    pyfile = _load_python_resource(os.path.abspath(filepath))
-				except ImportError as err:
+                except Exception as err:
-					pyfile = None
+                    print("Error!")
-					print "Error!"
+                    print('Failed to import the python file "%s"' % filepath)
-					print "Failed to import the python file \"%s\"" % os.path.join(path, filename)
+                    print(err)
-					print err
+                    raise
-				
+                Resource.putChild(filename[:-3].encode("utf-8"), pyfile.PyResource())
-				if pyfile:
-					Resource.putChild(filename[:-3], pyfile.PyResource())
             elif filetype == "pyc":
-				pass#ignore
+                continue
             else:
-				Resource.putChild(filename, FileResource(os.path.join(path, filename)))
+                Resource.putChild(filename.encode("utf-8"), FileResource(filepath))
-		for foldername in dirs:# os.path.isdir(i):
+
-			if foldername[:2] <> "__":
+        for foldername in dirs:
+            if not foldername.startswith("__"):
                 folder = FolderResource()
                 LoadHatenadirStructure(folder, os.path.join(path, foldername))
-				Resource.putChild(foldername, folder)
+                Resource.putChild(foldername.encode("utf-8"), folder)
+        break
+
 
-#when loading:
 def Setup():
-	root = Root()
+    return Root()
-	return root
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-