Moved the database to a separate file, getting ready for mondoDB support. Added a more welcoming flipnote

This commit is contained in:
pbsds committed 2014-01-30 19:39:59 +01:00
1 parent a5d5a82ef3
commit 7d0b413961
14 files changed
+218 -161

No files matched your search

+11
View File
@@ -0,0 +1,11 @@
DB_type = "plaintext"#"plaintext" or "mondoDB"
if DB_type == "plaintext":
from database import Database
elif DB_type == "mondoDB":#not yet implemented
from database import Database#hue
else:
import sys
print "Unsupported database type \"%s\"" % DB_type
sys.exit()
View File
Whitespace-only changes.
@@ -1 +1 @@
74CC1F_0D376FC60C9DC_000 4 5 0 0 0 0
74CC1F_0D3AA40F2B3A0_000 0 0 0 0 0 0 0
+152
View File
@@ -0,0 +1,152 @@
from twisted.internet import reactor
import atexit, os
from Hatenatools import TMB
#The database handling flipnote files and info
#this one stores them in plaintext, only works with one server worker
class Database:
def __init__(self):
#read database stuff into memory:
if os.path.exists("database/new_flipnotes.dat"):
f = open("database/new_flipnotes.dat", "rb")#contains the newest 5000 flipnotes
file = f.read()
f.close()
else:
file = ""
if file:
self.Newest = [tuple(i.split("\t")) for i in file.split("\n")]#[i] = [creatorID, filename]
else:
self.Newest = []
self.Creator = {}#to store creator info updates before writing to disk. Creator[id][n] = [filename, views, stars, green stars, red stars, blue stars, purple stars, Channel, Downloads]
#to check if an update is neccesary(caching):
self.new = False#True when new flipnotes has been uploaded
self.Views = 0
self.Stars = 0
self.Downloads = 0
#schtuff
reactor.callLater(60*3, self.flusher)
atexit.register(self.write)
def flusher(self):#Automatically flushes the files every 3 minutes and trims down memory usage
reactor.callLater(60*3, self.flusher)
self.write()
def write(self):
if self.new:
#trim newest:
if len(self.Newest) > 5000:
self.Newest = self.Newest[:5000]
#write to file:
f = open("database/new_flipnotes.dat", "wb")
f.write("\n".join(("\t".join(i) for i in self.Newest)))
f.close()
self.new = False
#write creator changes to file:
for ID in self.Creator.keys():
f = open("database/Creators/%s/flipnotes.dat" % ID, "wb")
f.write("\n".join(("\t".join(map(str, i)) for i in self.Creator[ID])))
f.close()
del self.Creator[ID]
#interface:
def CreatorExists(self, CreatorID):
return os.path.exists("database/Creators/" + CreatorID) or (CreatorID in self.Creator)
def FlipnoteExists(self, CreatorID, filename):
return os.path.exists(self.FlipnotePath(CreatorID, filename))
def GetCreator(self, CreatorID, Store=False):#Returns a list of all the self.GetFlipnote(). "Store" holds it in memory for a while, use this when making changes or reading it often
if CreatorID in self.Creator:
return self.Creator[CreatorID]
else:
if not os.path.exists("database/Creators/" + CreatorID):
return None
f = open("database/Creators/%s/flipnotes.dat" % CreatorID, "rb")
ret = [i.split("\t") for i in f.read().split("\n")]
f.close()
#update to newer format:
#current format = [filename, views, stars, green stars, red stars, blue stars, purple stars, Channel, Downloads]
for i in xrange(len(ret)):
if len(ret[i]) < 9:
filename = ret[i][0]#take this as a give for now
for n, default in enumerate((filename, 0, 0, 0, 0, 0, 0, "", 0)):
if len(ret[i]) <= n:
ret[i].append(default)
if Store:
self.Creator[CreatorID] = ret
return ret
def GetFlipnote(self, CreatorID, filename, Store=False):#returns: [filename, views, stars, green stars, red stars, blue stars, purple stars, Channel, Downloads]
for i in (self.GetCreator(CreatorID, Store) or []):
if i[0] == filename:
return i
return False
def GetFlipnotePPM(self, CreatorID, filename):#the ppm binary data
f = open(self.FlipnotePath(CreatorID, filename), "rb")
ret = f.read()
f.close()
return ret
def GetFlipnoteTMB(self, CreatorID, filename):#the tmb binary data
f = open(self.FlipnotePath(CreatorID, filename), "rb")
ret = f.read(0x6a0)
f.close()
return ret
def AddFlipnote(self, content, Channel=""):#content = ppm binary data
tmb = TMB().Read(content)
if not tmb:
return False
#CreatorID = tmb.Username
CreatorID = tmb.EditorAuthorID
filename = tmb.CurrentFilename[:-4]
del tmb
if self.FlipnoteExists(CreatorID, filename):#already exists
return False
#add to database:
self.new = True
self.Newest.insert(0, (CreatorID, filename))
if not self.GetCreator(CreatorID, True):
self.Creator[CreatorID] = [[filename, 0, 0, 0, 0, 0, 0, Channel, 0]]
else:
self.Creator[CreatorID].append([filename, 0, 0, 0, 0, 0, 0, Channel, 0])
#write flipnote to file:
if not os.path.isdir("database/Creators/" + CreatorID):
os.mkdir("database/Creators/" + CreatorID)
f = open(self.FlipnotePath(CreatorID, filename), "wb")
f.write(content)
f.close()
return CreatorID, filename
def AddView(self, CreatorID, filename):
for i, flipnote in enumerate(self.GetCreator(CreatorID, True) or []):
if flipnote[0] == filename:
self.Creator[CreatorID][i][1] = int(flipnote[1]) + 1
self.Views += 1
return True
return False
def AddStar(self, CreatorID, filename, amount=1):#todo: add support for other colored stars
for i, flipnote in enumerate(self.GetCreator(CreatorID, True) or []):
if flipnote[0] == filename:
self.Creator[CreatorID][i][2] = int(flipnote[2]) + amount
self.Stars += 1
return True
return False
def AddDownload(self, CreatorID, filename):
for i, flipnote in enumerate(self.GetCreator(CreatorID, True) or []):
if flipnote[0] == filename:
self.Creator[CreatorID][i][8] = int(flipnote[8]) + 1
self.Downloads += 1
return True
return False
#internal helpers:
def FlipnotePath(self, CreatorID, filename):#use self.GetFlipnotePPM() instead
return "database/Creators/%s/%s.ppm" % (CreatorID, filename)
Database = Database()#is loaded, yesth!
+1 -1
View File
@@ -1 +1 @@
913304C0CC74CC1F 74CC1F_0D376FC60C9DC_000
913304C0CC74CC1F 74CC1F_0D3AA40F2B3A0_000
+11 -123
View File
@@ -1,8 +1,9 @@
from twisted.web import static, resource
from twisted.internet import reactor
import sys, os, atexit, glob, imp
import sys, os, glob, imp
from Hatenatools import *#UGO, PPM, TMB and NTFT
#from DB import Database#flipnote handler
#from DB import Database#flipnote handler
ServerLog = None#Is set to the log class by server.py
Silent = False
@@ -16,9 +17,10 @@ class AccessDenied(resource.Resource):
isLeaf = True
def render(self, request):
ServerLog.write("%s got 403 when requesting \"%s\"" % (request.getClientIP(), request.path), Silent)
print "debug:",request.getAllHeaders()
request.setResponseCode(403)
return "403 - Access denied\nThis proxy is only for Flipnote Hatena for the DSi."
return "403 - Access denied\nThis proxy is only allowed to use for Flipnote Hatena for the DSi."
AccessDenied = AccessDenied()
class NotFound(resource.Resource):
isLeaf = True
@@ -28,7 +30,7 @@ class NotFound(resource.Resource):
ServerLog.write("%s got 404 when requesting \"%s\"" % (request.getClientIP(), path), Silent)
request.setResponseCode(404)
return "404 - Not Found\nThis proxy is only for Flipnote Hatena for the DSi."
return "404 - Not Found\nThis proxy is only allowed to use for Flipnote Hatena for the DSi."
NotFound = NotFound()
class ConnectionTest(resource.Resource):#used for people setting up the proxy in their DSi. It's nice to see it's actually working.
#http://conntest.nintendowifi.net/
@@ -39,7 +41,6 @@ class ConnectionTest(resource.Resource):#used for people setting up the proxy in
return '<html><head><title>HTML Page</title></head><body bgcolor="#FFFFFF">This is test.html page</body></html>'
ConnectionTest = ConnectionTest()
#Root structure:
class Root(resource.Resource):#filters out non-hantena clients
isLeaf = False
@@ -71,7 +72,7 @@ class Root(resource.Resource):#filters out non-hantena clients
if request.getHost() == "conntest.nintendowifi.net":
return ConnectionTest
Log(request, "root")
return "Welcome to hatena.pbsds.net!\nThis is in early stages, so please don't expect too much."
return "Welcome to hatena.pbsds.net!\nThis is still in early stages, so please don't expect too much."
class ds(resource.Resource):#child of Root
isLeaf = False
def __init__(self):
@@ -93,7 +94,7 @@ class UgoRoot(resource.Resource):#child of ds. (v2-xx)
def __init__(self):
resource.Resource.__init__(self)
LoadHatenaStructure(self)
LoadHatenadirStructure(self)
def getChild(self, name, request):
if name == "":
return self
@@ -161,8 +162,8 @@ class FolderResource(resource.Resource):
def render(self, request):
path = "/".join(request.path.split("/")[3:])
Log(request, path)
return "This is a folder, but I'm to lazy to list the contents..."
def LoadHatenaStructure(Resource, path="hatenadir/ds/v2-xx", root = True):
return "I am a folder, but I'm to lazy to list my contents..."
def LoadHatenadirStructure(Resource, path="hatenadir/ds/v2-xx", root = True):
for i in glob.glob("%s\*" % path):
if os.path.isfile(i):
filename = os.path.basename(i)
@@ -194,125 +195,12 @@ def LoadHatenaStructure(Resource, path="hatenadir/ds/v2-xx", root = True):
elif os.path.isdir(i):
if os.path.basename(i)[:2] <> "__":
folder = FolderResource()
LoadHatenaStructure(folder, os.path.join(path, os.path.basename(i)), False)
LoadHatenadirStructure(folder, os.path.join(path, os.path.basename(i)), False)
Resource.putChild(os.path.basename(i), folder)
#Flipnote handler:
class Database:
def __init__(self):
#read:
f = open("database/new_flipnotes.dat", "rb")#contains the newest 5000 flipnotes
self.Newest = [tuple(i.split("\t")) for i in f.read().split("\n")]#[i] = [creatorID, filename]
self.new = False#True when new flipnotes has been uploaded
f.close()
self.Creator = {}#to store creator info updates before writing to disk. Creator[id][n] = [filename, views, stars, green stars, red stars, blue stars, purple stars, Channel]
reactor.callLater(60*3, self.flusher)
atexit.register(self.write)
#to check if an update is neccesary:
self.Views = 0
self.Stars = 0
def flusher(self):#Automatically flushes the files every 3 minutes and trims down memory usage
reactor.callLater(60*3, self.flusher)
#trim newest:
if len(self.Newest) > 5000:
self.Newest = self.Newest[:5000]
self.write()
def write(self):
if self.new:
f = open("database/new_flipnotes.dat", "wb")
f.write("\n".join(("\t".join(i) for i in self.Newest)))
f.close()
self.new = False
for ID in self.Creator.keys():
f = open("database/Creators/%s/flipnotes.dat" % ID, "wb")
f.write("\n".join(("\t".join(map(str, i)) for i in self.Creator[ID])))
f.close()
del self.Creator[ID]
#helpers:
def FlipnotePath(self, CreatorID, filename):
return "database/Creators/%s/%s" % (CreatorID, filename)
def CreatorExists(self, CreatorID):
return os.path.exists("database/Creators/" + CreatorID) or (CreatorID in self.Creator)
def FlipnoteExists(self, CreatorID, filename):
return os.path.exists("database/Creators/%s/%s.ppm" % (CreatorID, filename))
#interface:
def GetCreator(self, CreatorID, Store=False):#"Store" holds it in memory, use this when making changes or reading it often
if CreatorID in self.Creator:
return self.Creator[CreatorID]
else:
if not os.path.exists("database/Creators/" + CreatorID):
return None
f = open("database/Creators/%s/flipnotes.dat" % CreatorID, "rb")
ret = [i.split("\t") for i in f.read().split("\n")]
f.close()
if Store:
self.Creator[CreatorID] = ret
return ret
def GetFlipnote(self, CreatorID, filename):#[filename, views, stars, green stars, red stars, blue stars, purple stars, Channel]
for i in (self.GetCreator(CreatorID) or []):
if i[0] == filename:
return i
return False
def AddFlipnote(self, content, Channel=""):
tmb = TMB().Read(content)
if not tmb:
return False
#CreatorID = tmb.Username
CreatorID = tmb.EditorAuthorID
filename = tmb.CurrentFilename[:-4]
del tmb
if os.path.exists("database/Creators/%s/%s.ppm" % (CreatorID, filename)):#already exists
return False
#add to database:
self.new = True
self.Newest.insert(0, (CreatorID, filename))
if not self.GetCreator(CreatorID, True):
self.Creator[CreatorID] = [[filename, 0, 0, 0, 0, 0, 0, Channel]]
else:
self.Creator[CreatorID].append([filename, 0, 0, 0, 0, 0, 0, Channel])
#write flipnote to file:
if not os.path.isdir("database/Creators/" + CreatorID):
os.mkdir("database/Creators/" + CreatorID)
f = open("database/Creators/%s/%s.ppm" % (CreatorID, filename), "wb")
f.write(content)
f.close()
return CreatorID, filename
def AddView(self, CreatorID, filename):
for i, flipnote in enumerate(self.GetCreator(CreatorID, True) or []):
if flipnote[0] == filename:
self.Creator[CreatorID][i][1] = int(flipnote[1]) + 1
self.Views += 1
return True
return False
def AddStar(self, CreatorID, filename, amount=1):
for i, flipnote in enumerate(self.GetCreator(CreatorID, True) or []):
if flipnote[0] == filename:
self.Creator[CreatorID][i][2] = int(flipnote[2]) + amount
self.Stars += 1
return True
return False
Database = Database()
#when loading:
def Setup():
print "Setting up hatena...",
root = Root()
print "Done!"
return root
@@ -2,7 +2,8 @@ from twisted.web import resource
from twisted.internet import reactor
import time
from hatena import Database, Log, NotFound
from hatena import Log, NotFound
from DB import Database
from Hatenatools import UGO
#makes a flipnote list of the most seen ones among the recently uploaded
@@ -77,12 +78,9 @@ class PyResource(resource.Resource):
ugo.Items.append(("button", 115, "Previous", "http://flipnote.hatena.com/ds/v2-xx/frontpage/hotmovies.uls?page=%i" % (page-1), ("", ""), None))
#Flipnotes
for creatorid, filename in flipnotes:#[i*50 : i*50+50]:
for creatorid, filename in flipnotes:
stars = str(Database.GetFlipnote(creatorid, filename)[2])
f = open(Database.FlipnotePath(creatorid, filename+".ppm"), "rb")
ugo.Items.append(("button", 3, "", "http://flipnote.hatena.com/ds/v2-xx/movie/%s/%s.ppm" % (creatorid, filename), (stars, "765", "573", "0"), ("bleh", f.read(0x6a0))))
f.close()
ugo.Items.append(("button", 3, "", "http://flipnote.hatena.com/ds/v2-xx/movie/%s/%s.ppm" % (creatorid, filename), (stars, "765", "573", "0"), (filename+".ppm", Database.GetFlipnoteTMB(creatorid, filename))))
#next page
if next:
@@ -2,7 +2,8 @@ from twisted.web import resource
from twisted.internet import reactor
import time
from hatena import Database, Log, NotFound
from hatena import Log, NotFound
from DB import Database
from Hatenatools import UGO
#makes a flipnote list of the most seen ones among the recently uploaded
@@ -14,7 +15,7 @@ class PyResource(resource.Resource):
self.pages = []#10 first pages
self.neweststar = None
self.newestflip = None
reactor.callLater(2, self.Update)#or it'll clash with hotmovies.ugo
reactor.callLater(2, self.Update)#or it'll clash with the others in the terminal
def render(self, request):
page = int(request.args["page"][0]) if "page" in request.args else 1
@@ -79,10 +80,7 @@ class PyResource(resource.Resource):
#Flipnotes
for creatorid, filename in flipnotes:#[i*50 : i*50+50]:
stars = str(Database.GetFlipnote(creatorid, filename)[2])
f = open(Database.FlipnotePath(creatorid, filename+".ppm"), "rb")
ugo.Items.append(("button", 3, "", "http://flipnote.hatena.com/ds/v2-xx/movie/%s/%s.ppm" % (creatorid, filename), (stars, "765", "573", "0"), ("bleh", f.read(0x6a0))))
f.close()
ugo.Items.append(("button", 3, "", "http://flipnote.hatena.com/ds/v2-xx/movie/%s/%s.ppm" % (creatorid, filename), (stars, "765", "573", "0"), (filename+".ppm", Database.GetFlipnoteTMB(creatorid, filename))))
#next page
if next:
@@ -2,7 +2,8 @@ from twisted.web import resource
from twisted.internet import reactor
import time
from hatena import Database, Log, NotFound
from hatena import Log, NotFound
from DB import Database
from Hatenatools import UGO
#makes a flipnote list of the newest flipnotes
@@ -13,7 +14,7 @@ class PyResource(resource.Resource):
self.pages = []#10 first pages
self.newestflip = None
self.Update()
reactor.callLater(4, self.Update)#or it'll clash with the others in the terminal
def render(self, request):
page = int(request.args["page"][0]) if "page" in request.args else 1
@@ -73,10 +74,7 @@ class PyResource(resource.Resource):
#Flipnotes
for creatorid, filename in flipnotes:#[i*50 : i*50+50]:
stars = str(Database.GetFlipnote(creatorid, filename)[2])
f = open(Database.FlipnotePath(creatorid, filename+".ppm"), "rb")
ugo.Items.append(("button", 3, "", "http://flipnote.hatena.com/ds/v2-xx/movie/%s/%s.ppm" % (creatorid, filename), (stars, "765", "573", "0"), ("bleh", f.read(0x6a0))))
f.close()
ugo.Items.append(("button", 3, "", "http://flipnote.hatena.com/ds/v2-xx/movie/%s/%s.ppm" % (creatorid, filename), (stars, "765", "573", "0"), (filename+".ppm", Database.GetFlipnoteTMB(creatorid, filename))))
#next page
if next:
+15 -11
View File
@@ -1,7 +1,8 @@
from twisted.web import resource
import os
from hatena import Database, Log, ServerLog, Silent, NotFound
from hatena import Log, ServerLog, Silent, NotFound
from DB import Database
from Hatenatools import TMB
#The movie folder:
@@ -58,14 +59,12 @@ class CreatorIDFileResource(resource.Resource):
Log(request, path)
#add a view:
Database.AddView(creator, file[:4])
Database.AddView(creator, file[:-4])
#read ppm file:
f = open(Database.FlipnotePath(creator, file), "rb")
data = f.read()
f.close()
data = Database.GetFlipnotePPM(creator, file[:-4])
#send file to client:
#send file to the client:
request.responseHeaders.setRawHeaders('content-type', ['text/plain'])
return data
elif filetype == "info":
@@ -74,7 +73,8 @@ class CreatorIDFileResource(resource.Resource):
request.responseHeaders.setRawHeaders('content-type', ['text/plain'])
return "0\n0\n"#undocumented what it means
elif filetype == "htm":
#maybe cache the details page of Database.Newest
#maybe cache the details page of Database.Newest?
if "mode" in request.args:
if request.args["mode"][0] == "commentshalfsize":
pass
@@ -104,9 +104,11 @@ class CreatorIDFileResource(resource.Resource):
elif filetype == "dl":
path = "/".join(request.path.split("/")[3:])
Log(request, path, True)
#this is POSTed to when it've been downloaded.
#todo: keep count
return "Noted ;)"#muhahaha!
#this is POSTed to when it've been stored to memory.
Database.AddDownload(creator, file[:-3])
return "Noted ;)"
else:
path = "/".join(request.path.split("/")[3:])
ServerLog.write("%s got 403 when requesting %s" % (request.getClientIP(), path), Silent)
@@ -118,7 +120,7 @@ class CreatorIDFileResource(resource.Resource):
flipnote = Database.GetFlipnote(CreatorID, filename)#flipnote = [filename, views, stars, green stars, red stars, blue stars, purple stars, Channel], all probably strings
if not flipnote:
return "This flipnote doesn't exist!"
tmb = TMB().ReadFile(Database.FlipnotePath(CreatorID, filename+".ppm"))
tmb = TMB().Read(Database.GetFlipnoteTMB(CreatorID, filename))
if not tmb:
return "This flipnote is corrupt!"
@@ -159,6 +161,8 @@ class CreatorIDFileResource(resource.Resource):
#Comments:
Comments = "0"
#doto: add original author info too
#add the entries to page:
return DetailsPageTemplate.replace("%%CreatorID%%", CreatorID).replace("%%Filename%%", filename).replace("%%CommentCount%%", Comments).replace("%%Spinoff%%", Spinnoff).replace("%%PageEntries%%", PageEntrySeparator.join(Entries))
+2 -1
View File
@@ -2,7 +2,8 @@ from twisted.web import resource
from twisted.internet import reactor
import time
from hatena import Database, ServerLog, Silent
from hatena import ServerLog, Silent
from DB import Database
from Hatenatools import TMB
#Handle flipnote uploads
+13 -6
View File
@@ -1,15 +1,15 @@
#Settings:
useWSGI = False#not fully tested and WILL NOT support multible instances/workers
useWSGI = False#not fully tested and WILL NOT support multible instances/workers with the plaintext database
port = 8080
#import:
print "Importing modules..."
print "Importing modules...",
from twisted.web import server#filehost
from twisted.internet import reactor
if useWSGI: from twisted.application import internet, service
import sys, time, os, atexit
print "Done!"
#Logging
class Log:
@@ -83,15 +83,21 @@ class Log:
Print = write
Log = Log()
#init database:
print "Initializing flipnote database...",
import DB
print "Done!"
#Setup hatena server:
print "Setting up flipnote database..."
print "Setting up hatena site...",
import hatena
hatena.ServerLog = Log
print "Setting up hatena site structure..."
site = server.Site(hatena.Setup())
print "Done!"
#make the hatena server accept proxy connections:
print "Setting up proxy hack..."
print "Setting up proxy hack...",
silent = True
old_lineReceived = site.protocol.lineReceived
def lineReceived(self, line):
@@ -104,6 +110,7 @@ def lineReceived(self, line):
if not silent: print line
old_lineReceived(self, line)
site.protocol.lineReceived = lineReceived
print "Done!"
#run!
print "Server start!\n"