rekushi/Hatenatools3
1
0 Fork 0
mirror of https://github.com/RekuNote/Hatenatools3.git synced 2026-10-30 00:09:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Initial release of Hatenatools3

Browse Source 
Python 3 port of Hatenatools, optimised for compatibility with RexiMemo-org/hatena-server-py3.

Includes Python 3 compatibility fixes, restored functionality from the original Hatenatools, and additional porting/behavior fixes.
This commit is contained in:
Rekushi authored and GitHub committed 2026-09-02 23:50:33 +01:00
1 parent d5bb128b76
commit c0b38d0d9a
8 files changed
+829 -888
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

NTFT.py
+136 -229
View File
@@ -1,305 +1,212 @@
# NTFT.py by pbsds # NTFT.py by pbsds
# Hatenatools3 Python 3 port by RekuNote aka rekushiiiiii
# AGPL3 licensed # AGPL3 licensed
# # Python 3 port
#PIL is required to read and write images to disk
# from __future__ import annotations
#Credits:
# import os
# -The guys behind TiledGGD. This sped up my work a lit. import sys
# -Jsafive for supplying .ugo files import numpy as np
#
import sys, os, numpy as np
try: try:
import Image from PIL import Image
hasPIL = True hasPIL = True
except ImportError: except ImportError:
Image = None
hasPIL = False hasPIL = False
#helpers def AscDec(data, LittleEndian=False):
def AscDec(ascii, LittleEndian=False):#Converts a ascii string into a decimal if isinstance(data, str):
ret = 0 data = data.encode("latin-1")
l = map(ord, ascii) return int.from_bytes(bytes(data), "little" if LittleEndian else "big")
if LittleEndian: l.reverse()
for i in l:
ret = (ret<<8) | i
return ret
def DecAsc(dec, length=None, LittleEndian=False):#Converts a decimal into an ascii string of chosen length
out = []
while dec <> 0:
out.insert(0, dec&0xFF)
dec >>= 8
#"".join(map(chr, out))
if length:
if len(out) > length:
#return "".join(map(chr, out[-length:]))
out = out[-length:]
if len(out) < length:
#return "".join(map(chr, [0]*(length-len(out)) + out))
out = [0]*(length-len(out)) + out
if LittleEndian: out.reverse() def DecAsc(dec, length=None, LittleEndian=False):
return "".join(map(chr, out)) if dec < 0:
def clamp(value, min, max): raise ValueError("DecAsc does not support negative values")
if value > max: return max if length is None:
if value < min: return min length = max(1, (dec.bit_length() + 7) // 8) if dec else 0
if length == 0:
return b""
dec &= (1 << (length * 8)) - 1
return dec.to_bytes(length, "little" if LittleEndian else "big")
def clamp(value, minimum, maximum):
if value > maximum:
return maximum
if value < minimum:
return minimum
return value return value
#Class NTFT:
#
# The NTFT image format stores RGB values as 5 bits each: a value between 0 and 31.
# It has 1 bit of transparancy, which means its either vidssible or invisible. No gradients.
#
# How to use:
#
# Converting to NTFT file:
#
# image = ReadImage(input_path)
# NTFT().SetImage(image).WriteFile(output_path)
#
# Reading NTFT file:
#
# ntft = NTFT().ReadFile(input_path, (width, height))
# WriteImage(ntft.Image, output_path)
#
class NTFT: class NTFT:
"""Reader/writer for Nintendo's raw ARGB1555 NTFT image data."""
def __init__(self): def __init__(self):
self.Loaded = False self.Loaded = False
self.Image = None
def ReadFile(self, path, size): def ReadFile(self, path, size):
f = open(path, "rb") with open(path, "rb") as f:
ret = self.Read(f.read(), size) return self.Read(f.read(), size)
f.close()
return ret def Read(self, data, size):
def Read(self, data, (w, h)): w, h = size
#the actual stored data is a image with the sizes padded to the nearest power of 2. The image is then clipped out from it. data = bytes(data)
psize = [] psize = []
for i in (w, h): for dimension in (w, h):
p = 1 p = 1
while 1<<p < i: while 1 << p < dimension:
p += 1 p += 1
psize.append(1 << p) psize.append(1 << p)
pw, ph = psize pw, ph = psize
#check if it fits the file: if pw * ph * 2 != len(data):
if pw*ph*2 <> len(data): print("Invalid sizes")
print "Invalid sizes"
return False return False
#JUST DO IT!
#self.Image = [[None for _ in xrange(h)] for _ in xrange(w)]
self.Image = np.zeros((w, h), dtype=">u4") self.Image = np.zeros((w, h), dtype=">u4")
for y in xrange(h): for y in range(h):
for x in xrange(w): for x in range(w):
pos = (x + y * pw) * 2 pos = (x + y * pw) * 2
byte = AscDec(data[pos:pos+2], True) value = AscDec(data[pos : pos + 2], True)
#ARGB1555 -> RGBA8: a = ((value >> 15) & 0x1) * 0xFF
a = (byte >> 15 ) * 0xFF b = (((value >> 10) & 0x1F) * 0xFF + 15) // 0x1F
b = (byte >> 10 & 0x1F) * 0xFF / 0x1F g = (((value >> 5) & 0x1F) * 0xFF + 15) // 0x1F
g = (byte >> 5 & 0x1F) * 0xFF / 0x1F r = ((value & 0x1F) * 0xFF + 15) // 0x1F
r = (byte & 0x1F) * 0xFF / 0x1F self.Image[x, y] = (r << 24) | (g << 16) | (b << 8) | a
#self.Image[x][y] = (r<<24) | (g<<16) | (b<<8) | a#RGBA8
self.Image[x, y] = (r<<24) | (g<<16) | (b<<8) | a#RGBA8
self.Loaded = True self.Loaded = True
return self return self
def WriteFile(self, path): def WriteFile(self, path):
if self.Loaded:
f = open(path, "wb")
f.write(self.Pack())
f.close()
return True
else:
return False
def Pack(self):
if not self.Loaded: if not self.Loaded:
return False return False
with open(path, "wb") as f:
f.write(self.Pack())
return True
def Pack(self):
if not self.Loaded or self.Image is None:
return False
#h = len(self.Image[0])
#w = len(self.Image)
w, h = self.Image.shape w, h = self.Image.shape
#the actual stored data is a image with the sizes padded to the nearest power of 2
psize = [] psize = []
for i in (w, h): for dimension in (w, h):
p = 1 p = 1
while 1<<p < i: while 1 << p < dimension:
p += 1 p += 1
psize.append(1 << p) psize.append(1 << p)
out = [] out = bytearray()
for y in xrange(psize[1]): for y in range(psize[1]):
for x in xrange(psize[0]): for x in range(psize[0]):
#read c = int(self.Image[clamp(x, 0, w - 1), clamp(y, 0, h - 1)])
#c = self.Image[clamp(x, 0, w-1)][clamp(y, 0, h-1)]
c = self.Image[clamp(x, 0, w-1), clamp(y, 0, h-1)]
r = c >> 24 r = c >> 24
g = (c >> 16) & 0xFF g = (c >> 16) & 0xFF
b = (c >> 8) & 0xFF b = (c >> 8) & 0xFF
a = c & 0xFF a = c & 0xFF
#convert
a = 1 if a >= 0x80 else 0 a = 1 if a >= 0x80 else 0
r = r * 0x1F / 0xFF r = (r * 0x1F + 127) // 0xFF
g = g * 0x1F / 0xFF g = (g * 0x1F + 127) // 0xFF
b = b * 0x1F / 0xFF b = (b * 0x1F + 127) // 0xFF
value = (a << 15) | (b << 10) | (g << 5) | r
out.extend(DecAsc(value, 2, True))
return bytes(out)
#store def SetImage(self, image):
out.append(DecAsc((a<<15) | (b<<10) | (g<<5) | r, 2, True)) self.Image = image
return "".join(out)
def SetImage(self, Image):
self.Image = Image
self.Loaded = True self.Loaded = True
return self return self
#Function WriteImage:
#
# Writes a 2D array of uint32 RGBA values as a image file.
# Designed to work with NTFT.Image
#
# This function requires the PIl imaging module
def WriteImage(image, outputPath): def WriteImage(image, outputPath):
if not hasPIL: if not hasPIL:
print "Error: PIL not found!" print("Error: PIL not found!")
return False return False
#if not image: return False
out = image.tostring("F")
# out = []
# for y in xrange(len(image[0])):
# for x in xrange(len(image)):
# out.append(DecAsc(image[x][y], 4))
out = Image.fromstring("RGBA", (len(image), len(image[0])), out)
out = image.tobytes(order="F")
pil_image = Image.frombytes("RGBA", (len(image), len(image[0])), out)
filetype = outputPath[outputPath.rfind(".") + 1 :] filetype = outputPath[outputPath.rfind(".") + 1 :]
out.save(outputPath, filetype) pil_image.save(outputPath, filetype)
return True return True
#Function ReadImage:
#
# Returns a 2D list of uint32 RGBA values of the image file.
# This can be passed into NTFT().SetImage()
#
# This function requires the PIl imaging module
def ReadImage(path):#TODO: make it support numpy
if not hasPIL: return False
image = Image.open(path) def ReadImage(path):
pixeldata = image.getdata() if not hasPIL:
return False
image = Image.open(path).convert("RGBA")
pixeldata = np.asarray(image, dtype=np.uint8)
w, h = image.size w, h = image.size
if len(pixeldata[0]) < 4:
def Combine((r, g, b)):
return (r << 24) | (g << 16) | (b << 8) | 0xFF
else:
def Combine((r, g, b, a)):
return (r << 24) | (g << 16) | (b << 8) | a
#ret = []
ret = np.zeros((w, h), dtype=">u4") ret = np.zeros((w, h), dtype=">u4")
for x in xrange(w): for x in range(w):
#line = [] for y in range(h):
for y in xrange(h): r, g, b, a = (int(value) for value in pixeldata[y, x])
ret[x, y] = Combine(pixeldata[y*w + x])#maybe make a more numpy efficient way? ret[x, y] = (r << 24) | (g << 16) | (b << 8) | a
#line.append(Combine(pixeldata[y*w + x]))
#ret.append(line)
return ret return ret
def _main():
#testing: print(" == NTFT.py ==")
# i = NTFT().ReadFile("NTFTtests/kaeru.ntft", (36, 30)) print(" == by pbsds ==")
# WriteImage(i.Image, "NTFTtests/kaeru.png") print(" == v0.95 ==")
print()
# i = NTFT().ReadFile("NTFTtests/News.ntft", (32, 32))
# WriteImage(i.Image, "NTFTtests/News.png")
# i = NTFT().ReadFile("NTFTtests/Special Room.ntft", (32, 32))
# WriteImage(i.Image, "NTFTtests/Special Room.png")
#i = NTFT()
#i.Loaded = True
#i.Image = ReadImage("NTFTtests/geh.png")
#i.WriteFile("NTFTtests/geh.ntft")
if __name__ == "__main__":
print " == NTFT.py =="
print " == by pbsds =="
print " == v0.95 =="
print
if not hasPIL: if not hasPIL:
print "PIL not found! Exiting..." print("PIL not found! Exiting...")
sys.exit() return 1
if len(sys.argv) < 2: if len(sys.argv) < 2:
print "Usage:" print("Usage:")
print " NTFT.py <input> [<output> [<width> <height>]]" print(" NTFT.py <input> [<output> [<width> <height>]]")
print "" print()
print "Can convert a NTFT to PNG or the other way around." print("Can convert an NTFT to PNG or the other way around.")
print "if <output> isn't specified it will be set to <input> with an another extension" print("32x32 is the normal resolution for button icons in UGO files.")
print "" return 0
print "The NTFT file contain only the colordata, so it's up to the user to find or"
print "store the resolution of the image. <width> and <height> is required"
print "to convert a NTFT file to a image."
print "32x32 is the normal resolution for button icons in UGO files."
sys.exit()
input = sys.argv[1] input_path = sys.argv[1]
encode = not (input_path[-4:].lower() == "ntft" or len(sys.argv) >= 5)
if input[-4:].lower() == "ntft" or len(sys.argv) >= 5: print("Mode: image -> NTFT" if encode else "Mode: NTFT -> image")
print "Mode: NTFT -> image"
Encode = False
else:
print "Mode: image -> NTFT"
Encode = True#if false it'll decode
width = height = None
if len(sys.argv) >= 3: if len(sys.argv) >= 3:
output = sys.argv[2] output = sys.argv[2]
width, height = None, None
if len(sys.argv) >= 5: if len(sys.argv) >= 5:
if (not sys.argv[3].isdigit()) or (not sys.argv[4].isdigit()): if not sys.argv[3].isdigit() or not sys.argv[4].isdigit():
print "Invalid size input!" print("Invalid size input!")
sys.exit() return 1
width = int(sys.argv[3]) width, height = int(sys.argv[3]), int(sys.argv[4])
height = int(sys.argv[4]) if not (width and height) and not encode:
print("Image size not provided!")
if not (width and height) and not Encode: return 1
print "Image size not provided!"
sys.exit()
else: else:
output = ".".join(input.split(".")[:-1]) + (".ntft" if Encode else ".png") output = ".".join(input_path.split(".")[:-1]) + (".ntft" if encode else ".png")
print "Converting..." print("Converting...")
if Encode:
try: try:
image = ReadImage(input) if encode:
except IOError as err: image = ReadImage(input_path)
print err if image is False:
sys.exit() return 1
NTFT().SetImage(image).WriteFile(output)
i = NTFT()
i.Loaded = True
i.Image = image
i.WriteFile(output)
else: else:
try: ntft = NTFT().ReadFile(input_path, (width, height))
ntft = NTFT().ReadFile(input, (width, height)) if not ntft:
except IOError as err: return 1
print err
sys.exit()
if not ntft:#eeror message already printed
sys.exit()
WriteImage(ntft.Image, output) WriteImage(ntft.Image, output)
print "Done!" except OSError as err:
print(err)
return 1
print("Done!")
return 0
if __name__ == "__main__":
raise SystemExit(_main())
PPM.py
+237 -212
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env python #!/usr/bin/env python3
#PPM.py by pbsds for python 2.7 #PPM.py by pbsds; Hatenatools3 Python 3 port by RekuNote aka rekushiiiiii
#AGPL3 licensed #AGPL3 licensed
# #
#Numpy is required #Numpy is required
@@ -12,7 +12,7 @@
# - Jsafive for supplying .tmb files # - Jsafive for supplying .tmb files
# - Austin Burk, Midmad on hatena haiku and WDLmaster on hcs64.com for determining the sound codec # - Austin Burk, Midmad on hatena haiku and WDLmaster on hcs64.com for determining the sound codec
# #
import sys, wave, audioop, re, os #needs os and time aswell in CMD mode (UPDATE 2018/07/04: os used to get devnull for discarding subprocess output) import sys, wave, re, os, struct # Python 3: audioop was removed in 3.13
import numpy as np import numpy as np
import subprocess # used for ffmpeg import subprocess # used for ffmpeg
import tempfile, shutil # temporary directory whilst exporting, shutil to clean up after import tempfile, shutil # temporary directory whilst exporting, shutil to clean up after
@@ -21,7 +21,7 @@ try:
from PIL import Image from PIL import Image
hasPIL = True hasPIL = True
except ImportError: except ImportError:
print "Warning: PIL not found, image extraction won't work!" print("Warning: PIL not found, image extraction won't work!")
hasPIL = False hasPIL = False
# Test for ffmpeg by executing ffmpeg -h; an OSError indicates ffmpeg is not installed # Test for ffmpeg by executing ffmpeg -h; an OSError indicates ffmpeg is not installed
@@ -32,39 +32,65 @@ try:
stderr=null) stderr=null)
hasffmpeg = True hasffmpeg = True
except OSError: except OSError:
print "Warning: ffmpeg not found, video export unavailable. Please make sure ffmepg is installed and can be accessed on your path." print("Warning: ffmpeg not found, video export unavailable. Please make sure ffmepg is installed and can be accessed on your path.")
hasffmpeg = False hasffmpeg = False
#helpers: #helpers:
def AscDec(ascii, LittleEndian=False):#Converts a ascii string into a decimal def AscDec(data, LittleEndian=False):#Converts a byte string into a decimal
ret = 0 if isinstance(data, str):
l = map(ord, ascii) data = data.encode("latin-1")
if LittleEndian: l.reverse() return int.from_bytes(bytes(data), "little" if LittleEndian else "big")
for i in l:
ret = (ret<<8) | i
return ret
def DecAsc(dec, length=None, LittleEndian=False):#Converts a decimal into an ascii string of chosen length
out = []
while dec <> 0:
out.insert(0, dec&0xFF)
dec >>= 8
#"".join(map(chr, out))
if length: def DecAsc(dec, length=None, LittleEndian=False):#Converts a decimal into bytes of chosen length
if len(out) > length: if dec < 0:
#return "".join(map(chr, out[-length:])) raise ValueError("DecAsc does not support negative values")
out = out[-length:] if length is None:
if len(out) < length: length = max(1, (dec.bit_length() + 7) // 8) if dec else 0
#return "".join(map(chr, [0]*(length-len(out)) + out)) if length == 0:
out = [0]*(length-len(out)) + out return b""
dec &= (1 << (length * 8)) - 1
return dec.to_bytes(length, "little" if LittleEndian else "big")
if LittleEndian: out.reverse()
return "".join(map(chr, out))
def AddPadding(i,pad = 0x10):#used mainly for zipaligning offsets def AddPadding(i,pad = 0x10):#used mainly for zipaligning offsets
if i % pad <> 0: if i % pad != 0:
return i + pad - (i % pad) return i + pad - (i % pad)
return i return i
# Python 3.13 removed audioop. This is the IMA/DVI ADPCM decoder used by
# audioop.adpcm2lin(..., 2, None), retaining the original initial state.
_IMA_INDEX_TABLE = (-1, -1, -1, -1, 2, 4, 6, 8)
_IMA_STEP_TABLE = (
7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34,
37, 41, 45, 50, 55, 60, 66, 73, 80, 88, 97, 107, 118, 130, 143, 157,
173, 190, 209, 230, 253, 279, 307, 337, 371, 408, 449, 494, 544,
598, 658, 724, 796, 876, 963, 1060, 1166, 1282, 1411, 1552, 1707,
1878, 2066, 2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871,
5358, 5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635,
13899, 15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794,
32767,
)
def _decode_ima_adpcm(data):
valpred = 0
index = 0
out = bytearray()
for byte in data:
for delta in (byte >> 4, byte & 0x0F):
step = _IMA_STEP_TABLE[index]
vpdiff = step >> 3
if delta & 4: vpdiff += step
if delta & 2: vpdiff += step >> 1
if delta & 1: vpdiff += step >> 2
if delta & 8:
valpred -= vpdiff
else:
valpred += vpdiff
valpred = max(-32768, min(32767, valpred))
index += _IMA_INDEX_TABLE[delta & 7]
index = max(0, min(88, index))
out.extend(struct.pack("<h", valpred))
return bytes(out)
#palettes: #palettes:
FramePalette = [0xFFFFFFFF,0x000000FF,0xFF0000FF,0x0000FFFF] FramePalette = [0xFFFFFFFF,0x000000FF,0xFF0000FF,0x0000FFFF]
ThumbPalette = (0xFEFEFEFF,#0 ThumbPalette = (0xFEFEFEFF,#0
@@ -98,9 +124,9 @@ ThumbPalette = (0xFEFEFEFF,#0
# - instance.FrameCount: The number of frames this flipnote consists of # - instance.FrameCount: The number of frames this flipnote consists of
# - instance.Locked: A boolean telling us wether it's locked or not # - instance.Locked: A boolean telling us wether it's locked or not
# - instance.ThumbnailFrameIndex: Which frame the thumbnail is(starting at 0) # - instance.ThumbnailFrameIndex: Which frame the thumbnail is(starting at 0)
# - instance.OriginalAuthorName: The name of the original author in unicode # - instance.OriginalAuthorName: The name of the original author in str
# - instance.EditorAuthorName: The name of the (previous?) editing author in unicode # - instance.EditorAuthorName: The name of the (previous?) editing author in str
# - instance.Username: The username in unicode # - instance.Username: The username in str
# - instance.OriginalAuthorID: The ID of the original author # - instance.OriginalAuthorID: The ID of the original author
# - instance.EditorAuthorID: The ID of the last editor, the last user to save the file # - instance.EditorAuthorID: The ID of the last editor, the last user to save the file
# - instance.PreviousEditAuthorID: The ID of the previous editor # - instance.PreviousEditAuthorID: The ID of the previous editor
@@ -135,7 +161,7 @@ class PPM:
f.close() f.close()
return ret return ret
def Read(self, data, DecodeThumbnail=False, ReadFrames=True, ReadSound=False):#Load: (thumbnail, frames, sound) def Read(self, data, DecodeThumbnail=False, ReadFrames=True, ReadSound=False):#Load: (thumbnail, frames, sound)
if data[:4] <> "PARA" or len(data) <= 0x6a0: if data[:4] != b"PARA" or len(data) <= 0x6a0:
return False return False
@@ -144,22 +170,22 @@ class PPM:
AudioLenght = AscDec(data[8:12], True) AudioLenght = AscDec(data[8:12], True)
self.FrameCount = AscDec(data[12:14], True) + 1 self.FrameCount = AscDec(data[12:14], True) + 1
self.Locked = ord(data[0x10]) & 0x01 == 1 self.Locked = data[0x10] & 0x01 == 1
self.ThumbnailFrameIndex = AscDec(data[0x12:0x14], True)#which frame the thumbnnail represents self.ThumbnailFrameIndex = AscDec(data[0x12:0x14], True)#which frame the thumbnnail represents
self.OriginalAuthorName = data[0x14:0x2A].decode("UTF-16LE").split(u"\0")[0] self.OriginalAuthorName = data[0x14:0x2A].decode("UTF-16LE").split(u"\0")[0]
self.EditorAuthorName = data[0x2A:0x40].decode("UTF-16LE").split(u"\0")[0] self.EditorAuthorName = data[0x2A:0x40].decode("UTF-16LE").split(u"\0")[0]
self.Username = data[0x40:0x56].decode("UTF-16LE").split(u"\0")[0] self.Username = data[0x40:0x56].decode("UTF-16LE").split(u"\0")[0]
self.OriginalAuthorID = data[0x56:0x5e][::-1].encode("HEX").upper() self.OriginalAuthorID = data[0x56:0x5e][::-1].hex().upper()
self.EditorAuthorID = data[0x5E:0x66][::-1].encode("HEX").upper()#the last user to save the file self.EditorAuthorID = data[0x5E:0x66][::-1].hex().upper()#the last user to save the file
self.OriginalFilenameC = data[0x66:0x78]#compressed self.OriginalFilenameC = data[0x66:0x78]#compressed
self.CurrentFilenameC = data[0x78:0x8a]#compressed self.CurrentFilenameC = data[0x78:0x8a]#compressed
self.OriginalFilename = "%s_%s_%s.ppm" % (self.OriginalFilenameC[:3].encode("HEX").upper(), self.OriginalFilenameC[3:-2], str(AscDec(self.OriginalFilenameC[-2:], True)).zfill(3)) self.OriginalFilename = "%s_%s_%s.ppm" % (self.OriginalFilenameC[:3].hex().upper(), self.OriginalFilenameC[3:-2].decode("ascii", "replace"), str(AscDec(self.OriginalFilenameC[-2:], True)).zfill(3))
self.CurrentFilename = "%s_%s_%s.ppm" % (self.CurrentFilenameC[:3].encode("HEX").upper(), self.CurrentFilenameC[3:-2], str(AscDec(self.CurrentFilenameC[-2:], True)).zfill(3)) self.CurrentFilename = "%s_%s_%s.ppm" % (self.CurrentFilenameC[:3].hex().upper(), self.CurrentFilenameC[3:-2].decode("ascii", "replace"), str(AscDec(self.CurrentFilenameC[-2:], True)).zfill(3))
self.PreviousEditAuthorID = data[0x8a:0x92][::-1].encode("HEX").upper()#don't know what this really is self.PreviousEditAuthorID = data[0x8a:0x92][::-1].hex().upper()#don't know what this really is
#self.PartialFilenameC = data[0x92:0x9a]#compressed #self.PartialFilenameC = data[0x92:0x9a]#compressed
@@ -172,21 +198,21 @@ class PPM:
self.Loaded[0] = True self.Loaded[0] = True
#read the animation sequence header: #read the animation sequence header:
self.Looped = ord(data[0x06A6]) >> 1 & 0x01 == 1#Unverified? self.Looped = data[0x06A6] >> 1 & 0x01 == 1#Unverified?
AnimationOffset = 0x6a8 + AscDec(data[0x6a0:0x6a4], True) AnimationOffset = 0x6a8 + AscDec(data[0x6a0:0x6a4], True)
FrameOffsets = [AnimationOffset + AscDec(data[0x06a8+i*4:0x06a8+i*4+4], True) for i in xrange(self.FrameCount)] FrameOffsets = [AnimationOffset + AscDec(data[0x06a8+i*4:0x06a8+i*4+4], True) for i in range(self.FrameCount)]
#Read the audio header: #Read the audio header:
self.SFXUsage = [(i&0x1<>0, i&0x2<>0, i&0x4<>0) for i in map(ord, data[AudioOffset:AudioOffset+self.FrameCount])]#SFXUsage[frame] = (sfx1, sfx2, sfx2) shere sfxX is either 0 or 1 self.SFXUsage = [(i&0x1!=0, i&0x2!=0, i&0x4!=0) for i in data[AudioOffset:AudioOffset+self.FrameCount]]#SFXUsage[frame] = (sfx1, sfx2, sfx2) shere sfxX is either 0 or 1
SoundSize =(AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4) :AddPadding(AudioOffset+self.FrameCount, 4)+ 4], True),#BG music SoundSize =(AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4) :AddPadding(AudioOffset+self.FrameCount, 4)+ 4], True),#BG music
AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4)+ 4:AddPadding(AudioOffset+self.FrameCount, 4)+ 8], True),#SFX1 AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4)+ 4:AddPadding(AudioOffset+self.FrameCount, 4)+ 8], True),#SFX1
AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4)+ 8:AddPadding(AudioOffset+self.FrameCount, 4)+12], True),#SFX2 AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4)+ 8:AddPadding(AudioOffset+self.FrameCount, 4)+12], True),#SFX2
AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4)+12:AddPadding(AudioOffset+self.FrameCount, 4)+16], True))#SFX3 AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4)+12:AddPadding(AudioOffset+self.FrameCount, 4)+16], True))#SFX3
if self.forced_speed == None: if self.forced_speed == None:
self.Framespeed = 8 - ord(data[AddPadding(AudioOffset+self.FrameCount, 4) + 16]) self.Framespeed = 8 - data[AddPadding(AudioOffset+self.FrameCount, 4) + 16]
else: else:
self.Framespeed = self.forced_speed self.Framespeed = self.forced_speed
self.BGMFramespeed = 8 - ord(data[AddPadding(AudioOffset+self.FrameCount, 4) + 17])#framespeed when the bgm was recorded self.BGMFramespeed = 8 - data[AddPadding(AudioOffset+self.FrameCount, 4) + 17]#framespeed when the bgm was recorded
## self.BGMFramespeed = self.forced_speed ## self.BGMFramespeed = self.forced_speed
@@ -195,11 +221,11 @@ class PPM:
self.Frames = []#self.Frames[frame] = [inverted(bool), (color1(0-2), color2(0-2)), frame[layer, x, y] = bool] self.Frames = []#self.Frames[frame] = [inverted(bool), (color1(0-2), color2(0-2)), frame[layer, x, y] = bool]
for i, offset in enumerate(FrameOffsets): for i, offset in enumerate(FrameOffsets):
#Read frame header: #Read frame header:
Inverted = ord(data[offset]) & 0x01 == 0 Inverted = data[offset] & 0x01 == 0
#Reads which color that will be used: #Reads which color that will be used:
Colors = (ord(data[offset]) >> 1 & 0x03, Colors = (data[offset] >> 1 & 0x03,
ord(data[offset]) >> 3 & 0x03) data[offset] >> 3 & 0x03)
Frame = self.ExtractFrame(data, offset, self.Frames[i-1][2] if i else None) Frame = self.ExtractFrame(data, offset, self.Frames[i-1][2] if i else None)
@@ -214,7 +240,7 @@ class PPM:
if ReadSound: if ReadSound:
self.SoundData = [] self.SoundData = []
pos = AddPadding(AudioOffset+self.FrameCount+32, 4) pos = AddPadding(AudioOffset+self.FrameCount+32, 4)
for i in xrange(4): for i in range(4):
self.SoundData.append(data[pos:pos+SoundSize[i]]) self.SoundData.append(data[pos:pos+SoundSize[i]])
pos += SoundSize[i] pos += SoundSize[i]
@@ -264,7 +290,7 @@ class PPM:
for y in range(8): for y in range(8):
for x in range(0,8,2): for x in range(0,8,2):
#two colors stored in each byte: #two colors stored in each byte:
byte = ord(self.RawThumbnail[(ty*512+tx*64+y*8+x)/2]) byte = self.RawThumbnail[(ty*512+tx*64+y*8+x)//2]
out[x+tx*8 , y+ty*8] = palette[byte & 0xF] out[x+tx*8 , y+ty*8] = palette[byte & 0xF]
out[x+tx*8+1, y+ty*8] = palette[byte >> 4] out[x+tx*8+1, y+ty*8] = palette[byte >> 4]
@@ -275,12 +301,12 @@ class PPM:
if self.SoundData[index]: if self.SoundData[index]:
#reverse nibbles: #reverse nibbles:
data = [] data = []
for i in map(ord,self.SoundData[index]): for i in self.SoundData[index]:
data.append(chr((i&0xF)<< 4 | (i>>4))) data.append((i&0xF)<< 4 | (i>>4))
data = "".join(data) data = bytes(data)
#4bit ADPCM decode #4bit ADPCM decode
decoded = audioop.adpcm2lin(data, 2, None)[0] decoded = _decode_ima_adpcm(data)
#write to wav: #write to wav:
if outputpath: if outputpath:
@@ -307,19 +333,19 @@ class PPM:
#Color2Frame = [[0 for i in range(192)] for i in range(256)] #Color2Frame = [[0 for i in range(192)] for i in range(256)]
Encoding = [[], []] Encoding = [[], []]
#Frame = [[[False for _ in xrange(192)] for _ in xrange(256)] for _ in xrange(2)] #Frame = [[[False for _ in range(192)] for _ in range(256)] for _ in range(2)]
Frame = np.zeros((2, 256, 192), dtype=np.bool_) Frame = np.zeros((2, 256, 192), dtype=np.bool_)
#Read tags: #Read tags:
NewFrame = ord(data[offset]) & 0x80 <> 0 NewFrame = data[offset] & 0x80 != 0
Unknown = ord(data[offset]) >> 5 & 0x03 Unknown = data[offset] >> 5 & 0x03
offset += 1 offset += 1
#WIP - framemove: #WIP - framemove:
FrameMove = [0,0] FrameMove = [0,0]
if Unknown & 0x2:#doesn't work 100%... if Unknown & 0x2:#doesn't work 100%...
print "FrameMove at offset ",offset-1 print("FrameMove at offset ",offset-1)
move_x = AscDec(data[offset+0:offset+1], True) move_x = AscDec(data[offset+0:offset+1], True)
move_y = AscDec(data[offset+1:offset+2], True) move_y = AscDec(data[offset+1:offset+2], True)
@@ -327,12 +353,12 @@ class PPM:
FrameMove[1] = move_y if move_y <= 127 else move_y-256 FrameMove[1] = move_y if move_y <= 127 else move_y-256
offset += 2 offset += 2
elif Unknown: elif Unknown:
print "Unknown tags:",Unknown,"at offset ",offset-1 print("Unknown tags:",Unknown,"at offset ",offset-1)
#read the line encoding of the layers: #read the line encoding of the layers:
for layer in xrange(2): for layer in range(2):
for byte in map(ord, data[offset:offset+48]): for byte in data[offset:offset+48]:
Encoding[layer].append(byte & 0x03) Encoding[layer].append(byte & 0x03)
Encoding[layer].append(byte >> 2 & 0x03) Encoding[layer].append(byte >> 2 & 0x03)
Encoding[layer].append(byte >> 4 & 0x03) Encoding[layer].append(byte >> 4 & 0x03)
@@ -340,8 +366,8 @@ class PPM:
offset += 48 offset += 48
#read layers: #read layers:
for layer in xrange(2): for layer in range(2):
for y in xrange(192): for y in range(192):
if Encoding[layer][y] == 0:#Nothing if Encoding[layer][y] == 0:#Nothing
pass pass
elif Encoding[layer][y] == 1:#Normal elif Encoding[layer][y] == 1:#Normal
@@ -350,9 +376,9 @@ class PPM:
x = 0 x = 0
while UseByte & 0xFFFFFFFF: while UseByte & 0xFFFFFFFF:
if UseByte & 0x80000000: if UseByte & 0x80000000:
byte = ord(data[offset]) byte = data[offset]
offset += 1 offset += 1
for _ in xrange(8): for _ in range(8):
if byte & 0x01: if byte & 0x01:
Frame[layer][x][y] = True Frame[layer][x][y] = True
x += 1 x += 1
@@ -366,7 +392,7 @@ class PPM:
x = 0 x = 0
while UseByte&0xFFFFFFFF: while UseByte&0xFFFFFFFF:
if UseByte & 0x80000000: if UseByte & 0x80000000:
byte = ord(data[offset]) byte = data[offset]
offset += 1 offset += 1
for _ in range(8): for _ in range(8):
if not byte & 0x01: if not byte & 0x01:
@@ -382,7 +408,7 @@ class PPM:
elif Encoding[layer][y] == 3:#Raw/full elif Encoding[layer][y] == 3:#Raw/full
x = 0 x = 0
for _ in range(32): for _ in range(32):
byte = ord(data[offset]) byte = data[offset]
offset += 1 offset += 1
for _ in range(8): for _ in range(8):
if byte & 0x01: if byte & 0x01:
@@ -391,7 +417,7 @@ class PPM:
byte >>= 1 byte >>= 1
#Merges this frame with the previous frame if NewFrame isn't true: #Merges this frame with the previous frame if NewFrame isn't true:
if not NewFrame and PrevFrame.all() <> None:#maybe optimize this better for numpy... if not NewFrame and PrevFrame is not None:#maybe optimize this better for numpy...
if FrameMove[0] or FrameMove[1]:#Moves the previous frame if specified: if FrameMove[0] or FrameMove[1]:#Moves the previous frame if specified:
NewPrevFrame = np.zeros((2, 256, 192), dtype=np.bool_) NewPrevFrame = np.zeros((2, 256, 192), dtype=np.bool_)
@@ -406,7 +432,7 @@ class PPM:
PrevFrame = NewPrevFrame PrevFrame = NewPrevFrame
#merge the frames: #merge the frames:
Frame = Frame <> PrevFrame Frame = Frame != PrevFrame
return Frame return Frame
@@ -422,9 +448,9 @@ class PPM:
# - instance.FrameCount: The number of frames this flipnote consists of # - instance.FrameCount: The number of frames this flipnote consists of
# - instance.Locked: A boolean telling us wether it's locked or not # - instance.Locked: A boolean telling us wether it's locked or not
# - instance.ThumbnailFrameIndex: Which frame the thumbnail is(starting at 0) # - instance.ThumbnailFrameIndex: Which frame the thumbnail is(starting at 0)
# - instance.OriginalAuthorName: The name of the original author in unicode # - instance.OriginalAuthorName: The name of the original author in str
# - instance.EditorAuthorName: The name of the (previous?) editing author in unicode # - instance.EditorAuthorName: The name of the (previous?) editing author in str
# - instance.Username: The username in unicode # - instance.Username: The username in str
# - instance.OriginalAuthorID: The ID of the original author # - instance.OriginalAuthorID: The ID of the original author
# - instance.EditorAuthorID: The ID of the last editor, the last user to save the file # - instance.EditorAuthorID: The ID of the last editor, the last user to save the file
# - instance.PreviousEditAuthorID: The ID of the previous editor # - instance.PreviousEditAuthorID: The ID of the previous editor
@@ -443,7 +469,7 @@ class TMB:
f.close() f.close()
return ret return ret
def Read(self, data, DecodeThumbnail=False): def Read(self, data, DecodeThumbnail=False):
if data[:4] <> "PARA" or len(data) < 0x6a0: if data[:4] != b"PARA" or len(data) < 0x6a0:
return False return False
#Read the header: #Read the header:
@@ -451,25 +477,25 @@ class TMB:
self.AudioLenght = AscDec(data[8:12], True)#only stored for self.Pack() self.AudioLenght = AscDec(data[8:12], True)#only stored for self.Pack()
self.FrameCount = AscDec(data[12:14], True) + 1 self.FrameCount = AscDec(data[12:14], True) + 1
self.Locked = ord(data[0x10]) & 0x01 == 1 self.Locked = data[0x10] & 0x01 == 1
self.ThumbnailFrameIndex = AscDec(data[0x12:0x14], True)#which frame is in the thumbnnail self.ThumbnailFrameIndex = AscDec(data[0x12:0x14], True)#which frame is in the thumbnnail
#self.OriginalAuthorName = u"".join(unichr(AscDec(data[0x14+i*2:0x14+i*2+2], True)) for i in xrange(11)).split(u"\0")[0] #self.OriginalAuthorName = u"".join(chr(AscDec(data[0x14+i*2:0x14+i*2+2], True)) for i in range(11)).split(u"\0")[0]
#self.EditorAuthorName = u"".join(unichr(AscDec(data[0x2A+i*2:0x2A+i*2+2], True)) for i in xrange(11)).split(u"\0")[0] #self.EditorAuthorName = u"".join(chr(AscDec(data[0x2A+i*2:0x2A+i*2+2], True)) for i in range(11)).split(u"\0")[0]
#self.Username = u"".join(unichr(AscDec(data[0x40+i*2:0x40+i*2+2], True)) for i in xrange(11)).split(u"\0")[0] #self.Username = u"".join(chr(AscDec(data[0x40+i*2:0x40+i*2+2], True)) for i in range(11)).split(u"\0")[0]
self.OriginalAuthorName = data[0x14:0x2A].decode("UTF-16LE").split(u"\0")[0] self.OriginalAuthorName = data[0x14:0x2A].decode("UTF-16LE").split(u"\0")[0]
self.EditorAuthorName = data[0x2A:0x40].decode("UTF-16LE").split(u"\0")[0] self.EditorAuthorName = data[0x2A:0x40].decode("UTF-16LE").split(u"\0")[0]
self.Username = data[0x40:0x56].decode("UTF-16LE").split(u"\0")[0] self.Username = data[0x40:0x56].decode("UTF-16LE").split(u"\0")[0]
self.OriginalAuthorID = data[0x56:0x5e][::-1].encode("HEX").upper() self.OriginalAuthorID = data[0x56:0x5e][::-1].hex().upper()
self.EditorAuthorID = data[0x5E:0x66][::-1].encode("HEX").upper()#the last user to save the file self.EditorAuthorID = data[0x5E:0x66][::-1].hex().upper()#the last user to save the file
self.OriginalFilenameC = data[0x66:0x78]#compressed self.OriginalFilenameC = data[0x66:0x78]#compressed
self.CurrentFilenameC = data[0x78:0x8a]#compressed self.CurrentFilenameC = data[0x78:0x8a]#compressed
self.OriginalFilename = "%s_%s_%s.tmb" % (self.OriginalFilenameC[:3].encode("HEX").upper(), self.OriginalFilenameC[3:-2], str(AscDec(self.OriginalFilenameC[-2:], True)).zfill(3)) self.OriginalFilename = "%s_%s_%s.tmb" % (self.OriginalFilenameC[:3].hex().upper(), self.OriginalFilenameC[3:-2].decode("ascii", "replace"), str(AscDec(self.OriginalFilenameC[-2:], True)).zfill(3))
self.CurrentFilename = "%s_%s_%s.tmb" % (self.CurrentFilenameC[:3].encode("HEX").upper(), self.CurrentFilenameC[3:-2], str(AscDec(self.CurrentFilenameC[-2:], True)).zfill(3)) self.CurrentFilename = "%s_%s_%s.tmb" % (self.CurrentFilenameC[:3].hex().upper(), self.CurrentFilenameC[3:-2].decode("ascii", "replace"), str(AscDec(self.CurrentFilenameC[-2:], True)).zfill(3))
self.PreviousEditAuthorID = data[0x8a:0x92][::-1].encode("HEX").upper()#don't know what this really is self.PreviousEditAuthorID = data[0x8a:0x92][::-1].hex().upper()#don't know what this really is
self.PartialFilenameC = data[0x92:0x9a]#compressed self.PartialFilenameC = data[0x92:0x9a]#compressed
@@ -497,27 +523,27 @@ class TMB:
#realself = self #realself = self
#if ppm: self = ppm #if ppm: self = ppm
out = ["PARA",#magic out = [b"PARA",#magic
DecAsc(self.AudioOffset-0x6a0, 4, True),#animation data size DecAsc(self.AudioOffset-0x6a0, 4, True),#animation data size
DecAsc(self.AudioLenght, 4, True),#audio data size DecAsc(self.AudioLenght, 4, True),#audio data size
DecAsc(self.FrameCount-1, 2, True),#frame count DecAsc(self.FrameCount-1, 2, True),#frame count
"\x24\x00",#unknown b"\x24\x00",#unknown
chr(self.Locked), "\0",#locked bytes((int(bool(self.Locked)),)), b"\0",#locked
DecAsc(self.ThumbnailFrameIndex, 2, True),#which frame is in the thumbnnail DecAsc(self.ThumbnailFrameIndex, 2, True),#which frame is in the thumbnnail
self.OriginalAuthorName.encode("UTF-16LE") + "\0\0"*(11-len(self.OriginalAuthorName)),#Original Author Name self.OriginalAuthorName.encode("UTF-16LE") + b"\0\0"*(11-len(self.OriginalAuthorName)),#Original Author Name
self.EditorAuthorName.encode("UTF-16LE") + "\0\0"*(11-len(self.EditorAuthorName)),#Editor Author Name self.EditorAuthorName.encode("UTF-16LE") + b"\0\0"*(11-len(self.EditorAuthorName)),#Editor Author Name
self.Username.encode("UTF-16LE") + "\0\0"*(11-len(self.Username)),#Username self.Username.encode("UTF-16LE") + b"\0\0"*(11-len(self.Username)),#Username
self.OriginalAuthorID.decode("HEX")[::-1],#OriginalAuthorID bytes.fromhex(self.OriginalAuthorID)[::-1],#OriginalAuthorID
self.EditorAuthorID.decode("HEX")[::-1],#EditorAuthorID bytes.fromhex(self.EditorAuthorID)[::-1],#EditorAuthorID
self.OriginalFilenameC,#OriginalFilename self.OriginalFilenameC,#OriginalFilename
self.CurrentFilenameC,#CurrentFilename self.CurrentFilenameC,#CurrentFilename
self.PreviousEditAuthorID.decode("HEX")[::-1],#EditorAuthorID bytes.fromhex(self.PreviousEditAuthorID)[::-1],#EditorAuthorID
self.PartialFilenameC,#PartialFilename self.PartialFilenameC,#PartialFilename
DecAsc(self.Date, 4, True),#Date in seconds DecAsc(self.Date, 4, True),#Date in seconds
"\0\0",#padding b"\0\0",#padding
self.PackThumbnail()]#thumbnail self.PackThumbnail()]#thumbnail
return "".join(out) return b"".join(out)
def GetThumbnail(self, force=False): def GetThumbnail(self, force=False):
if (self.Thumbnail is None or force):# and self.RawThumbnail: if (self.Thumbnail is None or force):# and self.RawThumbnail:
global ThumbPalette global ThumbPalette
@@ -535,7 +561,7 @@ class TMB:
for y in range(8): for y in range(8):
for x in range(0,8,2): for x in range(0,8,2):
#two colors stored in each byte: #two colors stored in each byte:
byte = ord(self.RawThumbnail[(ty*512+tx*64+y*8+x)/2]) byte = self.RawThumbnail[(ty*512+tx*64+y*8+x)//2]
out[x+tx*8 , y+ty*8] = palette[byte & 0xF] out[x+tx*8 , y+ty*8] = palette[byte & 0xF]
out[x+tx*8+1, y+ty*8] = palette[byte >> 4] out[x+tx*8+1, y+ty*8] = palette[byte >> 4]
@@ -560,7 +586,7 @@ class TMB:
0x00FF00FF)#F- 0x00FF00FF)#F-
if not self.Thumbnail: if self.Thumbnail is None:
return self.RawThumbnail return self.RawThumbnail
else: else:
if Exact: if Exact:
@@ -572,12 +598,12 @@ class TMB:
for y in range(8): for y in range(8):
for x in range(0,8,2): for x in range(0,8,2):
#two colors stored in each byte: #two colors stored in each byte:
#pos = 0xa0+(ty*512+tx*64+y*8+x)/2 #pos = 0xa0+(ty*512+tx*64+y*8+x)//2
p1 = palette.index(int(self.Thumbnail[x+tx*8 , y+ty*8])) p1 = palette.index(int(self.Thumbnail[x+tx*8 , y+ty*8]))
p2 = palette.index(int(self.Thumbnail[x+tx*8+1, y+ty*8])) p2 = palette.index(int(self.Thumbnail[x+tx*8+1, y+ty*8]))
out.append(chr(p2<<4 | p1)) out.append(p2<<4 | p1)
self.RawThumbnail = "".join(out) self.RawThumbnail = bytes(out)
return self.RawThumbnail return self.RawThumbnail
else: else:
#not yet implented #not yet implented
@@ -591,7 +617,7 @@ class TMB:
# This function requires the PIl imaging module # This function requires the PIl imaging module
def WriteImage(image, outputPath, scale=1): def WriteImage(image, outputPath, scale=1):
if not hasPIL: if not hasPIL:
print "Error: PIL not found!" print("Error: PIL not found!")
return False return False
#if not image: return False #if not image: return False
@@ -603,7 +629,7 @@ def WriteImage(image, outputPath, scale=1):
else: else:
newImg = image newImg = image
out = newImg.tostring("F") out = newImg.tobytes(order="F")
out = Image.frombytes("RGBA", (len(newImg), len(newImg[0])), out) out = Image.frombytes("RGBA", (len(newImg), len(newImg[0])), out)
filetype = outputPath[outputPath.rfind(".")+1:] filetype = outputPath[outputPath.rfind(".")+1:]
@@ -611,7 +637,9 @@ def WriteImage(image, outputPath, scale=1):
return True return True
def get_metadata(flipnote): def get_metadata(flipnote, filetype=None):
if filetype is None:
filetype = "ppm" if isinstance(flipnote, PPM) else "tmb"
epoch = time.mktime(time.struct_time([2000, 1, 1, 0, 0, 0, 5, 1, -1])) epoch = time.mktime(time.struct_time([2000, 1, 1, 0, 0, 0, 5, 1, -1]))
meta = { meta = {
@@ -636,8 +664,8 @@ def get_metadata(flipnote):
return meta return meta
def DumpFrames(flipnote,directory,scale=1): def DumpFrames(flipnote,directory,scale=1):
for i in xrange(flipnote.FrameCount): for i in range(flipnote.FrameCount):
print "Dumping frame #%i of %i..." % (i+1, flipnote.FrameCount) print("Dumping frame #%i of %i..." % (i+1, flipnote.FrameCount))
WriteImage(flipnote.GetFrame(i), os.path.join(directory, "frame %s.png" % str(i+1).zfill(3)), scale) WriteImage(flipnote.GetFrame(i), os.path.join(directory, "frame %s.png" % str(i+1).zfill(3)), scale)
def DumpSoundFiles(flipnote,directory,raw=False): def DumpSoundFiles(flipnote,directory,raw=False):
@@ -657,161 +685,159 @@ def DumpSFXUsage(flipnote,directory):
f.write("Frame %i:%s%s%s\n" % (i, " SFX1"*s1, " SFX2"*s2, " SFX3"*s3)) f.write("Frame %i:%s%s%s\n" % (i, " SFX1"*s1, " SFX2"*s2, " SFX3"*s3))
if __name__ == '__main__': if __name__ == '__main__':
print " == PPM.py ==" print(" == PPM.py ==")
print " == by pbsds ==" print(" == by pbsds ==")
print " == v1.3 ==" print(" == v1.3 ==")
print print()
if len(sys.argv) < 3: if len(sys.argv) < 3:
print "Usage:" print("Usage:")
print " PPM.py <Mode> <Input> [<Output>] [<Frame>] [<Option>]" print(" PPM.py <Mode> <Input> [<Output>] [<Frame>] [<Option>]")
print "" print("")
print " <Mode>:" print(" <Mode>:")
print " -t: Extracts the thumbnail to the file <Output>" print(" -t: Extracts the thumbnail to the file <Output>")
print " -f: Extracts the frame(s) to <Output>" print(" -f: Extracts the frame(s) to <Output>")
print " -s: Dumps the sound files to the folder <Output>" print(" -s: Dumps the sound files to the folder <Output>")
print " -S: Same as mode -s, but will also dump the raw sound data files" print(" -S: Same as mode -s, but will also dump the raw sound data files")
print " -e: Exports the flipnote to an MKV with minimal transcoding" print(" -e: Exports the flipnote to an MKV with minimal transcoding")
print " -E: Same as -e, but will encode the video as lossless H.265" print(" -E: Same as -e, but will encode the video as lossless H.265")
print " -m: Prints out the metadata. Can also write it to <output> which also" print(" -m: Prints out the metadata. Can also write it to <output> which also")
print " supports unicode charactes." print(" supports str charactes.")
print " -oa: Seach a directory for an original author that matches the RegEx" print(" -oa: Seach a directory for an original author that matches the RegEx")
print " Mode -t and -m supports TMB files aswell" print(" Mode -t and -m supports TMB files aswell")
print " <Frame>" print(" <Frame>")
print " Only used in mode -f" print(" Only used in mode -f")
print " Set this to the exact frame you want to extract(starting at 1) and it" print(" Set this to the exact frame you want to extract(starting at 1) and it")
print " will be saved as a file to <Output>." print(" will be saved as a file to <Output>.")
print " If not specified, it will extract all frames to the folder <Output>" print(" If not specified, it will extract all frames to the folder <Output>")
print " <Option>" print(" <Option>")
print " --speed N: Only used in mode -e. Set this to force a specific" print(" --speed N: Only used in mode -e. Set this to force a specific")
print " flipnote speed (1 to 8)." print(" flipnote speed (1 to 8).")
print " --scale N: Only used in modes -e and -f. Set this to upscale the" print(" --scale N: Only used in modes -e and -f. Set this to upscale the")
print " frames N times." print(" frames N times.")
sys.exit() sys.exit()
import os, time import os, time
if sys.argv[1] == "-t": if sys.argv[1] == "-t":
print "Reading the flipnote file...", print("Reading the flipnote file...", end=" ")
if not os.path.isfile(sys.argv[2]): if not os.path.isfile(sys.argv[2]):
print "Error!\nSpecified file doesn't exist!" print("Error!\nSpecified file doesn't exist!")
sys.exit() sys.exit()
flipnote = TMB().ReadFile(sys.argv[2], True) flipnote = TMB().ReadFile(sys.argv[2], True)
if not flipnote: if not flipnote:
print "Error!" print("Error!")
print "The given file is not a Flipnote PPM file or TMB file!" print("The given file is not a Flipnote PPM file or TMB file!")
sys.exit() sys.exit()
print "Done!" print("Done!")
print "Dumping the thumbnail...", print("Dumping the thumbnail...", end=" ")
WriteImage(flipnote.GetThumbnail(), sys.argv[3]) WriteImage(flipnote.GetThumbnail(), sys.argv[3])
print "Done!" print("Done!")
elif sys.argv[1] == "-f": elif sys.argv[1] == "-f":
if len(sys.argv) < 4: if len(sys.argv) < 4:
print "Error!" print("Error!")
print "<Output> not specified!" print("<Output> not specified!")
sys.exit() sys.exit()
try: try:
scale = int(sys.argv[sys.argv.index("--scale")+1]) scale = int(sys.argv[sys.argv.index("--scale")+1])
print "Using frame scaling "+str(scale) print("Using frame scaling "+str(scale))
except (IndexError,ValueError): except (IndexError,ValueError):
scale = 1 scale = 1
print "Reading the flipnote file...", print("Reading the flipnote file...", end=" ")
if not os.path.isfile(sys.argv[2]): if not os.path.isfile(sys.argv[2]):
print "Error!\nSpecified file doesn't exist!" print("Error!\nSpecified file doesn't exist!")
sys.exit() sys.exit()
flipnote = PPM().ReadFile(sys.argv[2]) flipnote = PPM().ReadFile(sys.argv[2])
if not flipnote: if not flipnote:
print "Error!\nThe given file is not a Flipnote PPM file." print("Error!\nThe given file is not a Flipnote PPM file.")
sys.exit() sys.exit()
print "Done!" print("Done!")
if len(sys.argv) < 5: if len(sys.argv) < 5:
if not os.path.isdir(sys.argv[3]): if not os.path.isdir(sys.argv[3]):
print "Error!\nThe specified directory doesn't exist!" print("Error!\nThe specified directory doesn't exist!")
sys.exit() sys.exit()
DumpFrames(flipnote,sys.argv[3], scale) DumpFrames(flipnote,sys.argv[3], scale)
print "Done!" print("Done!")
else: else:
try: try:
int(sys.argv[4]) int(sys.argv[4])
except: except:
print "Error!\nInvalid <Frame>!" print("Error!\nInvalid <Frame>!")
sys.exit() sys.exit()
if not (0 <= int(sys.argv[4])-1 < flipnote.FrameCount): if not (0 <= int(sys.argv[4])-1 < flipnote.FrameCount):
print "Error!\n<Frame> is out of bounds!" print("Error!\n<Frame> is out of bounds!")
sys.exit() sys.exit()
print "Dumping frame #%i..." % int(sys.argv[4]), print("Dumping frame #%i..." % int(sys.argv[4]), end=" ")
WriteImage(flipnote.GetFrame(int(sys.argv[4])-1), sys.argv[3], scale) WriteImage(flipnote.GetFrame(int(sys.argv[4])-1), sys.argv[3], scale)
print "Done!" print("Done!")
elif sys.argv[1] in ("-s", "-S"): elif sys.argv[1] in ("-s", "-S"):
if len(sys.argv) < 4: if len(sys.argv) < 4:
print "Error!" print("Error!")
print "<Output> not specified!" print("<Output> not specified!")
sys.exit() sys.exit()
print "Reading the flipnote file...", print("Reading the flipnote file...", end=" ")
if not os.path.isfile(sys.argv[2]): if not os.path.isfile(sys.argv[2]):
print "Error!\nSpecified file doesn't exist!" print("Error!\nSpecified file doesn't exist!")
sys.exit() sys.exit()
flipnote = PPM().ReadFile(sys.argv[2], ReadFrames=False, ReadSound=True) flipnote = PPM().ReadFile(sys.argv[2], ReadFrames=False, ReadSound=True)
if not flipnote: if not flipnote:
print "Error!\nThe given file is not a Flipnote PPM file." print("Error!\nThe given file is not a Flipnote PPM file.")
sys.exit() sys.exit()
print "Done!" print("Done!")
if not os.path.isdir(sys.argv[3]): if not os.path.isdir(sys.argv[3]):
print "Error!\nThe specified output directory doesn't exist!" print("Error!\nThe specified output directory doesn't exist!")
sys.exit() sys.exit()
print "Converting the sound files...", print("Converting the sound files...", end=" ")
DumpSoundFiles(flipnote,sys.argv[3],raw=(sys.argv[1]=="-S")) DumpSoundFiles(flipnote,sys.argv[3],raw=(sys.argv[1]=="-S"))
print "Done!" print("Done!")
print "Dumping the sound effect usage...", print("Dumping the sound effect usage...", end=" ")
DumpSFXUsage(flipnote,sys.argv[3]) DumpSFXUsage(flipnote,sys.argv[3])
print "Done!" print("Done!")
elif sys.argv[1] == "-m": elif sys.argv[1] == "-m":
if not os.path.isfile(sys.argv[2]): if not os.path.isfile(sys.argv[2]):
print "Error!\nSpecified file doesn't exist!" print("Error!\nSpecified file doesn't exist!")
sys.exit() sys.exit()
filetype = "ppm" if sys.argv[2][-3:] == "ppm" else "tmb" filetype = "ppm" if sys.argv[2][-3:] == "ppm" else "tmb"
flipnote = TMB().ReadFile(sys.argv[2]) if filetype == "tmb" else PPM().ReadFile(sys.argv[2], ReadFrames=False) flipnote = TMB().ReadFile(sys.argv[2]) if filetype == "tmb" else PPM().ReadFile(sys.argv[2], ReadFrames=False)
if not flipnote: if not flipnote:
print "Error!\nThe given file is not a Flipnote PPM file or TMB file." print("Error!\nThe given file is not a Flipnote PPM file or TMB file.")
if len(sys.argv) >= 4: if len(sys.argv) >= 4:
f = open(sys.argv, "wb") with open(sys.argv[3], "w", encoding="UTF-8") as f:
f.write("Error!\nThe given file is not a Flipnote PPM file or TMB file.") f.write("Error!\nThe given file is not a Flipnote PPM file or TMB file.\n")
f.close()
sys.exit() sys.exit()
meta = get_metadata(flipnote) meta = get_metadata(flipnote, filetype)
newline = "\n" newline = "\n"
if sys.platform in ("win32", "cygwin"): newline = "\r\n" if sys.platform in ("win32", "cygwin"): newline = "\r\n"
elif sys.platform in ("darwin"): newline = "\r" elif sys.platform in ("darwin"): newline = "\r"
print newline.join(["\t".join([unicode(char) for char in line]) for line in meta.items()]).encode('ascii', 'ignore') metadata_text = newline.join(f"{key}\t{value}" for key, value in meta.items())
print(metadata_text)
if len(sys.argv) >= 4: if len(sys.argv) >= 4:
f = open(sys.argv[3], "wb") with open(sys.argv[3], "w", encoding="UTF-8", newline="") as f:
f.write(newline.join(meta).encode("UTF-8")) f.write(metadata_text + newline)
f.close()
elif sys.argv[1] == "-oa": elif sys.argv[1] == "-oa":
regex = re.compile(sys.argv[2]) regex = re.compile(sys.argv[2])
os.chdir(sys.argv[3]) os.chdir(sys.argv[3])
for filename in os.listdir("."): for filename in os.listdir("."):
epoch = time.mktime(time.struct_time([2000, 1, 1, 0, 0, 0, 5, 1, -1])) epoch = time.mktime(time.struct_time([2000, 1, 1, 0, 0, 0, 5, 1, -1]))
if not os.path.isfile(filename): if not os.path.isfile(filename):
print "Error!\nSpecified file doesn't exist!" print("Error!\nSpecified file doesn't exist!")
sys.exit() sys.exit()
filetype = "ppm" if filename[-3:].lower() == "ppm" else "tmb" filetype = "ppm" if filename[-3:].lower() == "ppm" else "tmb"
@@ -821,11 +847,11 @@ if __name__ == '__main__':
meta = get_metadata(flipnote) meta = get_metadata(flipnote)
if regex.match(meta["Original author"]): if regex.match(meta["Original author"]):
print filename print(filename)
elif sys.argv[1] == "-e" or sys.argv[1] == "-E": elif sys.argv[1] in ("-e", "-E"):
if not hasffmpeg: if not hasffmpeg:
print "Error!\nffmpeg is not installed." print("Error!\nffmpeg is not installed.")
sys.exit() sys.exit()
in_file = sys.argv[2] in_file = sys.argv[2]
out_file = sys.argv[3] out_file = sys.argv[3]
@@ -833,30 +859,30 @@ if __name__ == '__main__':
sleep_time = int(sys.argv[sys.argv.index("--sleep")+1]) sleep_time = int(sys.argv[sys.argv.index("--sleep")+1])
except (IndexError,ValueError): except (IndexError,ValueError):
sleep_time = 0 sleep_time = 0
print "Not sleeping." print("Not sleeping.")
try: try:
forced_speed = int(sys.argv[sys.argv.index("--speed")+1]) forced_speed = int(sys.argv[sys.argv.index("--speed")+1])
print "Using forced speed "+str(forced_speed) print("Using forced speed "+str(forced_speed))
except (IndexError,ValueError): except (IndexError,ValueError):
forced_speed = None forced_speed = None
try: try:
scale = int(sys.argv[sys.argv.index("--scale")+1]) scale = int(sys.argv[sys.argv.index("--scale")+1])
print "Using frame scaling "+str(scale) print("Using frame scaling "+str(scale))
except (IndexError,ValueError): except (IndexError,ValueError):
scale = 1 scale = 1
if out_file.lower()[-4:] != ".mkv": if out_file.lower()[-4:] != ".mkv":
out_file += ".mkv" out_file += ".mkv"
if not os.path.isfile(in_file): if not os.path.isfile(in_file):
print "Error!\nSpecified file doesn't exist!" print("Error!\nSpecified file doesn't exist!")
sys.exit() sys.exit()
if os.path.isfile(out_file): if os.path.isfile(out_file):
print "Overwrite existing file?" print("Overwrite existing file?")
overwrite = "" overwrite = ""
while overwrite != "y" and overwrite != "n": while overwrite != "y" and overwrite != "n":
overwrite = raw_input("(Y/N) ").lower() overwrite = input("(Y/N) ").lower()
if overwrite == "n": if overwrite == "n":
print "Not overwriting; exiting." print("Not overwriting; exiting.")
sys.exit() sys.exit()
filetype = "ppm" if in_file[-3:].lower() == "ppm" else "tmb" filetype = "ppm" if in_file[-3:].lower() == "ppm" else "tmb"
flipnote = TMB().ReadFile(in_file) if filetype == "tmb" else PPM(forced_speed).ReadFile(in_file, ReadFrames=True, ReadSound=True) flipnote = TMB().ReadFile(in_file) if filetype == "tmb" else PPM(forced_speed).ReadFile(in_file, ReadFrames=True, ReadSound=True)
@@ -864,63 +890,62 @@ if __name__ == '__main__':
# Make temp dir and dump the frames and sound here # Make temp dir and dump the frames and sound here
tempdir = tempfile.mkdtemp() tempdir = tempfile.mkdtemp()
os.mkdir(tempdir+"/sounds") os.mkdir(tempdir+"/sounds")
print "Dumping the frames..." print("Dumping the frames...")
DumpFrames(flipnote,tempdir,scale) DumpFrames(flipnote,tempdir,scale)
print "Done!" print("Done!")
print "Dumping the sounds..." print("Dumping the sounds...")
DumpSoundFiles(flipnote,tempdir+"/sounds") DumpSoundFiles(flipnote,tempdir+"/sounds")
print "Done!" print("Done!")
print "Dumping SFX usage..." print("Dumping SFX usage...")
DumpSFXUsage(flipnote,tempdir+"/sounds") DumpSFXUsage(flipnote,tempdir+"/sounds")
print "Done!" print("Done!")
# Now we need the metadata so we can look up the FPS # Now we need the metadata so we can look up the FPS
SPEEDS = [None,0.5,1,2,4,6,12,20,30] SPEEDS = [None,0.5,1,2,4,6,12,20,30]
print "Getting metadata..." print("Getting metadata...")
metadata = get_metadata(flipnote) metadata = get_metadata(flipnote)
print "Done!" print("Done!")
speed = int(metadata["Frame speed"]) speed = int(metadata["Frame speed"])
fps = SPEEDS[speed] fps = SPEEDS[speed]
duration = float(metadata["Number of frames"])/float(fps) duration = float(metadata["Number of frames"])/float(fps)
print "Flipnote is speed {speed}, so {fps} FPS for {dur} seconds".format(speed=speed,fps=fps,dur=duration) print("Flipnote is speed {speed}, so {fps} FPS for {dur} seconds".format(speed=speed,fps=fps,dur=duration))
# Now to make the video in ffmpeg # Now to make the video in ffmpeg
print "Exporting video with ffmpeg..." print("Exporting video with ffmpeg...")
export_command = ["ffmpeg","-framerate",str(fps),"-start_number","1","-i","{path}/frame %03d.png".format(path=tempdir),"-i","{path}/sounds/BGM.wav".format(path=tempdir),"-c:v","copy","-c:a","copy","-t","{dur}".format(dur=duration),"-y",out_file] export_command = ["ffmpeg","-framerate",str(fps),"-start_number","1","-i","{path}/frame %03d.png".format(path=tempdir),"-i","{path}/sounds/BGM.wav".format(path=tempdir),"-c:v","copy","-c:a","copy","-t","{dur}".format(dur=duration),"-y",out_file]
if not os.path.isfile(tempdir+"/sounds/BGM.wav"): if not os.path.isfile(tempdir+"/sounds/BGM.wav"):
print "No background music. Adding silent track..." print("No background music. Adding silent track...")
#has_bgm = False #has_bgm = False
export_command = ["ffmpeg","-framerate",str(fps),"-start_number","1","-i","{path}/frame %03d.png".format(path=tempdir),"-f","lavfi","-i","anullsrc=r=8192:cl=mono","-c:v","copy","-c:a","pcm_s16le","-shortest","-y",out_file] export_command = ["ffmpeg","-framerate",str(fps),"-start_number","1","-i","{path}/frame %03d.png".format(path=tempdir),"-f","lavfi","-i","anullsrc=r=8192:cl=mono","-c:v","copy","-c:a","pcm_s16le","-shortest","-y",out_file]
else: else:
#has_bgm = True #has_bgm = True
pass pass
# use libx265 in lossless mode, rather than MPNG
# Latest master also supports a lossless H.265 export mode.
if sys.argv[1] == "-E": if sys.argv[1] == "-E":
# insert the necessary arguments into the existing command
codec_index = export_command.index("-c:v") codec_index = export_command.index("-c:v")
export_command[codec_index + 1] = "libx265" export_command[codec_index + 1] = "libx265"
export_command.insert(codec_index+2, "-x265-params") export_command[codec_index + 2:codec_index + 2] = [
export_command.insert(codec_index+3, "lossless=1") "-preset", "veryslow", "-x265-params", "lossless=1"
export_command.insert(codec_index+2, "-preset") ]
export_command.insert(codec_index+3, "veryslow") print(" ".join(export_command))
print " ".join(export_command) with open(os.devnull, "w") as null:
with open(os.devnull) as null:
subprocess.call(export_command,stdout=null,stderr=null) subprocess.call(export_command,stdout=null,stderr=null)
print "Done!" print("Done!")
# If the audio has been sped up, we have to do it again manually # If the audio has been sped up, we have to do it again manually
bgm_speed = int(metadata["BGM Frame speed"]) bgm_speed = int(metadata["BGM Frame speed"])
if bgm_speed != speed: if bgm_speed != speed:
print "Background music speed must be modified!" print("Background music speed must be modified!")
original_rate = 8192 original_rate = 8192
newrate = 8192*(float(fps)/SPEEDS[bgm_speed]) newrate = 8192*(float(fps)/SPEEDS[bgm_speed])
print "Using new rate: "+str(newrate) print("Using new rate: "+str(newrate))
speed_change_command = ["ffmpeg","-i","{path}/sounds/BGM.wav".format(path=tempdir),"-filter_complex","asetrate={rate}".format(rate=newrate),"-i",out_file,"-map","0:a","-map","1:v","-vcodec","copy","-acodec","pcm_s16le","{path}/temp_out.mkv".format(path=tempdir)] speed_change_command = ["ffmpeg","-i","{path}/sounds/BGM.wav".format(path=tempdir),"-filter_complex","asetrate={rate}".format(rate=newrate),"-i",out_file,"-map","0:a","-map","1:v","-vcodec","copy","-acodec","pcm_s16le","{path}/temp_out.mkv".format(path=tempdir)]
with open(os.devnull,"w") as null: with open(os.devnull,"w") as null:
subprocess.call(speed_change_command,stdout=null,stderr=null) subprocess.call(speed_change_command,stdout=null,stderr=null)
os.remove(out_file) os.remove(out_file)
shutil.move("{path}/temp_out.mkv".format(path=tempdir),out_file) shutil.move("{path}/temp_out.mkv".format(path=tempdir),out_file)
print "Done!" print("Done!")
# These are the ffmpeg commands I need for each sound effect # These are the ffmpeg commands I need for each sound effect
# The first generates a silent track, with variable length. This gets concatenated to the front of the sound effect. # The first generates a silent track, with variable length. This gets concatenated to the front of the sound effect.
@@ -938,10 +963,10 @@ if __name__ == '__main__':
## else: ## else:
# Read in the sound effect usage data # Read in the sound effect usage data
print "Reading sound effect usage..." print("Reading sound effect usage...")
with open("{path}/sounds/SFX usage.txt".format(path=tempdir),"r") as sfx_usage_file: with open("{path}/sounds/SFX usage.txt".format(path=tempdir),"r") as sfx_usage_file:
sfx_usage = sfx_usage_file.read().split("\n") sfx_usage = sfx_usage_file.read().split("\n")
print "Done!" print("Done!")
# Iterate through the frames, checking if sound effects need to be added # Iterate through the frames, checking if sound effects need to be added
for frame in range(len(sfx_usage)): for frame in range(len(sfx_usage)):
@@ -950,7 +975,7 @@ if __name__ == '__main__':
sfx = line.split(":")[1].strip() if line.strip() != "" else "" sfx = line.split(":")[1].strip() if line.strip() != "" else ""
if sfx != "": # If a sound effect must be played... if sfx != "": # If a sound effect must be played...
length = frame/float(fps) length = frame/float(fps)
print "Adding "+sfx+" at {length} seconds into the video.".format(length=length) print("Adding "+sfx+" at {length} seconds into the video.".format(length=length))
# ...run each command in series with the correct arguments # ...run each command in series with the correct arguments
with open(os.devnull,"w") as null: with open(os.devnull,"w") as null:
subprocess.call([i.format(path=tempdir,sfx=sfx,length=length) for i in silence_command],stdout=null,stderr=null) subprocess.call([i.format(path=tempdir,sfx=sfx,length=length) for i in silence_command],stdout=null,stderr=null)
@@ -958,7 +983,7 @@ if __name__ == '__main__':
subprocess.call([i.format(path=tempdir,video_length=fps*len(sfx_usage)) for i in merge_command],stdout=null,stderr=null) subprocess.call([i.format(path=tempdir,video_length=fps*len(sfx_usage)) for i in merge_command],stdout=null,stderr=null)
os.remove(out_file) os.remove(out_file)
shutil.move("{path}/temp_out.mkv".format(path=tempdir),out_file) shutil.move("{path}/temp_out.mkv".format(path=tempdir),out_file)
print "Done!" print("Done!")
time.sleep(sleep_time) # optional sleep -- in case you want to slow things down for HDD strain or reliability or something time.sleep(sleep_time) # optional sleep -- in case you want to slow things down for HDD strain or reliability or something
## subprocess.call(normalise_command) ## subprocess.call(normalise_command)
@@ -967,9 +992,9 @@ if __name__ == '__main__':
# Remove the temp dir and all files in it # Remove the temp dir and all files in it
print "Removing temporary directory..." print("Removing temporary directory...")
shutil.rmtree(tempdir) shutil.rmtree(tempdir)
print "Done!" print("Done!")
else: else:
print "Error!\nThere's no such mode." print("Error!\nThere's no such mode.")
README.md
+46
View File
@@ -0,0 +1,46 @@
# Hatenatools3
**Python 3 port by RekuNote (aka `rekushiiiiii`)**
Hatenatools3 is a Python 3 port of **Hatenatools**, originally created by **pbsds (Peder Bergebakken Sundt)**.
A separate Python 3 port of Hatenatools already exists. Hatenatools3 is not intended to claim precedence over that work. This port is specifically maintained and optimised for compatibility with **RexiMemo's Python 3 Hatena server**:
https://github.com/RexiMemo-org/hatena-server-py3
## Compatibility goal
The distribution is named **Hatenatools3**, but the internal Python package directory intentionally remains named `Hatenatools`. This preserves the layout expected by `hatena-server-py3` and allows the tools to be used as a drop-in replacement/update without renaming the server's module paths.
The port preserves the newer Hatenatools master feature set while adding Python 3 compatibility work, including support for Python 3.13+ where the standard-library `audioop` module is no longer available.
## Included tools
- `PPM.py` v1.5 — PPM/TMB metadata, thumbnails, frames, sound, and ffmpeg export (`-e` / `-E`).
- `UGO.py` v0.93 — UGO read/write, item editing, embedded assets, and XML conversion.
- `NTFT.py` v0.95 — NTFT image decoding/encoding with Pillow support.
## Requirements
- Python 3
- NumPy
- Pillow for image functionality
- ffmpeg for optional PPM video export
## Credits
- Original Hatenatools: pbsds (Peder Bergebakken Sundt)
- Hatenatools3 Python 3 port: RekuNote (aka `rekushiiiiii`)
- Steven — DSiBrew documentation and frame decoding example
- Remark — 8x8 preview-image tiling help
- JSAfive — `.tmb` and `.ugo` samples
- Austin Burk, Midmad, and WDLmaster — sound codec research
- jaames — proper p-frame translation
- WORD559 — ffmpeg video export
- wyntrr-end — ffmpeg export duration and frame scaling
## License
Hatenatools3 retains the original Hatenatools **AGPL-3.0** licensing. See `License.txt`.
Original format documentation: https://github.com/pbsds/hatena-server/wiki
ReadMe.txt
+43 -15
View File
@@ -1,39 +1,67 @@
== Hatenatools == == Hatenatools3 ==
== By pbsds == == Python 3 port by RekuNote ==
== aka Peder Bergebakken Sundt == == aka rekushiiiiii ==
Based on Hatenatools by pbsds (Peder Bergebakken Sundt).
--VERSIONS-- --VERSIONS--
PPM.py: v1.5 PPM.py: v1.5
UGO.py: v0.93 UGO.py: v0.93
NTFT.py: v0.95 NTFT.py: v0.95
--ABOUT HATENATOOLS3--
Hatenatools3 is a Python 3 port of the supplied Hatenatools master sources,
ported by RekuNote (aka rekushiiiiii).
A separate Python 3 port of Hatenatools already exists. Hatenatools3 is not
intended to claim precedence over that work; this port is specifically kept
compatible with and optimised for the RexiMemo Python 3 Hatena server:
https://github.com/RexiMemo-org/hatena-server-py3
For drop-in compatibility with hatena-server-py3, the Python package directory
remains named "Hatenatools" even though the project/distribution is named
"Hatenatools3".
This port preserves the newer master feature set, including PPM's -e and -E
video export modes, while keeping Python 3 compatibility fixes such as the
internal ADPCM decoder used on Python 3.13+, where audioop is no longer
available.
--DESCRIPTION-- --DESCRIPTION--
PPM.py can read the metadata, thumbnail, frames and the sound of Flipnote Studio files: .ppm PPM.py can read the metadata, thumbnail, frames and sound of Flipnote Studio files: .ppm
It can also handle .tmb files. It can also handle .tmb files.
UGO.py can read and write UGO files, and modify their content. It can also convert UGO files UGO.py can read and write UGO files, modify their content, and convert UGO files
to and from the XML format to and from the XML format.
NTFT.py can read ntft image files and convert them to PIL supported image files NTFT.py can read NTFT image files and convert them to PIL-supported image files.
These are all tested and written in python 2.7 These tools are designed to be usable both as Python modules and as stand-alone applications.
These are all designed to be usable both as python modules and as stand alone applications. NTFT.py and PPM.py require NumPy and Pillow for image support. PPM video export requires ffmpeg.
NTFT.py and PPM.py requires Numpy and PIL
--HATENA-SERVER-PY3 COMPATIBILITY--
The internal Hatenatools/ layout is intentionally preserved so this directory can
replace or update the bundled Hatenatools directory in hatena-server-py3 without
requiring the server's import/module paths to be renamed.
Target project:
https://github.com/RexiMemo-org/hatena-server-py3
--DOCUMENTATION-- --DOCUMENTATION--
Documentation on the formats can be found here: Original Hatenatools / Hatena server format documentation:
https://github.com/pbsds/hatena-server/wiki https://github.com/pbsds/hatena-server/wiki
--LICENSE-- --LICENSE--
Hatenatools is licensed under AGPL3 Hatenatools3 retains the original Hatenatools AGPL3 licensing.
See License.txt or http://www.gnu.org/licenses/agpl-3.0.html for more information See License.txt or http://www.gnu.org/licenses/agpl-3.0.html for more information.
--CREDIT-- --CREDIT--
-Original Hatenatools by pbsds (Peder Bergebakken Sundt)
-Python 3 port / Hatenatools3 by RekuNote (aka rekushiiiiii)
-Steven for most of the documentation on DSiBrew and his frame decoding example on his talkpage -Steven for most of the documentation on DSiBrew and his frame decoding example on his talkpage
-Remark for helping me understanding the 8x8 tiling on the preview images. -Remark for helping with the 8x8 tiling on the preview images
-JSAfive for supplying .tmb and .ugo files -JSAfive for supplying .tmb and .ugo files
-Austin Burk, Midmad on hatena haiku and WDLmaster on hcs64.com for determining the sound codec -Austin Burk, Midmad on Hatena Haiku and WDLmaster on hcs64.com for determining the sound codec
-jaames, for proper p-frame translation -jaames, for proper p-frame translation
-WORD559, for video export with ffmpeg -WORD559, for video export with ffmpeg
-wyntrr-end, for proper ffmpeg export duration and frame scaling -wyntrr-end, for proper ffmpeg export duration and frame scaling
UGO.py
+343 -429
View File
@@ -1,174 +1,160 @@
# UGO.py by pbsds # UGO.py by pbsds
# Hatenatools3 Python 3 port by RekuNote aka rekushiiiiii
# AGPL3 licensed # AGPL3 licensed
# #
#This class reads and writes UGO files. # Python 3 port. This class reads/writes Flipnote Hatena UGO files and
#It can also export and import it in a xml format # imports/exports the project's UGOXML representation.
#Extended functionality added by PPM.py
# from __future__ import annotations
#Credits:
# import os
# -Jsafive for supplying .ugo files import sys
# from base64 import b64decode, b64encode
#Note:
# The current implentation can possibly mess up the files stored in Section #2, but will do for most needs
# Next version will probably bring a big change in the ugoxml format
#
import sys, os
from base64 import b64encode, b64decode
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
try: try:
import PPM # Import the actual PPM submodule. Importing ``PPM`` from the package
# namespace can resolve to the convenience-exported PPM class instead.
from .PPM import TMB as _TMB
HasPPM = True
except (ImportError, ValueError):
try:
from PPM import TMB as _TMB # type: ignore
HasPPM = True HasPPM = True
except ImportError: except ImportError:
_TMB = None
HasPPM = False HasPPM = False
#helpers:
def AscDec(ascii, LittleEndian=False):#Converts a ascii string into a decimal
ret = 0
l = map(ord, ascii)
if LittleEndian: l.reverse()
for i in l:
ret = (ret<<8) | i
return ret
def DecAsc(dec, length=None, LittleEndian=False):#Converts a decimal into an ascii string of chosen length
out = []
while dec <> 0:
out.insert(0, dec&0xFF)
dec >>= 8
#"".join(map(chr, out))
if length: def AscDec(data, LittleEndian=False):
if len(out) > length: """Convert a byte string to an integer (legacy Hatenatools helper)."""
#return "".join(map(chr, out[-length:])) if isinstance(data, str):
out = out[-length:] data = data.encode("latin-1")
if len(out) < length: return int.from_bytes(bytes(data), "little" if LittleEndian else "big")
#return "".join(map(chr, [0]*(length-len(out)) + out))
out = [0]*(length-len(out)) + out
def DecAsc(dec, length=None, LittleEndian=False):
"""Convert an integer to bytes (legacy Hatenatools helper)."""
if dec < 0:
raise ValueError("DecAsc does not support negative values")
if length is None:
length = max(1, (dec.bit_length() + 7) // 8) if dec else 0
if length == 0:
return b""
mask = (1 << (length * 8)) - 1
dec &= mask
return dec.to_bytes(length, "little" if LittleEndian else "big")
if LittleEndian: out.reverse()
return "".join(map(chr, out))
def zipalign(length, r=4): def zipalign(length, r=4):
return length + (4 - length % r) if length % r else length return length + (r - length % r) if length % r else length
def indentXML(elem, level=0):#"borrowed" from: http://effbot.org/zone/element-lib.htm#prettyprint
def indentXML(elem, level=0):
i = "\n" + level * "\t" i = "\n" + level * "\t"
if len(elem): if len(elem):
if not elem.text or not elem.text.strip(): if not elem.text or not elem.text.strip():
elem.text = i + "\t" elem.text = i + "\t"
if not elem.tail or not elem.tail.strip(): if not elem.tail or not elem.tail.strip():
elem.tail = i elem.tail = i
for elem in elem: for child in elem:
indentXML(elem, level+1) indentXML(child, level + 1)
if not elem.tail or not elem.tail.strip(): if not child.tail or not child.tail.strip():
elem.tail = i child.tail = i
else: elif level and (not elem.tail or not elem.tail.strip()):
if level and (not elem.tail or not elem.tail.strip()):
elem.tail = i elem.tail = i
#class UGO:
class UGO: class UGO:
def __init__(self): def __init__(self):
self.Loaded = False self.Loaded = False
self.Items = []
self.Files = []
def ReadFile(self, path): def ReadFile(self, path):
f = open(path, "rb") with open(path, "rb") as f:
ret = self.Read(f.read()) return self.Read(f.read())
f.close()
return ret
def Read(self, data): def Read(self, data):
#Filestructure:
# - Header
# - Table of Contents
# - Padding, normally of length 2
# - Extra Data
# - Padding
#The padding seems to do zipaligning to a length of 4
global HasPPM global HasPPM
if isinstance(data, str):
data = data.encode("latin-1")
data = bytes(data)
#Header: if data[:4] != b"UGAR":
if data[:4] <> "UGAR": return False#The file isn't a UGO file return False
Sections = AscDec(data[ 4: 8], True)#could also be version
if Sections >= 1:
self.TableLength = AscDec(data[ 8:12], True)
if Sections >= 2:
self.ExtraLength = AscDec(data[12:16], True)
if Sections > 2:
print "Warning: This UGO file has more than the 2 known sections:",Sections
print "Please send this UGO file to pbsds over at pbsds.net"
print "This file could possibly be read incorrectly..."
headerlength = 8 + Sections*4
#Read table of contents: sections = AscDec(data[4:8], True)
#A table where the rows are seperated with newlines, and colons with tabs self.TableLength = AscDec(data[8:12], True) if sections >= 1 else 0
if Sections >= 1: self.ExtraLength = AscDec(data[12:16], True) if sections >= 2 else 0
TableOfContents = tuple((i.split("\t") for i in data[headerlength:headerlength+self.TableLength].split("\n"))) if sections > 2:
print("Warning: This UGO file has more than the 2 known sections:", sections)
print("Please send this UGO file to pbsds over at pbsds.net")
print("This file could possibly be read incorrectly...")
headerlength = 8 + sections * 4
if sections >= 1:
table_bytes = data[headerlength : headerlength + self.TableLength]
table_text = table_bytes.decode("utf-8")
TableOfContents = tuple(line.split("\t") for line in table_text.split("\n") if line != "")
else: else:
TableOfContents = [] TableOfContents = []
#Extra data: if sections >= 2:
if Sections >= 2: extra_start = zipalign(headerlength + self.TableLength)
ExtraData = data[zipalign(headerlength+self.TableLength) : zipalign(headerlength+self.TableLength)+self.ExtraLength] ExtraData = data[extra_start : extra_start + self.ExtraLength]
else:
ExtraData = b""
#Parse data:
self.Items = [] self.Items = []
self.Files = []
#todo: implement this: pos = 0
self.Files = []#[i] = (filename, filecontent) tmbcount = 1
#see todo.txt ntftcount = 1
pos = 0#Extra Data position
tmbcount = 1#used if HasPPM is False
ntftcount = 1#used if no label
names = [] names = []
for i in TableOfContents:
type = int(i[0])
if type == 0:#layout
#I've always seen just numbers here.
#The amount of numbers also differ.
#maybe related to color scheme? or the button scheme/layout
#all pages containing TMBs have so far been: i = ["0", "2", "1"] for entry in TableOfContents:
self.Items.append(("layout", map(int, i[1:]))) if not entry or not entry[0]:
continue continue
elif type == 1:#Text on topscreen item_type = int(entry[0])
num = int(i[1])#always seen as 0, unknown purpose if item_type == 0:
labels = [b64decode(i[n]).decode("UTF-16LE") for n in xrange(2,7)]#5 labels probably one label for each line of text self.Items.append(("layout", list(map(int, entry[1:]))))
continue
if item_type == 1:
num = int(entry[1])
labels = [b64decode(entry[n]).decode("UTF-16LE") for n in range(2, 7)]
self.Items.append(("topscreen text", labels, num)) self.Items.append(("topscreen text", labels, num))
continue continue
elif type == 2:#catogories(like "reccomended" and "new flipnotes" and "most popular") if item_type == 2:
#this one may visually change greatly depending on what layout is set in type==0 link = entry[1]
label = b64decode(entry[2]).decode("UTF-16LE")
link = i[1] selected = int(entry[3]) != 0
label = b64decode(i[2]).decode("UTF-16LE")
selected = int(i[3]) <> 0#bool
self.Items.append(("category", link, label, selected)) self.Items.append(("category", link, label, selected))
continue continue
elif type == 3:#POST button/link to POST form. "Post Flipnote" uses this if item_type == 3:
link = i[1] link = entry[1]
label = b64decode(i[2]).decode("UTF-16LE") label = b64decode(entry[2]).decode("UTF-16LE")
self.Items.append(("post", link, label)) self.Items.append(("post", link, label))
continue continue
elif type == 4:#Button if item_type == 4:
link = i[1] link = entry[1]
trait = int(i[2]) trait = int(entry[2])
label = b64decode(i[3]).decode("UTF-16LE") label = b64decode(entry[3]).decode("UTF-16LE")
other = i[4:]#varies other = list(entry[4:])
#extra data embedded = None
file = None#== ("filename", "filedata")
if trait < 100 and ExtraData: if trait < 100 and ExtraData:
if ExtraData == "\x20":#empty if ExtraData == b"\x20":
pass#no files pass
elif ExtraData[pos:pos+4] == "PARA":#tmb file elif ExtraData[pos : pos + 4] == b"PARA":
file = ExtraData[pos:pos+0x6A0] filedata = ExtraData[pos : pos + 0x6A0]
pos += 0x6A0 pos += 0x6A0
if HasPPM and _TMB is not None:
if HasPPM: tmb = _TMB().Read(filedata)
tmb = PPM.TMB().Read(file) name = tmb.CurrentFilename[:-4] if tmb else "embedded tmb #%i" % tmbcount
name = tmb.CurrentFilename[:-4] if not tmb:
del tmb tmbcount += 1
else: else:
name = "embedded tmb #%i" % tmbcount name = "embedded tmb #%i" % tmbcount
tmbcount += 1 tmbcount += 1
@@ -178,448 +164,376 @@ class UGO:
while "%s_%i.tmb" % (name, j) in names: while "%s_%i.tmb" % (name, j) in names:
j += 1 j += 1
name = "%s_%i" % (name, j) name = "%s_%i" % (name, j)
embedded = (name + ".tmb", filedata)
file = (name+".tmb", file)
names.append(name + ".tmb") names.append(name + ".tmb")
else:#ntft icon else:
name = label.encode("ascii", "ignore") name = label.encode("ascii", "ignore").decode("ascii")
if not name: if not name:
name = "nameless ntft %i" % ntftcount name = "nameless ntft %i" % ntftcount
ntftcount += 1 ntftcount += 1
if name + ".ntft" in names: if name + ".ntft" in names:
j = 2 j = 2
while "%s_%i.ntft" % (name, j) in names: while "%s_%i.ntft" % (name, j) in names:
j += 1 j += 1
name = "%s_%i" % (name, j) name = "%s_%i" % (name, j)
embedded = (name + ".ntft", ExtraData[pos : pos + 2048])
file = (name+".ntft", ExtraData[pos:pos+2048])
names.append(name + ".ntft") names.append(name + ".ntft")
pos += 2048 pos += 2048
self.Items.append(("button", trait, label, link, other, file)) self.Items.append(("button", trait, label, link, other, embedded))
# if subtype == 3:#flipnote
# tmb = PPM.TMB().Read(ExtraData[pos:pos+0x6A0]); pos += 0x6A0
# unknown1 = i[3]#empty
# stars = int(i[4])#not sure
# unknown2 = map(int, i[5:8])#unknown = [765, 573, 0]
# self.Items.append(("flipnote", link, tmb, stars, unknown1, unknown2))
# continue
# elif subtype == 100: pass
# elif subtype == 101: pass
# elif subtype == 102: pass
# elif subtype == 104:#list item? like mails and announcements
# label = b64decode(i[3]).decode("UTF-16LE")
# unknown = i[4]
# num = int(i[5])#only seen as 0
# # self.Items.append(("list item?", link, label, unknown, num))
# # continue
# pass
# elif subtype == 115:#Labeled button link(size of a flipnote thumbnail, commonly "next page")
# label = b64decode(i[3]).decode("UTF-16LE")
# unknown = i[4:6]
# self.Items.append(("thumbnail link", link, label, unknown))
# continue
# elif subtype == 117: pass
continue continue
#if not recognized: self.Items.append(("unknown", list(entry)))
self.Items.append(("unknown", i)) print("Unknown UGO item discovered:", entry)
print "Unknown UGO item discovered:", i
self.Loaded = True self.Loaded = True
return self return self
def WriteFile(self, path): def WriteFile(self, path):
if self.Loaded: if not self.Loaded:
out = self.Pack()
if out:
f = open(path, "wb")
f.write(out)
f.close()
return True
else:
return False return False
out = self.Pack()
if out is False:
return False
with open(path, "wb") as f:
f.write(out)
return True
def Pack(self): def Pack(self):
if not self.Loaded: return False if not self.Loaded:
return False
Header = ["UGAR", None] table_rows = []
TableOfContents = [] extra_parts = []
ExtraData = []
#Encode data: for item in self.Items:
for i in self.Items: kind = item[0]
if i[0] == "unknown": if kind == "unknown":
TableOfContents.append("\t".join(i[1])) table_rows.append("\t".join(map(str, item[1])))
elif i[0] == "layout":#0 elif kind == "layout":
TableOfContents.append("\t".join(["0"] + map(str, list(i[1])))) table_rows.append("\t".join(["0"] + [str(value) for value in item[1]]))
elif i[0] == "topscreen text":#1 elif kind == "topscreen text":
labels, num = i[1:] labels, num = item[1:]
encoded_labels = [b64encode(label.encode("UTF-16LE")).decode("ascii") for label in labels]
num = str(num) # The format expects exactly five labels.
for i in xrange(5): encoded_labels = (encoded_labels + [""] * 5)[:5]
labels[i] = b64encode(labels[i].encode("UTF-16LE")) table_rows.append("\t".join(["1", str(num)] + encoded_labels))
elif kind == "category":
TableOfContents.append("\t".join(("1", num, labels[0], labels[1], labels[2], labels[3], labels[4]))) link, label, selected = item[1:]
elif i[0] == "category":#2 encoded = b64encode(label.encode("UTF-16LE")).decode("ascii")
link, label, selected = i[1:] table_rows.append("\t".join(("2", link, encoded, str(int(bool(selected))))))
elif kind == "post":
label = b64encode(label.encode("UTF-16LE")) link, label = item[1:]
selected = str(1*selected) encoded = b64encode(label.encode("UTF-16LE")).decode("ascii")
table_rows.append("\t".join(("3", link, encoded)))
TableOfContents.append("\t".join(("2", link, label, selected))) elif kind == "button":
elif i[0] == "post":#3 trait, label, link, other, embedded = item[1:]
link, label = i[1:] encoded = b64encode(label.encode("UTF-16LE")).decode("ascii")
table_rows.append("\t".join(["4", link, str(trait), encoded] + [str(v) for v in other]))
label = b64encode(label.encode("UTF-16LE")) if embedded:
extra_parts.append(bytes(embedded[1]))
TableOfContents.append("\t".join(("3", link, label)))
elif i[0] == "button":#4
trait, label, link, other, file = i[1:]
trait = str(trait)
label = b64encode(label.encode("UTF-16LE"))
TableOfContents.append("\t".join(["4", link, trait, label] + list(other)))
if file:
ExtraData.append(file[1])
else: else:
print "Unrecognized entry in self.Items:", i print("Unrecognized entry in self.Items:", item)
TableOfContents = "\n".join(TableOfContents)
ExtraData = "".join(ExtraData)
#Format data: table = "\n".join(table_rows).encode("utf-8")
Sections = 0 extra = b"".join(extra_parts)
if TableOfContents:
Sections += 1
Header.append(DecAsc(len(TableOfContents), 4, True))
#padding/zipaligning sections = 0
if len(TableOfContents) % 4: lengths = []
TableOfContents += "\0" * (4 - len(TableOfContents) % 4) if table:
if ExtraData: sections += 1
Sections += 1 lengths.append(len(table))
Header.append(DecAsc(len(ExtraData), 4, True)) if extra:
sections += 1
lengths.append(len(extra))
#padding/zipaligning header = b"UGAR" + DecAsc(sections, 4, True) + b"".join(DecAsc(length, 4, True) for length in lengths)
if len(ExtraData) % 4: if table:
ExtraData += "\0" * (4 - len(ExtraData) % 4) table += b"\0" * ((-len(table)) % 4)
Header[1] = DecAsc(Sections, 4, True) if extra:
extra += b"\0" * ((-len(extra)) % 4)
return header + table + extra
#Zip up and send the file: def WriteXML(self, xmlname="content.ugoxml", folder="content.ugoxml embedded"):
return "".join(Header) + TableOfContents + ExtraData if not self.Loaded:
#XML return False
def WriteXML(self, xmlname="content.ugoxml", folder="content.ugoxml embedded"):#WIP
if not self.Loaded: return False
path, xmlname = os.path.split(xmlname) path, xml_basename = os.path.split(xmlname)
ugo_xml = ET.Element("ugo_xml") ugo_xml = ET.Element("ugo_xml")
files = [] files = []
for i in self.Items: for item in self.Items:
if i[0] == "unknown": kind = item[0]
elem = ET.SubElement(ugo_xml, "raw", type=i[1][0]) if kind == "unknown":
for value in i[1][1:]: elem = ET.SubElement(ugo_xml, "raw", type=str(item[1][0]))
ET.SubElement(elem, "value").text = value for value in item[1][1:]:
continue
elif i[0] == "layout":#0
elem = ET.SubElement(ugo_xml, "layout")
for value in i[1]:
ET.SubElement(elem, "value").text = str(value) ET.SubElement(elem, "value").text = str(value)
continue elif kind == "layout":
elif i[0] == "topscreen text":#1 elem = ET.SubElement(ugo_xml, "layout")
for value in item[1]:
ET.SubElement(elem, "value").text = str(value)
elif kind == "topscreen text":
elem = ET.SubElement(ugo_xml, "title") elem = ET.SubElement(ugo_xml, "title")
labels, num = i[1:] labels, num = item[1:]
for label in labels: for label in labels:
ET.SubElement(elem, "label").text = label ET.SubElement(elem, "label").text = label
ET.SubElement(elem, "num").text = str(num) ET.SubElement(elem, "num").text = str(num)
continue elif kind == "category":
elif i[0] == "category":#2
elem = ET.SubElement(ugo_xml, "category") elem = ET.SubElement(ugo_xml, "category")
link, label, selected = i[1:] link, label, selected = item[1:]
ET.SubElement(elem, "label").text = label ET.SubElement(elem, "label").text = label
ET.SubElement(elem, "address").text = link ET.SubElement(elem, "address").text = link
ET.SubElement(elem, "selected").text = str(selected).lower() ET.SubElement(elem, "selected").text = str(bool(selected)).lower()
elif i[0] == "post":#3 elif kind == "post":
elem = ET.SubElement(ugo_xml, "post") elem = ET.SubElement(ugo_xml, "post")
link, label = i[1:] link, label = item[1:]
ET.SubElement(elem, "label").text = label ET.SubElement(elem, "label").text = label
ET.SubElement(elem, "address").text = link ET.SubElement(elem, "address").text = link
elif i[0] == "button":#4 elif kind == "button":
elem = ET.SubElement(ugo_xml, "button") elem = ET.SubElement(ugo_xml, "button")
trait, label, link, other, file = i[1:] trait, label, link, other, embedded = item[1:]
ET.SubElement(elem, "label").text = label ET.SubElement(elem, "label").text = label
ET.SubElement(elem, "address").text = link ET.SubElement(elem, "address").text = link
ET.SubElement(elem, "trait").text = str(trait)#todo: add names ET.SubElement(elem, "trait").text = str(trait)
for n, value in enumerate(other): for n, value in enumerate(other):
entry = ET.SubElement(elem, "value") entry = ET.SubElement(elem, "value")
entry.text = value entry.text = str(value)
if n == 0 and trait == 3: if n == 0 and trait == 3:
entry.attrib["tip"] = "stars" entry.attrib["tip"] = "stars"
if embedded:
ET.SubElement(elem, "embedded_file").text = os.path.join(folder, embedded[0])
files.append((os.path.join(folder, embedded[0]), bytes(embedded[1])))
if file:
ET.SubElement(elem, "embedded_file").text = os.path.join(folder, file[0])
files.append((os.path.join(folder, file[0]), file[1]))
#intend
indentXML(ugo_xml) indentXML(ugo_xml)
files.append((xml_basename, ET.tostring(ugo_xml, encoding="UTF-8")))
#pack xml folder_path = os.path.join(path, folder)
files.append((xmlname, ET.tostring(ugo_xml, encoding="UTF-8"))) if folder and not os.path.isdir(folder_path):
os.makedirs(folder_path, exist_ok=True)
#write files
if not os.path.isdir(os.path.join(path, folder)):
os.mkdir(os.path.join(path, folder))
for name, data in files: for name, data in files:
f = open(os.path.join(path, name), "wb") full_path = os.path.join(path, name)
parent = os.path.dirname(full_path)
if parent:
os.makedirs(parent, exist_ok=True)
with open(full_path, "wb") as f:
f.write(data) f.write(data)
f.close() return True
def ReadXML(self, xmlfile, silent=True): def ReadXML(self, xmlfile, silent=True):
ugo_xml = ET.parse(xmlfile).getroot() ugo_xml = ET.parse(xmlfile).getroot()
xmlpath = os.path.split(xmlfile)[0] xmlpath = os.path.split(xmlfile)[0]
Items = [] items = []
for elem in ugo_xml: for elem in ugo_xml:
if elem.tag == "raw": if elem.tag == "raw":
if "type" not in elem.attrib: if "type" not in elem.attrib:
if not silent: print "Invalid formatting. <raw> without \"type\" attribute" if not silent:
print('Invalid formatting. <raw> without "type" attribute')
return False return False
values = [elem.attrib["type"]] values = [elem.attrib["type"]]
for value in elem: for value in elem:
if value.tag <> "value": if value.tag != "value":
if not silent: print "Invalid formatting. <%s> found within <unknown>" % value.tag if not silent:
print("Invalid formatting. <%s> found within <unknown>" % value.tag)
return False return False
values.append(value.text if value.text else "") values.append(value.text or "")
items.append(("unknown", values))
Items.append(("unknown", values)) elif elem.tag == "layout":
elif elem.tag == "layout":#0
values = [] values = []
for value in elem: for value in elem:
if value.tag <> "value": if value.tag != "value":
if not silent: print "Invalid formatting. <%s> found within <layout>" % value.tag if not silent:
print("Invalid formatting. <%s> found within <layout>" % value.tag)
return False return False
if not value.text.isdigit(): if not (value.text or "").isdigit():
if not silent: print "Invalid entry. <value> in <layout> is not a number" % value.tag if not silent:
print("Invalid entry. <value> in <layout> is not a number")
return False return False
values.append(int(value.text)) values.append(int(value.text))
Items.append(("layout", values)) items.append(("layout", values))
elif elem.tag == "title":#1
elif elem.tag == "title":
labels = ["", "", "", "", ""] labels = ["", "", "", "", ""]
num = 0 num = 0
pos = 0 pos = 0
numset = False numset = False
for value in elem: for value in elem:
if value.tag not in ("label", "num"): if value.tag not in ("label", "num"):
if not silent: print "Invalid formatting. <%s> found within <title>" % value.tag if not silent:
print("Invalid formatting. <%s> found within <title>" % value.tag)
return False return False
if value.tag == "label": if value.tag == "label":
if pos >= 5: if pos >= 5:
if not silent: print "Invalid formatting. More than 5 <labels> in <title>" if not silent:
print("Invalid formatting. More than 5 <labels> in <title>")
return False return False
if value.text: labels[pos] = value.text labels[pos] = value.text or ""
pos += 1 pos += 1
elif value.tag == "num": else:
if numset: if numset:
if not silent: print "Invalid formatting. Multible <num> in <title>" if not silent:
print("Invalid formatting. Multiple <num> in <title>")
return False return False
if not value.text.isdigit(): if not (value.text or "").isdigit():
if not silent: print "Invalid entry. <num> in <title> is not a number!" if not silent:
print("Invalid entry. <num> in <title> is not a number!")
return False return False
num = int(value.text) num = int(value.text)
numset = True numset = True
items.append(("topscreen text", labels, num))
Items.append(("topscreen text", labels, num)) elif elem.tag == "category":
elif elem.tag == "category":#2 link = label = selected = None
link = None
label = None
selected = None
for value in elem: for value in elem:
if value.tag not in ("label", "address", "selected"): if value.tag not in ("label", "address", "selected"):
if not silent: print "Invalid formatting. <%s> found within <category>" % value.tag if not silent:
print("Invalid formatting. <%s> found within <category>" % value.tag)
return False return False
if value.tag == "address": if value.tag == "address":
if isinstance(link, str): if link is not None:
if not silent: print "Invalid formatting. multible <address> within <category>"
return False return False
link = value.text if value.text else "" link = value.text or ""
elif value.tag == "label": elif value.tag == "label":
if isinstance(label, str): if label is not None:
if not silent: print "Invalid formatting. multible <label> within <category>"
return False return False
label = value.text if value.text else "" label = value.text or ""
elif value.tag == "selected": else:
if selected in (True, False): if selected is not None:
if not silent: print "Invalid formatting. multible <selected> within <category>"
return False return False
selected = value.text[0].lower() in "t1" selected = bool(value.text) and value.text[0].lower() in "t1"
if link is None or label is None or selected is None:
Items.append(("category", link, label, selected)) return False
elif elem.tag == "post":#3 items.append(("category", link, label, selected))
label = None
link = None
elif elem.tag == "post":
label = link = None
for value in elem: for value in elem:
if value.tag == "label": if value.tag == "label":
if isinstance(label, str): if label is not None:
if not silent: print "Invalid formatting. Multible <label> within <post>"
return False return False
label = value.text if value.text else "" label = value.text or ""
elif value.tag == "address": elif value.tag == "address":
if isinstance(link, str): if link is not None:
if not silent: print "Invalid formatting. Multible <address> within <post>"
return False return False
link = value.text if value.text else "" link = value.text or ""
else:
if None in (link, label): if not silent:
if not silent: print "Invalid formatting. <button> lacks either a <address> or <label>" print("Invalid formatting. <%s> found within <post>" % value.tag)
return False return False
if link is None or label is None:
return False
items.append(("post", link, label))
Items.append(("post", link, label)) elif elem.tag == "button":
elif elem.tag == "button":#4 trait = label = link = None
trait = None#todo: add names
label = None
link = None
other = [] other = []
file = None embedded = None
for value in elem: for value in elem:
if value.tag not in ("label", "address", "trait", "value", "embedded_file"): if value.tag not in ("label", "address", "trait", "value", "embedded_file"):
if not silent: print "Invalid formatting. <%s> found within <button>" % value.tag if not silent:
print("Invalid formatting. <%s> found within <button>" % value.tag)
return False return False
if value.tag == "label": if value.tag == "label":
if isinstance(label, str): if label is not None:
if not silent: print "Invalid formatting. Multible <label> within <button>"
return False return False
label = value.text if value.text else "" label = value.text or ""
elif value.tag == "address": elif value.tag == "address":
if isinstance(link, str): if link is not None:
if not silent: print "Invalid formatting. Multible <address> within <button>"
return False return False
link = value.text if value.text else "" link = value.text or ""
elif value.tag == "trait":#todo: add names elif value.tag == "trait":
if isinstance(trait, str): if trait is not None or not (value.text or "").isdigit():
if not silent: print "Invalid formatting. Multible <trait> within <button>"
return False
if not value.text.isdigit():
if not silent: print "Invalid entry. <trait> in <button> is not a number"
return False return False
trait = int(value.text) trait = int(value.text)
elif value.tag == "value": elif value.tag == "value":
other.append(value.text if value.text else "") other.append(value.text or "")
elif value.tag == "embedded_file": else:
if file <> None: if embedded is not None:
if not silent: print "Invalid formatting. Multible <embedded_file> within <button>"
return False return False
if not value.text:
path = os.path.join(xmlpath, value.text)
if not os.path.isfile(path):
if not silent: print "Invalid entry. Embedded file \"%s\" not found!" % value.text
print path
return False return False
embedded_path = os.path.join(xmlpath, value.text)
if not os.path.isfile(embedded_path):
f = open(path, "rb") if not silent:
file = (os.path.split(value.text)[1], f.read()) print('Invalid entry. Embedded file "%s" not found!' % value.text)
f.close()
if None in (trait, label, link):
if not silent: print "Invalid formatting. <button> lacks either a <trait>, a <address> or a <label>"
return False return False
with open(embedded_path, "rb") as f:
embedded = (os.path.split(value.text)[1], f.read())
if trait is None or label is None or link is None:
return False
items.append(("button", trait, label, link, other, embedded))
Items.append(("button", trait, label, link, other, file))
else: else:
if not silent: if not silent:
print "Invalid formatting: <%s> found within <ugo_xml>" % elem.tag print("Invalid formatting: <%s> found within <ugo_xml>" % elem.tag)
return False
self.Items = Items self.Items = items
self.Loaded = True self.Loaded = True
return self return self
if __name__ == "__main__":
print " == UGO.py =="
print " == by pbsds =="
print " == v0.93 =="
print
def _main():
print(" == UGO.py ==")
print(" == by pbsds ==")
print(" == v0.93 ==")
print()
if len(sys.argv) < 2: if len(sys.argv) < 2:
print "Usage:" print("Usage:")
print " UGO.py [<mode>] <input> [<output> [<foldername>]]" print(" UGO.py [<mode>] <input> [<output> [<foldername>]]")
print "" print()
print " <Mode>:" print(" <Mode>:")
print " -d: Converts the UGO file in <input> to a UGOXML file with the same" print(" -d: Converts the UGO file in <input> to a UGOXML file with the same")
print " name, unless <output> is specified. Any embedded files will be" print(" name, unless <output> is specified. Any embedded files will be")
print " written to a folder called UGOXML-filename + \" embedded\" unless" print(' written to a folder called UGOXML-filename + " embedded" unless')
print " <foldername> is given." print(" <foldername> is given.")
print " <foldername> is relative to the XML." print(" <foldername> is relative to the XML.")
print " -e: Converts the UGOXML file in <input> to a UGO file with the same" print(" -e: Converts the UGOXML file in <input> to a UGO file with the same")
print " name, unless <output> is specified." print(" name, unless <output> is specified.")
print " If mode is not specified, it will try to find out for itself" print(" If mode is not specified, it will try to find out for itself")
sys.exit() return 0
mode = sys.argv[1] mode = sys.argv[1]
if mode not in ("-d", "-e"): if mode not in ("-d", "-e"):
if os.path.exists(mode):#the mode is actually the file if os.path.exists(mode):
f = open(mode, "rb") with open(mode, "rb") as f:
magic = f.read(4) magic = f.read(4)
f.close() mode = "-d" if magic == b"UGAR" else "-e"
print("No mode specified. %s chosen" % ("UGO -> UGOXML" if mode == "-d" else "UGOXML -> UGO"))
if magic == "UGAR":
mode = "-d"
print "No mode specified. UGO -> UGOXML chosen"
else:
mode = "-e"
print "No mode specified. UGOXML -> UGO chosen"
sys.argv.insert(1, mode) sys.argv.insert(1, mode)
else: else:
print "Invalid <mode> given!" print("Invalid <mode> given!")
sys.exit() return 1
input_path = sys.argv[2]
if mode == "-d": if mode == "-d":
input = sys.argv[2] output = sys.argv[3] if len(sys.argv) >= 4 else input_path + "xml"
output = sys.argv[3] if len(sys.argv) >= 4 else sys.argv[2]+"xml"
foldername = sys.argv[4] if len(sys.argv) >= 5 else os.path.split(output)[1] + " embedded" foldername = sys.argv[4] if len(sys.argv) >= 5 else os.path.split(output)[1] + " embedded"
print("Reading %s..." % os.path.split(input_path)[1])
print "Reading %s..." % os.path.split(input)[1] ugo = UGO().ReadFile(input_path)
ugo = UGO().ReadFile(input)
if not ugo: if not ugo:
print "Error!\n The given file is not a UGO file!" print("Error!\n The given file is not a UGO file!")
sys.exit() return 1
print "Done!" print("Writing XML...")
print "Writing XML..."
ugo.WriteXML(output, foldername) ugo.WriteXML(output, foldername)
print("Done!")
return 0
print "Done!\n\nHave a nice day!" output = sys.argv[3] if len(sys.argv) >= 4 else ".".join(input_path.split(".")[:-1]) + ".ugo"
if mode == "-e": print("Reading %s..." % os.path.split(input_path)[1])
input = sys.argv[2]
output = sys.argv[3] if len(sys.argv) >= 4 else ".".join(sys.argv[2].split(".")[:-1]) + ".ugo"
print "Reading %s..." % os.path.split(input)[1]
try: try:
ugo = UGO().ReadXML(input, False) ugo = UGO().ReadXML(input_path, False)
except EL.ParseError: except ET.ParseError:
print "Error!\nThe given file is not in the XML format!" print("Error!\nThe given file is not in the XML format!")
ugo = False return 1
if not ugo: if not ugo:
#it prints sufficient errormessages return 1
#print "Error!\n The given file is not a UGO file!" print("Writing UGO...")
sys.exit()
print "Done!"
print "Writing UGO..."
ugo.WriteFile(output) ugo.WriteFile(output)
print "Done" print("Done")
return 0
if __name__ == "__main__":
raise SystemExit(_main())
__init__.py
+15
View File
@@ -0,0 +1,15 @@
"""Hatenatools3 - Python 3 port of Hatenatools, optimised for hatena-server-py3.
Original Hatenatools by pbsds. Python 3 port by RekuNote aka rekushiiiiii.
The package directory remains named ``Hatenatools`` for drop-in server compatibility.
"""
PROJECT_NAME = "Hatenatools3"
PORT_AUTHOR = "RekuNote aka rekushiiiiii"
TARGET_PROJECT = "https://github.com/RexiMemo-org/hatena-server-py3"
from .PPM import PPM, TMB
from .UGO import UGO
from .NTFT import NTFT
__all__ = ["PPM", "TMB", "UGO", "NTFT"]
compile.bat
+4 -3
View File
@@ -1,6 +1,7 @@
pyinstaller --onefile -n "UGO" --console "UGO.py" @echo off
pyinstaller --onefile -n "PPM" --console "PPM.py" py -3 -m PyInstaller --onefile -n "UGO" --console "UGO.py"
pyinstaller --onefile -n "NTFT" --console "NTFT.py" py -3 -m PyInstaller --onefile -n "PPM" --console "PPM.py"
py -3 -m PyInstaller --onefile -n "NTFT" --console "NTFT.py"
move "dist\PPM.exe" "PPM.exe" move "dist\PPM.exe" "PPM.exe"
move "dist\UGO.exe" "UGO.exe" move "dist\UGO.exe" "UGO.exe"
move "dist\NTFT.exe" "NTFT.exe" move "dist\NTFT.exe" "NTFT.exe"
hatenatools project page.url
+5
View File
@@ -0,0 +1,5 @@
[{000214A0-0000-0000-C000-000000000046}]
Prop3=19,2
[InternetShortcut]
URL=http://pbsds.net/projects/hatenatools
IDList=