rekushi/Hatenatools3
1
0 Fork 0
mirror of https://github.com/RekuNote/Hatenatools3.git synced 2026-10-30 00:09:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Initial release of Hatenatools3

Browse Source 
Python 3 port of Hatenatools, optimised for compatibility with RexiMemo-org/hatena-server-py3.

Includes Python 3 compatibility fixes, restored functionality from the original Hatenatools, and additional porting/behavior fixes.
This commit is contained in:
Rekushi authored and GitHub committed 2026-09-02 23:50:33 +01:00
1 parent d5bb128b76
commit c0b38d0d9a
8 files changed
+829 -888
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

NTFT.py
+136 -229
View File
@@ -1,305 +1,212 @@
# NTFT.py by pbsds
# Hatenatools3 Python 3 port by RekuNote aka rekushiiiiii
# AGPL3 licensed
#
#PIL is required to read and write images to disk
#
#Credits:
#
# -The guys behind TiledGGD. This sped up my work a lit.
# -Jsafive for supplying .ugo files
#
import sys, os, numpy as np
# Python 3 port
from __future__ import annotations
import os
import sys
import numpy as np
try:
import Image
from PIL import Image
hasPIL = True
except ImportError:
Image = None
hasPIL = False
#helpers
def AscDec(ascii, LittleEndian=False):#Converts a ascii string into a decimal
ret = 0
l = map(ord, ascii)
if LittleEndian: l.reverse()
for i in l:
ret = (ret<<8) | i
return ret
def DecAsc(dec, length=None, LittleEndian=False):#Converts a decimal into an ascii string of chosen length
out = []
while dec <> 0:
out.insert(0, dec&0xFF)
dec >>= 8
#"".join(map(chr, out))
def AscDec(data, LittleEndian=False):
if isinstance(data, str):
data = data.encode("latin-1")
return int.from_bytes(bytes(data), "little" if LittleEndian else "big")
if length:
if len(out) > length:
#return "".join(map(chr, out[-length:]))
out = out[-length:]
if len(out) < length:
#return "".join(map(chr, [0]*(length-len(out)) + out))
out = [0]*(length-len(out)) + out
if LittleEndian: out.reverse()
return "".join(map(chr, out))
def clamp(value, min, max):
if value > max: return max
if value < min: return min
def DecAsc(dec, length=None, LittleEndian=False):
if dec < 0:
raise ValueError("DecAsc does not support negative values")
if length is None:
length = max(1, (dec.bit_length() + 7) // 8) if dec else 0
if length == 0:
return b""
dec &= (1 << (length * 8)) - 1
return dec.to_bytes(length, "little" if LittleEndian else "big")
def clamp(value, minimum, maximum):
if value > maximum:
return maximum
if value < minimum:
return minimum
return value
#Class NTFT:
#
# The NTFT image format stores RGB values as 5 bits each: a value between 0 and 31.
# It has 1 bit of transparancy, which means its either vidssible or invisible. No gradients.
#
# How to use:
#
# Converting to NTFT file:
#
# image = ReadImage(input_path)
# NTFT().SetImage(image).WriteFile(output_path)
#
# Reading NTFT file:
#
# ntft = NTFT().ReadFile(input_path, (width, height))
# WriteImage(ntft.Image, output_path)
#
class NTFT:
"""Reader/writer for Nintendo's raw ARGB1555 NTFT image data."""
def __init__(self):
self.Loaded = False
self.Image = None
def ReadFile(self, path, size):
f = open(path, "rb")
ret = self.Read(f.read(), size)
f.close()
return ret
def Read(self, data, (w, h)):
#the actual stored data is a image with the sizes padded to the nearest power of 2. The image is then clipped out from it.
with open(path, "rb") as f:
return self.Read(f.read(), size)
def Read(self, data, size):
w, h = size
data = bytes(data)
psize = []
for i in (w, h):
for dimension in (w, h):
p = 1
while 1<<p < i:
while 1 << p < dimension:
p += 1
psize.append(1 << p)
pw, ph = psize
#check if it fits the file:
if pw*ph*2 <> len(data):
print "Invalid sizes"
if pw * ph * 2 != len(data):
print("Invalid sizes")
return False
#JUST DO IT!
#self.Image = [[None for _ in xrange(h)] for _ in xrange(w)]
self.Image = np.zeros((w, h), dtype=">u4")
for y in xrange(h):
for x in xrange(w):
for y in range(h):
for x in range(w):
pos = (x + y * pw) * 2
byte = AscDec(data[pos:pos+2], True)
value = AscDec(data[pos : pos + 2], True)
#ARGB1555 -> RGBA8:
a = (byte >> 15 ) * 0xFF
b = (byte >> 10 & 0x1F) * 0xFF / 0x1F
g = (byte >> 5 & 0x1F) * 0xFF / 0x1F
r = (byte & 0x1F) * 0xFF / 0x1F
#self.Image[x][y] = (r<<24) | (g<<16) | (b<<8) | a#RGBA8
self.Image[x, y] = (r<<24) | (g<<16) | (b<<8) | a#RGBA8
a = ((value >> 15) & 0x1) * 0xFF
b = (((value >> 10) & 0x1F) * 0xFF + 15) // 0x1F
g = (((value >> 5) & 0x1F) * 0xFF + 15) // 0x1F
r = ((value & 0x1F) * 0xFF + 15) // 0x1F
self.Image[x, y] = (r << 24) | (g << 16) | (b << 8) | a
self.Loaded = True
return self
def WriteFile(self, path):
if self.Loaded:
f = open(path, "wb")
f.write(self.Pack())
f.close()
return True
else:
return False
def Pack(self):
if not self.Loaded:
return False
with open(path, "wb") as f:
f.write(self.Pack())
return True
def Pack(self):
if not self.Loaded or self.Image is None:
return False
#h = len(self.Image[0])
#w = len(self.Image)
w, h = self.Image.shape
#the actual stored data is a image with the sizes padded to the nearest power of 2
psize = []
for i in (w, h):
for dimension in (w, h):
p = 1
while 1<<p < i:
while 1 << p < dimension:
p += 1
psize.append(1 << p)
out = []
for y in xrange(psize[1]):
for x in xrange(psize[0]):
#read
#c = self.Image[clamp(x, 0, w-1)][clamp(y, 0, h-1)]
c = self.Image[clamp(x, 0, w-1), clamp(y, 0, h-1)]
out = bytearray()
for y in range(psize[1]):
for x in range(psize[0]):
c = int(self.Image[clamp(x, 0, w - 1), clamp(y, 0, h - 1)])
r = c >> 24
g = (c >> 16) & 0xFF
b = (c >> 8) & 0xFF
a = c & 0xFF
#convert
a = 1 if a >= 0x80 else 0
r = r * 0x1F / 0xFF
g = g * 0x1F / 0xFF
b = b * 0x1F / 0xFF
r = (r * 0x1F + 127) // 0xFF
g = (g * 0x1F + 127) // 0xFF
b = (b * 0x1F + 127) // 0xFF
value = (a << 15) | (b << 10) | (g << 5) | r
out.extend(DecAsc(value, 2, True))
return bytes(out)
#store
out.append(DecAsc((a<<15) | (b<<10) | (g<<5) | r, 2, True))
return "".join(out)
def SetImage(self, Image):
self.Image = Image
def SetImage(self, image):
self.Image = image
self.Loaded = True
return self
#Function WriteImage:
#
# Writes a 2D array of uint32 RGBA values as a image file.
# Designed to work with NTFT.Image
#
# This function requires the PIl imaging module
def WriteImage(image, outputPath):
if not hasPIL:
print "Error: PIL not found!"
print("Error: PIL not found!")
return False
#if not image: return False
out = image.tostring("F")
# out = []
# for y in xrange(len(image[0])):
# for x in xrange(len(image)):
# out.append(DecAsc(image[x][y], 4))
out = Image.fromstring("RGBA", (len(image), len(image[0])), out)
out = image.tobytes(order="F")
pil_image = Image.frombytes("RGBA", (len(image), len(image[0])), out)
filetype = outputPath[outputPath.rfind(".") + 1 :]
out.save(outputPath, filetype)
pil_image.save(outputPath, filetype)
return True
#Function ReadImage:
#
# Returns a 2D list of uint32 RGBA values of the image file.
# This can be passed into NTFT().SetImage()
#
# This function requires the PIl imaging module
def ReadImage(path):#TODO: make it support numpy
if not hasPIL: return False
image = Image.open(path)
pixeldata = image.getdata()
def ReadImage(path):
if not hasPIL:
return False
image = Image.open(path).convert("RGBA")
pixeldata = np.asarray(image, dtype=np.uint8)
w, h = image.size
if len(pixeldata[0]) < 4:
def Combine((r, g, b)):
return (r << 24) | (g << 16) | (b << 8) | 0xFF
else:
def Combine((r, g, b, a)):
return (r << 24) | (g << 16) | (b << 8) | a
#ret = []
ret = np.zeros((w, h), dtype=">u4")
for x in xrange(w):
#line = []
for y in xrange(h):
ret[x, y] = Combine(pixeldata[y*w + x])#maybe make a more numpy efficient way?
#line.append(Combine(pixeldata[y*w + x]))
#ret.append(line)
for x in range(w):
for y in range(h):
r, g, b, a = (int(value) for value in pixeldata[y, x])
ret[x, y] = (r << 24) | (g << 16) | (b << 8) | a
return ret
#testing:
# i = NTFT().ReadFile("NTFTtests/kaeru.ntft", (36, 30))
# WriteImage(i.Image, "NTFTtests/kaeru.png")
# i = NTFT().ReadFile("NTFTtests/News.ntft", (32, 32))
# WriteImage(i.Image, "NTFTtests/News.png")
# i = NTFT().ReadFile("NTFTtests/Special Room.ntft", (32, 32))
# WriteImage(i.Image, "NTFTtests/Special Room.png")
#i = NTFT()
#i.Loaded = True
#i.Image = ReadImage("NTFTtests/geh.png")
#i.WriteFile("NTFTtests/geh.ntft")
if __name__ == "__main__":
print " == NTFT.py =="
print " == by pbsds =="
print " == v0.95 =="
print
def _main():
print(" == NTFT.py ==")
print(" == by pbsds ==")
print(" == v0.95 ==")
print()
if not hasPIL:
print "PIL not found! Exiting..."
sys.exit()
print("PIL not found! Exiting...")
return 1
if len(sys.argv) < 2:
print "Usage:"
print " NTFT.py <input> [<output> [<width> <height>]]"
print ""
print "Can convert a NTFT to PNG or the other way around."
print "if <output> isn't specified it will be set to <input> with an another extension"
print ""
print "The NTFT file contain only the colordata, so it's up to the user to find or"
print "store the resolution of the image. <width> and <height> is required"
print "to convert a NTFT file to a image."
print "32x32 is the normal resolution for button icons in UGO files."
sys.exit()
print("Usage:")
print(" NTFT.py <input> [<output> [<width> <height>]]")
print()
print("Can convert an NTFT to PNG or the other way around.")
print("32x32 is the normal resolution for button icons in UGO files.")
return 0
input = sys.argv[1]
if input[-4:].lower() == "ntft" or len(sys.argv) >= 5:
print "Mode: NTFT -> image"
Encode = False
else:
print "Mode: image -> NTFT"
Encode = True#if false it'll decode
input_path = sys.argv[1]
encode = not (input_path[-4:].lower() == "ntft" or len(sys.argv) >= 5)
print("Mode: image -> NTFT" if encode else "Mode: NTFT -> image")
width = height = None
if len(sys.argv) >= 3:
output = sys.argv[2]
width, height = None, None
if len(sys.argv) >= 5:
if (not sys.argv[3].isdigit()) or (not sys.argv[4].isdigit()):
print "Invalid size input!"
sys.exit()
width = int(sys.argv[3])
height = int(sys.argv[4])
if not (width and height) and not Encode:
print "Image size not provided!"
sys.exit()
if not sys.argv[3].isdigit() or not sys.argv[4].isdigit():
print("Invalid size input!")
return 1
width, height = int(sys.argv[3]), int(sys.argv[4])
if not (width and height) and not encode:
print("Image size not provided!")
return 1
else:
output = ".".join(input.split(".")[:-1]) + (".ntft" if Encode else ".png")
output = ".".join(input_path.split(".")[:-1]) + (".ntft" if encode else ".png")
print "Converting..."
if Encode:
print("Converting...")
try:
image = ReadImage(input)
except IOError as err:
print err
sys.exit()
i = NTFT()
i.Loaded = True
i.Image = image
i.WriteFile(output)
if encode:
image = ReadImage(input_path)
if image is False:
return 1
NTFT().SetImage(image).WriteFile(output)
else:
try:
ntft = NTFT().ReadFile(input, (width, height))
except IOError as err:
print err
sys.exit()
if not ntft:#eeror message already printed
sys.exit()
ntft = NTFT().ReadFile(input_path, (width, height))
if not ntft:
return 1
WriteImage(ntft.Image, output)
print "Done!"
except OSError as err:
print(err)
return 1
print("Done!")
return 0
if __name__ == "__main__":
raise SystemExit(_main())
PPM.py
+237 -212
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env python
#PPM.py by pbsds for python 2.7
#!/usr/bin/env python3
#PPM.py by pbsds; Hatenatools3 Python 3 port by RekuNote aka rekushiiiiii
#AGPL3 licensed
#
#Numpy is required
@@ -12,7 +12,7 @@
# - Jsafive for supplying .tmb files
# - Austin Burk, Midmad on hatena haiku and WDLmaster on hcs64.com for determining the sound codec
#
import sys, wave, audioop, re, os #needs os and time aswell in CMD mode (UPDATE 2018/07/04: os used to get devnull for discarding subprocess output)
import sys, wave, re, os, struct # Python 3: audioop was removed in 3.13
import numpy as np
import subprocess # used for ffmpeg
import tempfile, shutil # temporary directory whilst exporting, shutil to clean up after
@@ -21,7 +21,7 @@ try:
from PIL import Image
hasPIL = True
except ImportError:
print "Warning: PIL not found, image extraction won't work!"
print("Warning: PIL not found, image extraction won't work!")
hasPIL = False
# Test for ffmpeg by executing ffmpeg -h; an OSError indicates ffmpeg is not installed
@@ -32,39 +32,65 @@ try:
stderr=null)
hasffmpeg = True
except OSError:
print "Warning: ffmpeg not found, video export unavailable. Please make sure ffmepg is installed and can be accessed on your path."
print("Warning: ffmpeg not found, video export unavailable. Please make sure ffmepg is installed and can be accessed on your path.")
hasffmpeg = False
#helpers:
def AscDec(ascii, LittleEndian=False):#Converts a ascii string into a decimal
ret = 0
l = map(ord, ascii)
if LittleEndian: l.reverse()
for i in l:
ret = (ret<<8) | i
return ret
def DecAsc(dec, length=None, LittleEndian=False):#Converts a decimal into an ascii string of chosen length
out = []
while dec <> 0:
out.insert(0, dec&0xFF)
dec >>= 8
#"".join(map(chr, out))
def AscDec(data, LittleEndian=False):#Converts a byte string into a decimal
if isinstance(data, str):
data = data.encode("latin-1")
return int.from_bytes(bytes(data), "little" if LittleEndian else "big")
if length:
if len(out) > length:
#return "".join(map(chr, out[-length:]))
out = out[-length:]
if len(out) < length:
#return "".join(map(chr, [0]*(length-len(out)) + out))
out = [0]*(length-len(out)) + out
def DecAsc(dec, length=None, LittleEndian=False):#Converts a decimal into bytes of chosen length
if dec < 0:
raise ValueError("DecAsc does not support negative values")
if length is None:
length = max(1, (dec.bit_length() + 7) // 8) if dec else 0
if length == 0:
return b""
dec &= (1 << (length * 8)) - 1
return dec.to_bytes(length, "little" if LittleEndian else "big")
if LittleEndian: out.reverse()
return "".join(map(chr, out))
def AddPadding(i,pad = 0x10):#used mainly for zipaligning offsets
if i % pad <> 0:
if i % pad != 0:
return i + pad - (i % pad)
return i
# Python 3.13 removed audioop. This is the IMA/DVI ADPCM decoder used by
# audioop.adpcm2lin(..., 2, None), retaining the original initial state.
_IMA_INDEX_TABLE = (-1, -1, -1, -1, 2, 4, 6, 8)
_IMA_STEP_TABLE = (
7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34,
37, 41, 45, 50, 55, 60, 66, 73, 80, 88, 97, 107, 118, 130, 143, 157,
173, 190, 209, 230, 253, 279, 307, 337, 371, 408, 449, 494, 544,
598, 658, 724, 796, 876, 963, 1060, 1166, 1282, 1411, 1552, 1707,
1878, 2066, 2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871,
5358, 5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635,
13899, 15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794,
32767,
)
def _decode_ima_adpcm(data):
valpred = 0
index = 0
out = bytearray()
for byte in data:
for delta in (byte >> 4, byte & 0x0F):
step = _IMA_STEP_TABLE[index]
vpdiff = step >> 3
if delta & 4: vpdiff += step
if delta & 2: vpdiff += step >> 1
if delta & 1: vpdiff += step >> 2
if delta & 8:
valpred -= vpdiff
else:
valpred += vpdiff
valpred = max(-32768, min(32767, valpred))
index += _IMA_INDEX_TABLE[delta & 7]
index = max(0, min(88, index))
out.extend(struct.pack("<h", valpred))
return bytes(out)
#palettes:
FramePalette = [0xFFFFFFFF,0x000000FF,0xFF0000FF,0x0000FFFF]
ThumbPalette = (0xFEFEFEFF,#0
@@ -98,9 +124,9 @@ ThumbPalette = (0xFEFEFEFF,#0
# - instance.FrameCount: The number of frames this flipnote consists of
# - instance.Locked: A boolean telling us wether it's locked or not
# - instance.ThumbnailFrameIndex: Which frame the thumbnail is(starting at 0)
# - instance.OriginalAuthorName: The name of the original author in unicode
# - instance.EditorAuthorName: The name of the (previous?) editing author in unicode
# - instance.Username: The username in unicode
# - instance.OriginalAuthorName: The name of the original author in str
# - instance.EditorAuthorName: The name of the (previous?) editing author in str
# - instance.Username: The username in str
# - instance.OriginalAuthorID: The ID of the original author
# - instance.EditorAuthorID: The ID of the last editor, the last user to save the file
# - instance.PreviousEditAuthorID: The ID of the previous editor
@@ -135,7 +161,7 @@ class PPM:
f.close()
return ret
def Read(self, data, DecodeThumbnail=False, ReadFrames=True, ReadSound=False):#Load: (thumbnail, frames, sound)
if data[:4] <> "PARA" or len(data) <= 0x6a0:
if data[:4] != b"PARA" or len(data) <= 0x6a0:
return False
@@ -144,22 +170,22 @@ class PPM:
AudioLenght = AscDec(data[8:12], True)
self.FrameCount = AscDec(data[12:14], True) + 1
self.Locked = ord(data[0x10]) & 0x01 == 1
self.Locked = data[0x10] & 0x01 == 1
self.ThumbnailFrameIndex = AscDec(data[0x12:0x14], True)#which frame the thumbnnail represents
self.OriginalAuthorName = data[0x14:0x2A].decode("UTF-16LE").split(u"\0")[0]
self.EditorAuthorName = data[0x2A:0x40].decode("UTF-16LE").split(u"\0")[0]
self.Username = data[0x40:0x56].decode("UTF-16LE").split(u"\0")[0]
self.OriginalAuthorID = data[0x56:0x5e][::-1].encode("HEX").upper()
self.EditorAuthorID = data[0x5E:0x66][::-1].encode("HEX").upper()#the last user to save the file
self.OriginalAuthorID = data[0x56:0x5e][::-1].hex().upper()
self.EditorAuthorID = data[0x5E:0x66][::-1].hex().upper()#the last user to save the file
self.OriginalFilenameC = data[0x66:0x78]#compressed
self.CurrentFilenameC = data[0x78:0x8a]#compressed
self.OriginalFilename = "%s_%s_%s.ppm" % (self.OriginalFilenameC[:3].encode("HEX").upper(), self.OriginalFilenameC[3:-2], str(AscDec(self.OriginalFilenameC[-2:], True)).zfill(3))
self.CurrentFilename = "%s_%s_%s.ppm" % (self.CurrentFilenameC[:3].encode("HEX").upper(), self.CurrentFilenameC[3:-2], str(AscDec(self.CurrentFilenameC[-2:], True)).zfill(3))
self.OriginalFilename = "%s_%s_%s.ppm" % (self.OriginalFilenameC[:3].hex().upper(), self.OriginalFilenameC[3:-2].decode("ascii", "replace"), str(AscDec(self.OriginalFilenameC[-2:], True)).zfill(3))
self.CurrentFilename = "%s_%s_%s.ppm" % (self.CurrentFilenameC[:3].hex().upper(), self.CurrentFilenameC[3:-2].decode("ascii", "replace"), str(AscDec(self.CurrentFilenameC[-2:], True)).zfill(3))
self.PreviousEditAuthorID = data[0x8a:0x92][::-1].encode("HEX").upper()#don't know what this really is
self.PreviousEditAuthorID = data[0x8a:0x92][::-1].hex().upper()#don't know what this really is
#self.PartialFilenameC = data[0x92:0x9a]#compressed
@@ -172,21 +198,21 @@ class PPM:
self.Loaded[0] = True
#read the animation sequence header:
self.Looped = ord(data[0x06A6]) >> 1 & 0x01 == 1#Unverified?
self.Looped = data[0x06A6] >> 1 & 0x01 == 1#Unverified?
AnimationOffset = 0x6a8 + AscDec(data[0x6a0:0x6a4], True)
FrameOffsets = [AnimationOffset + AscDec(data[0x06a8+i*4:0x06a8+i*4+4], True) for i in xrange(self.FrameCount)]
FrameOffsets = [AnimationOffset + AscDec(data[0x06a8+i*4:0x06a8+i*4+4], True) for i in range(self.FrameCount)]
#Read the audio header:
self.SFXUsage = [(i&0x1<>0, i&0x2<>0, i&0x4<>0) for i in map(ord, data[AudioOffset:AudioOffset+self.FrameCount])]#SFXUsage[frame] = (sfx1, sfx2, sfx2) shere sfxX is either 0 or 1
self.SFXUsage = [(i&0x1!=0, i&0x2!=0, i&0x4!=0) for i in data[AudioOffset:AudioOffset+self.FrameCount]]#SFXUsage[frame] = (sfx1, sfx2, sfx2) shere sfxX is either 0 or 1
SoundSize =(AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4) :AddPadding(AudioOffset+self.FrameCount, 4)+ 4], True),#BG music
AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4)+ 4:AddPadding(AudioOffset+self.FrameCount, 4)+ 8], True),#SFX1
AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4)+ 8:AddPadding(AudioOffset+self.FrameCount, 4)+12], True),#SFX2
AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4)+12:AddPadding(AudioOffset+self.FrameCount, 4)+16], True))#SFX3
if self.forced_speed == None:
self.Framespeed = 8 - ord(data[AddPadding(AudioOffset+self.FrameCount, 4) + 16])
self.Framespeed = 8 - data[AddPadding(AudioOffset+self.FrameCount, 4) + 16]
else:
self.Framespeed = self.forced_speed
self.BGMFramespeed = 8 - ord(data[AddPadding(AudioOffset+self.FrameCount, 4) + 17])#framespeed when the bgm was recorded
self.BGMFramespeed = 8 - data[AddPadding(AudioOffset+self.FrameCount, 4) + 17]#framespeed when the bgm was recorded
## self.BGMFramespeed = self.forced_speed
@@ -195,11 +221,11 @@ class PPM:
self.Frames = []#self.Frames[frame] = [inverted(bool), (color1(0-2), color2(0-2)), frame[layer, x, y] = bool]
for i, offset in enumerate(FrameOffsets):
#Read frame header:
Inverted = ord(data[offset]) & 0x01 == 0
Inverted = data[offset] & 0x01 == 0
#Reads which color that will be used:
Colors = (ord(data[offset]) >> 1 & 0x03,
ord(data[offset]) >> 3 & 0x03)
Colors = (data[offset] >> 1 & 0x03,
data[offset] >> 3 & 0x03)
Frame = self.ExtractFrame(data, offset, self.Frames[i-1][2] if i else None)
@@ -214,7 +240,7 @@ class PPM:
if ReadSound:
self.SoundData = []
pos = AddPadding(AudioOffset+self.FrameCount+32, 4)
for i in xrange(4):
for i in range(4):
self.SoundData.append(data[pos:pos+SoundSize[i]])
pos += SoundSize[i]
@@ -264,7 +290,7 @@ class PPM:
for y in range(8):
for x in range(0,8,2):
#two colors stored in each byte:
byte = ord(self.RawThumbnail[(ty*512+tx*64+y*8+x)/2])
byte = self.RawThumbnail[(ty*512+tx*64+y*8+x)//2]
out[x+tx*8 , y+ty*8] = palette[byte & 0xF]
out[x+tx*8+1, y+ty*8] = palette[byte >> 4]
@@ -275,12 +301,12 @@ class PPM:
if self.SoundData[index]:
#reverse nibbles:
data = []
for i in map(ord,self.SoundData[index]):
data.append(chr((i&0xF)<< 4 | (i>>4)))
data = "".join(data)
for i in self.SoundData[index]:
data.append((i&0xF)<< 4 | (i>>4))
data = bytes(data)
#4bit ADPCM decode
decoded = audioop.adpcm2lin(data, 2, None)[0]
decoded = _decode_ima_adpcm(data)
#write to wav:
if outputpath:
@@ -307,19 +333,19 @@ class PPM:
#Color2Frame = [[0 for i in range(192)] for i in range(256)]
Encoding = [[], []]
#Frame = [[[False for _ in xrange(192)] for _ in xrange(256)] for _ in xrange(2)]
#Frame = [[[False for _ in range(192)] for _ in range(256)] for _ in range(2)]
Frame = np.zeros((2, 256, 192), dtype=np.bool_)
#Read tags:
NewFrame = ord(data[offset]) & 0x80 <> 0
Unknown = ord(data[offset]) >> 5 & 0x03
NewFrame = data[offset] & 0x80 != 0
Unknown = data[offset] >> 5 & 0x03
offset += 1
#WIP - framemove:
FrameMove = [0,0]
if Unknown & 0x2:#doesn't work 100%...
print "FrameMove at offset ",offset-1
print("FrameMove at offset ",offset-1)
move_x = AscDec(data[offset+0:offset+1], True)
move_y = AscDec(data[offset+1:offset+2], True)
@@ -327,12 +353,12 @@ class PPM:
FrameMove[1] = move_y if move_y <= 127 else move_y-256
offset += 2
elif Unknown:
print "Unknown tags:",Unknown,"at offset ",offset-1
print("Unknown tags:",Unknown,"at offset ",offset-1)
#read the line encoding of the layers:
for layer in xrange(2):
for byte in map(ord, data[offset:offset+48]):
for layer in range(2):
for byte in data[offset:offset+48]:
Encoding[layer].append(byte & 0x03)
Encoding[layer].append(byte >> 2 & 0x03)
Encoding[layer].append(byte >> 4 & 0x03)
@@ -340,8 +366,8 @@ class PPM:
offset += 48
#read layers:
for layer in xrange(2):
for y in xrange(192):
for layer in range(2):
for y in range(192):
if Encoding[layer][y] == 0:#Nothing
pass
elif Encoding[layer][y] == 1:#Normal
@@ -350,9 +376,9 @@ class PPM:
x = 0
while UseByte & 0xFFFFFFFF:
if UseByte & 0x80000000:
byte = ord(data[offset])
byte = data[offset]
offset += 1
for _ in xrange(8):
for _ in range(8):
if byte & 0x01:
Frame[layer][x][y] = True
x += 1
@@ -366,7 +392,7 @@ class PPM:
x = 0
while UseByte&0xFFFFFFFF:
if UseByte & 0x80000000:
byte = ord(data[offset])
byte = data[offset]
offset += 1
for _ in range(8):
if not byte & 0x01:
@@ -382,7 +408,7 @@ class PPM:
elif Encoding[layer][y] == 3:#Raw/full
x = 0
for _ in range(32):
byte = ord(data[offset])
byte = data[offset]
offset += 1
for _ in range(8):
if byte & 0x01:
@@ -391,7 +417,7 @@ class PPM:
byte >>= 1
#Merges this frame with the previous frame if NewFrame isn't true:
if not NewFrame and PrevFrame.all() <> None:#maybe optimize this better for numpy...
if not NewFrame and PrevFrame is not None:#maybe optimize this better for numpy...
if FrameMove[0] or FrameMove[1]:#Moves the previous frame if specified:
NewPrevFrame = np.zeros((2, 256, 192), dtype=np.bool_)
@@ -406,7 +432,7 @@ class PPM:
PrevFrame = NewPrevFrame
#merge the frames:
Frame = Frame <> PrevFrame
Frame = Frame != PrevFrame
return Frame
@@ -422,9 +448,9 @@ class PPM:
# - instance.FrameCount: The number of frames this flipnote consists of
# - instance.Locked: A boolean telling us wether it's locked or not
# - instance.ThumbnailFrameIndex: Which frame the thumbnail is(starting at 0)
# - instance.OriginalAuthorName: The name of the original author in unicode
# - instance.EditorAuthorName: The name of the (previous?) editing author in unicode
# - instance.Username: The username in unicode
# - instance.OriginalAuthorName: The name of the original author in str
# - instance.EditorAuthorName: The name of the (previous?) editing author in str
# - instance.Username: The username in str
# - instance.OriginalAuthorID: The ID of the original author
# - instance.EditorAuthorID: The ID of the last editor, the last user to save the file
# - instance.PreviousEditAuthorID: The ID of the previous editor
@@ -443,7 +469,7 @@ class TMB:
f.close()
return ret
def Read(self, data, DecodeThumbnail=False):
if data[:4] <> "PARA" or len(data) < 0x6a0:
if data[:4] != b"PARA" or len(data) < 0x6a0:
return False
#Read the header:
@@ -451,25 +477,25 @@ class TMB:
self.AudioLenght = AscDec(data[8:12], True)#only stored for self.Pack()
self.FrameCount = AscDec(data[12:14], True) + 1
self.Locked = ord(data[0x10]) & 0x01 == 1
self.Locked = data[0x10] & 0x01 == 1
self.ThumbnailFrameIndex = AscDec(data[0x12:0x14], True)#which frame is in the thumbnnail
#self.OriginalAuthorName = u"".join(unichr(AscDec(data[0x14+i*2:0x14+i*2+2], True)) for i in xrange(11)).split(u"\0")[0]
#self.EditorAuthorName = u"".join(unichr(AscDec(data[0x2A+i*2:0x2A+i*2+2], True)) for i in xrange(11)).split(u"\0")[0]
#self.Username = u"".join(unichr(AscDec(data[0x40+i*2:0x40+i*2+2], True)) for i in xrange(11)).split(u"\0")[0]
#self.OriginalAuthorName = u"".join(chr(AscDec(data[0x14+i*2:0x14+i*2+2], True)) for i in range(11)).split(u"\0")[0]
#self.EditorAuthorName = u"".join(chr(AscDec(data[0x2A+i*2:0x2A+i*2+2], True)) for i in range(11)).split(u"\0")[0]
#self.Username = u"".join(chr(AscDec(data[0x40+i*2:0x40+i*2+2], True)) for i in range(11)).split(u"\0")[0]
self.OriginalAuthorName = data[0x14:0x2A].decode("UTF-16LE").split(u"\0")[0]
self.EditorAuthorName = data[0x2A:0x40].decode("UTF-16LE").split(u"\0")[0]
self.Username = data[0x40:0x56].decode("UTF-16LE").split(u"\0")[0]
self.OriginalAuthorID = data[0x56:0x5e][::-1].encode("HEX").upper()
self.EditorAuthorID = data[0x5E:0x66][::-1].encode("HEX").upper()#the last user to save the file
self.OriginalAuthorID = data[0x56:0x5e][::-1].hex().upper()
self.EditorAuthorID = data[0x5E:0x66][::-1].hex().upper()#the last user to save the file
self.OriginalFilenameC = data[0x66:0x78]#compressed
self.CurrentFilenameC = data[0x78:0x8a]#compressed
self.OriginalFilename = "%s_%s_%s.tmb" % (self.OriginalFilenameC[:3].encode("HEX").upper(), self.OriginalFilenameC[3:-2], str(AscDec(self.OriginalFilenameC[-2:], True)).zfill(3))
self.CurrentFilename = "%s_%s_%s.tmb" % (self.CurrentFilenameC[:3].encode("HEX").upper(), self.CurrentFilenameC[3:-2], str(AscDec(self.CurrentFilenameC[-2:], True)).zfill(3))
self.OriginalFilename = "%s_%s_%s.tmb" % (self.OriginalFilenameC[:3].hex().upper(), self.OriginalFilenameC[3:-2].decode("ascii", "replace"), str(AscDec(self.OriginalFilenameC[-2:], True)).zfill(3))
self.CurrentFilename = "%s_%s_%s.tmb" % (self.CurrentFilenameC[:3].hex().upper(), self.CurrentFilenameC[3:-2].decode("ascii", "replace"), str(AscDec(self.CurrentFilenameC[-2:], True)).zfill(3))
self.PreviousEditAuthorID = data[0x8a:0x92][::-1].encode("HEX").upper()#don't know what this really is
self.PreviousEditAuthorID = data[0x8a:0x92][::-1].hex().upper()#don't know what this really is
self.PartialFilenameC = data[0x92:0x9a]#compressed
@@ -497,27 +523,27 @@ class TMB:
#realself = self
#if ppm: self = ppm
out = ["PARA",#magic
out = [b"PARA",#magic
DecAsc(self.AudioOffset-0x6a0, 4, True),#animation data size
DecAsc(self.AudioLenght, 4, True),#audio data size
DecAsc(self.FrameCount-1, 2, True),#frame count
"\x24\x00",#unknown
chr(self.Locked), "\0",#locked
b"\x24\x00",#unknown
bytes((int(bool(self.Locked)),)), b"\0",#locked
DecAsc(self.ThumbnailFrameIndex, 2, True),#which frame is in the thumbnnail
self.OriginalAuthorName.encode("UTF-16LE") + "\0\0"*(11-len(self.OriginalAuthorName)),#Original Author Name
self.EditorAuthorName.encode("UTF-16LE") + "\0\0"*(11-len(self.EditorAuthorName)),#Editor Author Name
self.Username.encode("UTF-16LE") + "\0\0"*(11-len(self.Username)),#Username
self.OriginalAuthorID.decode("HEX")[::-1],#OriginalAuthorID
self.EditorAuthorID.decode("HEX")[::-1],#EditorAuthorID
self.OriginalAuthorName.encode("UTF-16LE") + b"\0\0"*(11-len(self.OriginalAuthorName)),#Original Author Name
self.EditorAuthorName.encode("UTF-16LE") + b"\0\0"*(11-len(self.EditorAuthorName)),#Editor Author Name
self.Username.encode("UTF-16LE") + b"\0\0"*(11-len(self.Username)),#Username
bytes.fromhex(self.OriginalAuthorID)[::-1],#OriginalAuthorID
bytes.fromhex(self.EditorAuthorID)[::-1],#EditorAuthorID
self.OriginalFilenameC,#OriginalFilename
self.CurrentFilenameC,#CurrentFilename
self.PreviousEditAuthorID.decode("HEX")[::-1],#EditorAuthorID
bytes.fromhex(self.PreviousEditAuthorID)[::-1],#EditorAuthorID
self.PartialFilenameC,#PartialFilename
DecAsc(self.Date, 4, True),#Date in seconds
"\0\0",#padding
b"\0\0",#padding
self.PackThumbnail()]#thumbnail
return "".join(out)
return b"".join(out)
def GetThumbnail(self, force=False):
if (self.Thumbnail is None or force):# and self.RawThumbnail:
global ThumbPalette
@@ -535,7 +561,7 @@ class TMB:
for y in range(8):
for x in range(0,8,2):
#two colors stored in each byte:
byte = ord(self.RawThumbnail[(ty*512+tx*64+y*8+x)/2])
byte = self.RawThumbnail[(ty*512+tx*64+y*8+x)//2]
out[x+tx*8 , y+ty*8] = palette[byte & 0xF]
out[x+tx*8+1, y+ty*8] = palette[byte >> 4]
@@ -560,7 +586,7 @@ class TMB:
0x00FF00FF)#F-
if not self.Thumbnail:
if self.Thumbnail is None:
return self.RawThumbnail
else:
if Exact:
@@ -572,12 +598,12 @@ class TMB:
for y in range(8):
for x in range(0,8,2):
#two colors stored in each byte:
#pos = 0xa0+(ty*512+tx*64+y*8+x)/2
#pos = 0xa0+(ty*512+tx*64+y*8+x)//2
p1 = palette.index(int(self.Thumbnail[x+tx*8 , y+ty*8]))
p2 = palette.index(int(self.Thumbnail[x+tx*8+1, y+ty*8]))
out.append(chr(p2<<4 | p1))
out.append(p2<<4 | p1)
self.RawThumbnail = "".join(out)
self.RawThumbnail = bytes(out)
return self.RawThumbnail
else:
#not yet implented
@@ -591,7 +617,7 @@ class TMB:
# This function requires the PIl imaging module
def WriteImage(image, outputPath, scale=1):
if not hasPIL:
print "Error: PIL not found!"
print("Error: PIL not found!")
return False
#if not image: return False
@@ -603,7 +629,7 @@ def WriteImage(image, outputPath, scale=1):
else:
newImg = image
out = newImg.tostring("F")
out = newImg.tobytes(order="F")
out = Image.frombytes("RGBA", (len(newImg), len(newImg[0])), out)
filetype = outputPath[outputPath.rfind(".")+1:]
@@ -611,7 +637,9 @@ def WriteImage(image, outputPath, scale=1):
return True
def get_metadata(flipnote):
def get_metadata(flipnote, filetype=None):
if filetype is None:
filetype = "ppm" if isinstance(flipnote, PPM) else "tmb"
epoch = time.mktime(time.struct_time([2000, 1, 1, 0, 0, 0, 5, 1, -1]))
meta = {
@@ -636,8 +664,8 @@ def get_metadata(flipnote):
return meta
def DumpFrames(flipnote,directory,scale=1):
for i in xrange(flipnote.FrameCount):
print "Dumping frame #%i of %i..." % (i+1, flipnote.FrameCount)
for i in range(flipnote.FrameCount):
print("Dumping frame #%i of %i..." % (i+1, flipnote.FrameCount))
WriteImage(flipnote.GetFrame(i), os.path.join(directory, "frame %s.png" % str(i+1).zfill(3)), scale)
def DumpSoundFiles(flipnote,directory,raw=False):
@@ -657,161 +685,159 @@ def DumpSFXUsage(flipnote,directory):
f.write("Frame %i:%s%s%s\n" % (i, " SFX1"*s1, " SFX2"*s2, " SFX3"*s3))
if __name__ == '__main__':
print " == PPM.py =="
print " == by pbsds =="
print " == v1.3 =="
print
print(" == PPM.py ==")
print(" == by pbsds ==")
print(" == v1.3 ==")
print()
if len(sys.argv) < 3:
print "Usage:"
print " PPM.py <Mode> <Input> [<Output>] [<Frame>] [<Option>]"
print ""
print " <Mode>:"
print " -t: Extracts the thumbnail to the file <Output>"
print " -f: Extracts the frame(s) to <Output>"
print " -s: Dumps the sound files to the folder <Output>"
print " -S: Same as mode -s, but will also dump the raw sound data files"
print " -e: Exports the flipnote to an MKV with minimal transcoding"
print " -E: Same as -e, but will encode the video as lossless H.265"
print " -m: Prints out the metadata. Can also write it to <output> which also"
print " supports unicode charactes."
print " -oa: Seach a directory for an original author that matches the RegEx"
print " Mode -t and -m supports TMB files aswell"
print " <Frame>"
print " Only used in mode -f"
print " Set this to the exact frame you want to extract(starting at 1) and it"
print " will be saved as a file to <Output>."
print " If not specified, it will extract all frames to the folder <Output>"
print " <Option>"
print " --speed N: Only used in mode -e. Set this to force a specific"
print " flipnote speed (1 to 8)."
print " --scale N: Only used in modes -e and -f. Set this to upscale the"
print " frames N times."
print("Usage:")
print(" PPM.py <Mode> <Input> [<Output>] [<Frame>] [<Option>]")
print("")
print(" <Mode>:")
print(" -t: Extracts the thumbnail to the file <Output>")
print(" -f: Extracts the frame(s) to <Output>")
print(" -s: Dumps the sound files to the folder <Output>")
print(" -S: Same as mode -s, but will also dump the raw sound data files")
print(" -e: Exports the flipnote to an MKV with minimal transcoding")
print(" -E: Same as -e, but will encode the video as lossless H.265")
print(" -m: Prints out the metadata. Can also write it to <output> which also")
print(" supports str charactes.")
print(" -oa: Seach a directory for an original author that matches the RegEx")
print(" Mode -t and -m supports TMB files aswell")
print(" <Frame>")
print(" Only used in mode -f")
print(" Set this to the exact frame you want to extract(starting at 1) and it")
print(" will be saved as a file to <Output>.")
print(" If not specified, it will extract all frames to the folder <Output>")
print(" <Option>")
print(" --speed N: Only used in mode -e. Set this to force a specific")
print(" flipnote speed (1 to 8).")
print(" --scale N: Only used in modes -e and -f. Set this to upscale the")
print(" frames N times.")
sys.exit()
import os, time
if sys.argv[1] == "-t":
print "Reading the flipnote file...",
print("Reading the flipnote file...", end=" ")
if not os.path.isfile(sys.argv[2]):
print "Error!\nSpecified file doesn't exist!"
print("Error!\nSpecified file doesn't exist!")
sys.exit()
flipnote = TMB().ReadFile(sys.argv[2], True)
if not flipnote:
print "Error!"
print "The given file is not a Flipnote PPM file or TMB file!"
print("Error!")
print("The given file is not a Flipnote PPM file or TMB file!")
sys.exit()
print "Done!"
print("Done!")
print "Dumping the thumbnail...",
print("Dumping the thumbnail...", end=" ")
WriteImage(flipnote.GetThumbnail(), sys.argv[3])
print "Done!"
print("Done!")
elif sys.argv[1] == "-f":
if len(sys.argv) < 4:
print "Error!"
print "<Output> not specified!"
print("Error!")
print("<Output> not specified!")
sys.exit()
try:
scale = int(sys.argv[sys.argv.index("--scale")+1])
print "Using frame scaling "+str(scale)
print("Using frame scaling "+str(scale))
except (IndexError,ValueError):
scale = 1
print "Reading the flipnote file...",
print("Reading the flipnote file...", end=" ")
if not os.path.isfile(sys.argv[2]):
print "Error!\nSpecified file doesn't exist!"
print("Error!\nSpecified file doesn't exist!")
sys.exit()
flipnote = PPM().ReadFile(sys.argv[2])
if not flipnote:
print "Error!\nThe given file is not a Flipnote PPM file."
print("Error!\nThe given file is not a Flipnote PPM file.")
sys.exit()
print "Done!"
print("Done!")
if len(sys.argv) < 5:
if not os.path.isdir(sys.argv[3]):
print "Error!\nThe specified directory doesn't exist!"
print("Error!\nThe specified directory doesn't exist!")
sys.exit()
DumpFrames(flipnote,sys.argv[3], scale)
print "Done!"
print("Done!")
else:
try:
int(sys.argv[4])
except:
print "Error!\nInvalid <Frame>!"
print("Error!\nInvalid <Frame>!")
sys.exit()
if not (0 <= int(sys.argv[4])-1 < flipnote.FrameCount):
print "Error!\n<Frame> is out of bounds!"
print("Error!\n<Frame> is out of bounds!")
sys.exit()
print "Dumping frame #%i..." % int(sys.argv[4]),
print("Dumping frame #%i..." % int(sys.argv[4]), end=" ")
WriteImage(flipnote.GetFrame(int(sys.argv[4])-1), sys.argv[3], scale)
print "Done!"
print("Done!")
elif sys.argv[1] in ("-s", "-S"):
if len(sys.argv) < 4:
print "Error!"
print "<Output> not specified!"
print("Error!")
print("<Output> not specified!")
sys.exit()
print "Reading the flipnote file...",
print("Reading the flipnote file...", end=" ")
if not os.path.isfile(sys.argv[2]):
print "Error!\nSpecified file doesn't exist!"
print("Error!\nSpecified file doesn't exist!")
sys.exit()
flipnote = PPM().ReadFile(sys.argv[2], ReadFrames=False, ReadSound=True)
if not flipnote:
print "Error!\nThe given file is not a Flipnote PPM file."
print("Error!\nThe given file is not a Flipnote PPM file.")
sys.exit()
print "Done!"
print("Done!")
if not os.path.isdir(sys.argv[3]):
print "Error!\nThe specified output directory doesn't exist!"
print("Error!\nThe specified output directory doesn't exist!")
sys.exit()
print "Converting the sound files...",
print("Converting the sound files...", end=" ")
DumpSoundFiles(flipnote,sys.argv[3],raw=(sys.argv[1]=="-S"))
print "Done!"
print("Done!")
print "Dumping the sound effect usage...",
print("Dumping the sound effect usage...", end=" ")
DumpSFXUsage(flipnote,sys.argv[3])
print "Done!"
print("Done!")
elif sys.argv[1] == "-m":
if not os.path.isfile(sys.argv[2]):
print "Error!\nSpecified file doesn't exist!"
print("Error!\nSpecified file doesn't exist!")
sys.exit()
filetype = "ppm" if sys.argv[2][-3:] == "ppm" else "tmb"
flipnote = TMB().ReadFile(sys.argv[2]) if filetype == "tmb" else PPM().ReadFile(sys.argv[2], ReadFrames=False)
if not flipnote:
print "Error!\nThe given file is not a Flipnote PPM file or TMB file."
print("Error!\nThe given file is not a Flipnote PPM file or TMB file.")
if len(sys.argv) >= 4:
f = open(sys.argv, "wb")
f.write("Error!\nThe given file is not a Flipnote PPM file or TMB file.")
f.close()
with open(sys.argv[3], "w", encoding="UTF-8") as f:
f.write("Error!\nThe given file is not a Flipnote PPM file or TMB file.\n")
sys.exit()
meta = get_metadata(flipnote)
meta = get_metadata(flipnote, filetype)
newline = "\n"
if sys.platform in ("win32", "cygwin"): newline = "\r\n"
elif sys.platform in ("darwin"): newline = "\r"
print newline.join(["\t".join([unicode(char) for char in line]) for line in meta.items()]).encode('ascii', 'ignore')
metadata_text = newline.join(f"{key}\t{value}" for key, value in meta.items())
print(metadata_text)
if len(sys.argv) >= 4:
f = open(sys.argv[3], "wb")
f.write(newline.join(meta).encode("UTF-8"))
f.close()
with open(sys.argv[3], "w", encoding="UTF-8", newline="") as f:
f.write(metadata_text + newline)
elif sys.argv[1] == "-oa":
regex = re.compile(sys.argv[2])
os.chdir(sys.argv[3])
for filename in os.listdir("."):
epoch = time.mktime(time.struct_time([2000, 1, 1, 0, 0, 0, 5, 1, -1]))
if not os.path.isfile(filename):
print "Error!\nSpecified file doesn't exist!"
print("Error!\nSpecified file doesn't exist!")
sys.exit()
filetype = "ppm" if filename[-3:].lower() == "ppm" else "tmb"
@@ -821,11 +847,11 @@ if __name__ == '__main__':
meta = get_metadata(flipnote)
if regex.match(meta["Original author"]):
print filename
print(filename)
elif sys.argv[1] == "-e" or sys.argv[1] == "-E":
elif sys.argv[1] in ("-e", "-E"):
if not hasffmpeg:
print "Error!\nffmpeg is not installed."
print("Error!\nffmpeg is not installed.")
sys.exit()
in_file = sys.argv[2]
out_file = sys.argv[3]
@@ -833,30 +859,30 @@ if __name__ == '__main__':
sleep_time = int(sys.argv[sys.argv.index("--sleep")+1])
except (IndexError,ValueError):
sleep_time = 0
print "Not sleeping."
print("Not sleeping.")
try:
forced_speed = int(sys.argv[sys.argv.index("--speed")+1])
print "Using forced speed "+str(forced_speed)
print("Using forced speed "+str(forced_speed))
except (IndexError,ValueError):
forced_speed = None
try:
scale = int(sys.argv[sys.argv.index("--scale")+1])
print "Using frame scaling "+str(scale)
print("Using frame scaling "+str(scale))
except (IndexError,ValueError):
scale = 1
if out_file.lower()[-4:] != ".mkv":
out_file += ".mkv"
if not os.path.isfile(in_file):
print "Error!\nSpecified file doesn't exist!"
print("Error!\nSpecified file doesn't exist!")
sys.exit()
if os.path.isfile(out_file):
print "Overwrite existing file?"
print("Overwrite existing file?")
overwrite = ""
while overwrite != "y" and overwrite != "n":
overwrite = raw_input("(Y/N) ").lower()
overwrite = input("(Y/N) ").lower()
if overwrite == "n":
print "Not overwriting; exiting."
print("Not overwriting; exiting.")
sys.exit()
filetype = "ppm" if in_file[-3:].lower() == "ppm" else "tmb"
flipnote = TMB().ReadFile(in_file) if filetype == "tmb" else PPM(forced_speed).ReadFile(in_file, ReadFrames=True, ReadSound=True)
@@ -864,63 +890,62 @@ if __name__ == '__main__':
# Make temp dir and dump the frames and sound here
tempdir = tempfile.mkdtemp()
os.mkdir(tempdir+"/sounds")
print "Dumping the frames..."
print("Dumping the frames...")
DumpFrames(flipnote,tempdir,scale)
print "Done!"
print "Dumping the sounds..."
print("Done!")
print("Dumping the sounds...")
DumpSoundFiles(flipnote,tempdir+"/sounds")
print "Done!"
print "Dumping SFX usage..."
print("Done!")
print("Dumping SFX usage...")
DumpSFXUsage(flipnote,tempdir+"/sounds")
print "Done!"
print("Done!")
# Now we need the metadata so we can look up the FPS
SPEEDS = [None,0.5,1,2,4,6,12,20,30]
print "Getting metadata..."
print("Getting metadata...")
metadata = get_metadata(flipnote)
print "Done!"
print("Done!")
speed = int(metadata["Frame speed"])
fps = SPEEDS[speed]
duration = float(metadata["Number of frames"])/float(fps)
print "Flipnote is speed {speed}, so {fps} FPS for {dur} seconds".format(speed=speed,fps=fps,dur=duration)
print("Flipnote is speed {speed}, so {fps} FPS for {dur} seconds".format(speed=speed,fps=fps,dur=duration))
# Now to make the video in ffmpeg
print "Exporting video with ffmpeg..."
print("Exporting video with ffmpeg...")
export_command = ["ffmpeg","-framerate",str(fps),"-start_number","1","-i","{path}/frame %03d.png".format(path=tempdir),"-i","{path}/sounds/BGM.wav".format(path=tempdir),"-c:v","copy","-c:a","copy","-t","{dur}".format(dur=duration),"-y",out_file]
if not os.path.isfile(tempdir+"/sounds/BGM.wav"):
print "No background music. Adding silent track..."
print("No background music. Adding silent track...")
#has_bgm = False
export_command = ["ffmpeg","-framerate",str(fps),"-start_number","1","-i","{path}/frame %03d.png".format(path=tempdir),"-f","lavfi","-i","anullsrc=r=8192:cl=mono","-c:v","copy","-c:a","pcm_s16le","-shortest","-y",out_file]
else:
#has_bgm = True
pass
# use libx265 in lossless mode, rather than MPNG
# Latest master also supports a lossless H.265 export mode.
if sys.argv[1] == "-E":
# insert the necessary arguments into the existing command
codec_index = export_command.index("-c:v")
export_command[codec_index + 1] = "libx265"
export_command.insert(codec_index+2, "-x265-params")
export_command.insert(codec_index+3, "lossless=1")
export_command.insert(codec_index+2, "-preset")
export_command.insert(codec_index+3, "veryslow")
print " ".join(export_command)
with open(os.devnull) as null:
export_command[codec_index + 2:codec_index + 2] = [
"-preset", "veryslow", "-x265-params", "lossless=1"
]
print(" ".join(export_command))
with open(os.devnull, "w") as null:
subprocess.call(export_command,stdout=null,stderr=null)
print "Done!"
print("Done!")
# If the audio has been sped up, we have to do it again manually
bgm_speed = int(metadata["BGM Frame speed"])
if bgm_speed != speed:
print "Background music speed must be modified!"
print("Background music speed must be modified!")
original_rate = 8192
newrate = 8192*(float(fps)/SPEEDS[bgm_speed])
print "Using new rate: "+str(newrate)
print("Using new rate: "+str(newrate))
speed_change_command = ["ffmpeg","-i","{path}/sounds/BGM.wav".format(path=tempdir),"-filter_complex","asetrate={rate}".format(rate=newrate),"-i",out_file,"-map","0:a","-map","1:v","-vcodec","copy","-acodec","pcm_s16le","{path}/temp_out.mkv".format(path=tempdir)]
with open(os.devnull,"w") as null:
subprocess.call(speed_change_command,stdout=null,stderr=null)
os.remove(out_file)
shutil.move("{path}/temp_out.mkv".format(path=tempdir),out_file)
print "Done!"
print("Done!")
# These are the ffmpeg commands I need for each sound effect
# The first generates a silent track, with variable length. This gets concatenated to the front of the sound effect.
@@ -938,10 +963,10 @@ if __name__ == '__main__':
## else:
# Read in the sound effect usage data
print "Reading sound effect usage..."
print("Reading sound effect usage...")
with open("{path}/sounds/SFX usage.txt".format(path=tempdir),"r") as sfx_usage_file:
sfx_usage = sfx_usage_file.read().split("\n")
print "Done!"
print("Done!")
# Iterate through the frames, checking if sound effects need to be added
for frame in range(len(sfx_usage)):
@@ -950,7 +975,7 @@ if __name__ == '__main__':
sfx = line.split(":")[1].strip() if line.strip() != "" else ""
if sfx != "": # If a sound effect must be played...
length = frame/float(fps)
print "Adding "+sfx+" at {length} seconds into the video.".format(length=length)
print("Adding "+sfx+" at {length} seconds into the video.".format(length=length))
# ...run each command in series with the correct arguments
with open(os.devnull,"w") as null:
subprocess.call([i.format(path=tempdir,sfx=sfx,length=length) for i in silence_command],stdout=null,stderr=null)
@@ -958,7 +983,7 @@ if __name__ == '__main__':
subprocess.call([i.format(path=tempdir,video_length=fps*len(sfx_usage)) for i in merge_command],stdout=null,stderr=null)
os.remove(out_file)
shutil.move("{path}/temp_out.mkv".format(path=tempdir),out_file)
print "Done!"
print("Done!")
time.sleep(sleep_time) # optional sleep -- in case you want to slow things down for HDD strain or reliability or something
## subprocess.call(normalise_command)
@@ -967,9 +992,9 @@ if __name__ == '__main__':
# Remove the temp dir and all files in it
print "Removing temporary directory..."
print("Removing temporary directory...")
shutil.rmtree(tempdir)
print "Done!"
print("Done!")
else:
print "Error!\nThere's no such mode."
print("Error!\nThere's no such mode.")
README.md
+46
View File
@@ -0,0 +1,46 @@
# Hatenatools3
**Python 3 port by RekuNote (aka `rekushiiiiii`)**
Hatenatools3 is a Python 3 port of **Hatenatools**, originally created by **pbsds (Peder Bergebakken Sundt)**.
A separate Python 3 port of Hatenatools already exists. Hatenatools3 is not intended to claim precedence over that work. This port is specifically maintained and optimised for compatibility with **RexiMemo's Python 3 Hatena server**:
https://github.com/RexiMemo-org/hatena-server-py3
## Compatibility goal
The distribution is named **Hatenatools3**, but the internal Python package directory intentionally remains named `Hatenatools`. This preserves the layout expected by `hatena-server-py3` and allows the tools to be used as a drop-in replacement/update without renaming the server's module paths.
The port preserves the newer Hatenatools master feature set while adding Python 3 compatibility work, including support for Python 3.13+ where the standard-library `audioop` module is no longer available.
## Included tools
- `PPM.py` v1.5 — PPM/TMB metadata, thumbnails, frames, sound, and ffmpeg export (`-e` / `-E`).
- `UGO.py` v0.93 — UGO read/write, item editing, embedded assets, and XML conversion.
- `NTFT.py` v0.95 — NTFT image decoding/encoding with Pillow support.
## Requirements
- Python 3
- NumPy
- Pillow for image functionality
- ffmpeg for optional PPM video export
## Credits
- Original Hatenatools: pbsds (Peder Bergebakken Sundt)
- Hatenatools3 Python 3 port: RekuNote (aka `rekushiiiiii`)
- Steven — DSiBrew documentation and frame decoding example
- Remark — 8x8 preview-image tiling help
- JSAfive — `.tmb` and `.ugo` samples
- Austin Burk, Midmad, and WDLmaster — sound codec research
- jaames — proper p-frame translation
- WORD559 — ffmpeg video export
- wyntrr-end — ffmpeg export duration and frame scaling
## License
Hatenatools3 retains the original Hatenatools **AGPL-3.0** licensing. See `License.txt`.
Original format documentation: https://github.com/pbsds/hatena-server/wiki
ReadMe.txt
+43 -15
View File
@@ -1,39 +1,67 @@
== Hatenatools ==
== By pbsds ==
== aka Peder Bergebakken Sundt ==
== Hatenatools3 ==
== Python 3 port by RekuNote ==
== aka rekushiiiiii ==
Based on Hatenatools by pbsds (Peder Bergebakken Sundt).
--VERSIONS--
PPM.py: v1.5
UGO.py: v0.93
NTFT.py: v0.95
--ABOUT HATENATOOLS3--
Hatenatools3 is a Python 3 port of the supplied Hatenatools master sources,
ported by RekuNote (aka rekushiiiiii).
A separate Python 3 port of Hatenatools already exists. Hatenatools3 is not
intended to claim precedence over that work; this port is specifically kept
compatible with and optimised for the RexiMemo Python 3 Hatena server:
https://github.com/RexiMemo-org/hatena-server-py3
For drop-in compatibility with hatena-server-py3, the Python package directory
remains named "Hatenatools" even though the project/distribution is named
"Hatenatools3".
This port preserves the newer master feature set, including PPM's -e and -E
video export modes, while keeping Python 3 compatibility fixes such as the
internal ADPCM decoder used on Python 3.13+, where audioop is no longer
available.
--DESCRIPTION--
PPM.py can read the metadata, thumbnail, frames and the sound of Flipnote Studio files: .ppm
PPM.py can read the metadata, thumbnail, frames and sound of Flipnote Studio files: .ppm
It can also handle .tmb files.
UGO.py can read and write UGO files, and modify their content. It can also convert UGO files
to and from the XML format
UGO.py can read and write UGO files, modify their content, and convert UGO files
to and from the XML format.
NTFT.py can read ntft image files and convert them to PIL supported image files
NTFT.py can read NTFT image files and convert them to PIL-supported image files.
These are all tested and written in python 2.7
These are all designed to be usable both as python modules and as stand alone applications.
NTFT.py and PPM.py requires Numpy and PIL
These tools are designed to be usable both as Python modules and as stand-alone applications.
NTFT.py and PPM.py require NumPy and Pillow for image support. PPM video export requires ffmpeg.
--HATENA-SERVER-PY3 COMPATIBILITY--
The internal Hatenatools/ layout is intentionally preserved so this directory can
replace or update the bundled Hatenatools directory in hatena-server-py3 without
requiring the server's import/module paths to be renamed.
Target project:
https://github.com/RexiMemo-org/hatena-server-py3
--DOCUMENTATION--
Documentation on the formats can be found here:
Original Hatenatools / Hatena server format documentation:
https://github.com/pbsds/hatena-server/wiki
--LICENSE--
Hatenatools is licensed under AGPL3
See License.txt or http://www.gnu.org/licenses/agpl-3.0.html for more information
Hatenatools3 retains the original Hatenatools AGPL3 licensing.
See License.txt or http://www.gnu.org/licenses/agpl-3.0.html for more information.
--CREDIT--
-Original Hatenatools by pbsds (Peder Bergebakken Sundt)
-Python 3 port / Hatenatools3 by RekuNote (aka rekushiiiiii)
-Steven for most of the documentation on DSiBrew and his frame decoding example on his talkpage
-Remark for helping me understanding the 8x8 tiling on the preview images.
-Remark for helping with the 8x8 tiling on the preview images
-JSAfive for supplying .tmb and .ugo files
-Austin Burk, Midmad on hatena haiku and WDLmaster on hcs64.com for determining the sound codec
-Austin Burk, Midmad on Hatena Haiku and WDLmaster on hcs64.com for determining the sound codec
-jaames, for proper p-frame translation
-WORD559, for video export with ffmpeg
-wyntrr-end, for proper ffmpeg export duration and frame scaling
UGO.py
+343 -429
View File
@@ -1,174 +1,160 @@
# UGO.py by pbsds
# Hatenatools3 Python 3 port by RekuNote aka rekushiiiiii
# AGPL3 licensed
#
#This class reads and writes UGO files.
#It can also export and import it in a xml format
#Extended functionality added by PPM.py
#
#Credits:
#
# -Jsafive for supplying .ugo files
#
#Note:
# The current implentation can possibly mess up the files stored in Section #2, but will do for most needs
# Next version will probably bring a big change in the ugoxml format
#
import sys, os
from base64 import b64encode, b64decode
# Python 3 port. This class reads/writes Flipnote Hatena UGO files and
# imports/exports the project's UGOXML representation.
from __future__ import annotations
import os
import sys
from base64 import b64decode, b64encode
import xml.etree.ElementTree as ET
try:
import PPM
# Import the actual PPM submodule. Importing ``PPM`` from the package
# namespace can resolve to the convenience-exported PPM class instead.
from .PPM import TMB as _TMB
HasPPM = True
except (ImportError, ValueError):
try:
from PPM import TMB as _TMB # type: ignore
HasPPM = True
except ImportError:
_TMB = None
HasPPM = False
#helpers:
def AscDec(ascii, LittleEndian=False):#Converts a ascii string into a decimal
ret = 0
l = map(ord, ascii)
if LittleEndian: l.reverse()
for i in l:
ret = (ret<<8) | i
return ret
def DecAsc(dec, length=None, LittleEndian=False):#Converts a decimal into an ascii string of chosen length
out = []
while dec <> 0:
out.insert(0, dec&0xFF)
dec >>= 8
#"".join(map(chr, out))
if length:
if len(out) > length:
#return "".join(map(chr, out[-length:]))
out = out[-length:]
if len(out) < length:
#return "".join(map(chr, [0]*(length-len(out)) + out))
out = [0]*(length-len(out)) + out
def AscDec(data, LittleEndian=False):
"""Convert a byte string to an integer (legacy Hatenatools helper)."""
if isinstance(data, str):
data = data.encode("latin-1")
return int.from_bytes(bytes(data), "little" if LittleEndian else "big")
def DecAsc(dec, length=None, LittleEndian=False):
"""Convert an integer to bytes (legacy Hatenatools helper)."""
if dec < 0:
raise ValueError("DecAsc does not support negative values")
if length is None:
length = max(1, (dec.bit_length() + 7) // 8) if dec else 0
if length == 0:
return b""
mask = (1 << (length * 8)) - 1
dec &= mask
return dec.to_bytes(length, "little" if LittleEndian else "big")
if LittleEndian: out.reverse()
return "".join(map(chr, out))
def zipalign(length, r=4):
return length + (4 - length % r) if length % r else length
def indentXML(elem, level=0):#"borrowed" from: http://effbot.org/zone/element-lib.htm#prettyprint
return length + (r - length % r) if length % r else length
def indentXML(elem, level=0):
i = "\n" + level * "\t"
if len(elem):
if not elem.text or not elem.text.strip():
elem.text = i + "\t"
if not elem.tail or not elem.tail.strip():
elem.tail = i
for elem in elem:
indentXML(elem, level+1)
if not elem.tail or not elem.tail.strip():
elem.tail = i
else:
if level and (not elem.tail or not elem.tail.strip()):
for child in elem:
indentXML(child, level + 1)
if not child.tail or not child.tail.strip():
child.tail = i
elif level and (not elem.tail or not elem.tail.strip()):
elem.tail = i
#class UGO:
class UGO:
def __init__(self):
self.Loaded = False
self.Items = []
self.Files = []
def ReadFile(self, path):
f = open(path, "rb")
ret = self.Read(f.read())
f.close()
return ret
with open(path, "rb") as f:
return self.Read(f.read())
def Read(self, data):
#Filestructure:
# - Header
# - Table of Contents
# - Padding, normally of length 2
# - Extra Data
# - Padding
#The padding seems to do zipaligning to a length of 4
global HasPPM
if isinstance(data, str):
data = data.encode("latin-1")
data = bytes(data)
#Header:
if data[:4] <> "UGAR": return False#The file isn't a UGO file
Sections = AscDec(data[ 4: 8], True)#could also be version
if Sections >= 1:
self.TableLength = AscDec(data[ 8:12], True)
if Sections >= 2:
self.ExtraLength = AscDec(data[12:16], True)
if Sections > 2:
print "Warning: This UGO file has more than the 2 known sections:",Sections
print "Please send this UGO file to pbsds over at pbsds.net"
print "This file could possibly be read incorrectly..."
headerlength = 8 + Sections*4
if data[:4] != b"UGAR":
return False
#Read table of contents:
#A table where the rows are seperated with newlines, and colons with tabs
if Sections >= 1:
TableOfContents = tuple((i.split("\t") for i in data[headerlength:headerlength+self.TableLength].split("\n")))
sections = AscDec(data[4:8], True)
self.TableLength = AscDec(data[8:12], True) if sections >= 1 else 0
self.ExtraLength = AscDec(data[12:16], True) if sections >= 2 else 0
if sections > 2:
print("Warning: This UGO file has more than the 2 known sections:", sections)
print("Please send this UGO file to pbsds over at pbsds.net")
print("This file could possibly be read incorrectly...")
headerlength = 8 + sections * 4
if sections >= 1:
table_bytes = data[headerlength : headerlength + self.TableLength]
table_text = table_bytes.decode("utf-8")
TableOfContents = tuple(line.split("\t") for line in table_text.split("\n") if line != "")
else:
TableOfContents = []
#Extra data:
if Sections >= 2:
ExtraData = data[zipalign(headerlength+self.TableLength) : zipalign(headerlength+self.TableLength)+self.ExtraLength]
if sections >= 2:
extra_start = zipalign(headerlength + self.TableLength)
ExtraData = data[extra_start : extra_start + self.ExtraLength]
else:
ExtraData = b""
#Parse data:
self.Items = []
#todo: implement this:
self.Files = []#[i] = (filename, filecontent)
#see todo.txt
pos = 0#Extra Data position
tmbcount = 1#used if HasPPM is False
ntftcount = 1#used if no label
self.Files = []
pos = 0
tmbcount = 1
ntftcount = 1
names = []
for i in TableOfContents:
type = int(i[0])
if type == 0:#layout
#I've always seen just numbers here.
#The amount of numbers also differ.
#maybe related to color scheme? or the button scheme/layout
#all pages containing TMBs have so far been: i = ["0", "2", "1"]
self.Items.append(("layout", map(int, i[1:])))
for entry in TableOfContents:
if not entry or not entry[0]:
continue
elif type == 1:#Text on topscreen
num = int(i[1])#always seen as 0, unknown purpose
labels = [b64decode(i[n]).decode("UTF-16LE") for n in xrange(2,7)]#5 labels probably one label for each line of text
item_type = int(entry[0])
if item_type == 0:
self.Items.append(("layout", list(map(int, entry[1:]))))
continue
if item_type == 1:
num = int(entry[1])
labels = [b64decode(entry[n]).decode("UTF-16LE") for n in range(2, 7)]
self.Items.append(("topscreen text", labels, num))
continue
elif type == 2:#catogories(like "reccomended" and "new flipnotes" and "most popular")
#this one may visually change greatly depending on what layout is set in type==0
link = i[1]
label = b64decode(i[2]).decode("UTF-16LE")
selected = int(i[3]) <> 0#bool
if item_type == 2:
link = entry[1]
label = b64decode(entry[2]).decode("UTF-16LE")
selected = int(entry[3]) != 0
self.Items.append(("category", link, label, selected))
continue
elif type == 3:#POST button/link to POST form. "Post Flipnote" uses this
link = i[1]
label = b64decode(i[2]).decode("UTF-16LE")
if item_type == 3:
link = entry[1]
label = b64decode(entry[2]).decode("UTF-16LE")
self.Items.append(("post", link, label))
continue
elif type == 4:#Button
link = i[1]
trait = int(i[2])
label = b64decode(i[3]).decode("UTF-16LE")
other = i[4:]#varies
if item_type == 4:
link = entry[1]
trait = int(entry[2])
label = b64decode(entry[3]).decode("UTF-16LE")
other = list(entry[4:])
#extra data
file = None#== ("filename", "filedata")
embedded = None
if trait < 100 and ExtraData:
if ExtraData == "\x20":#empty
pass#no files
elif ExtraData[pos:pos+4] == "PARA":#tmb file
file = ExtraData[pos:pos+0x6A0]
if ExtraData == b"\x20":
pass
elif ExtraData[pos : pos + 4] == b"PARA":
filedata = ExtraData[pos : pos + 0x6A0]
pos += 0x6A0
if HasPPM:
tmb = PPM.TMB().Read(file)
name = tmb.CurrentFilename[:-4]
del tmb
if HasPPM and _TMB is not None:
tmb = _TMB().Read(filedata)
name = tmb.CurrentFilename[:-4] if tmb else "embedded tmb #%i" % tmbcount
if not tmb:
tmbcount += 1
else:
name = "embedded tmb #%i" % tmbcount
tmbcount += 1
@@ -178,448 +164,376 @@ class UGO:
while "%s_%i.tmb" % (name, j) in names:
j += 1
name = "%s_%i" % (name, j)
file = (name+".tmb", file)
embedded = (name + ".tmb", filedata)
names.append(name + ".tmb")
else:#ntft icon
name = label.encode("ascii", "ignore")
else:
name = label.encode("ascii", "ignore").decode("ascii")
if not name:
name = "nameless ntft %i" % ntftcount
ntftcount += 1
if name + ".ntft" in names:
j = 2
while "%s_%i.ntft" % (name, j) in names:
j += 1
name = "%s_%i" % (name, j)
file = (name+".ntft", ExtraData[pos:pos+2048])
embedded = (name + ".ntft", ExtraData[pos : pos + 2048])
names.append(name + ".ntft")
pos += 2048
self.Items.append(("button", trait, label, link, other, file))
# if subtype == 3:#flipnote
# tmb = PPM.TMB().Read(ExtraData[pos:pos+0x6A0]); pos += 0x6A0
# unknown1 = i[3]#empty
# stars = int(i[4])#not sure
# unknown2 = map(int, i[5:8])#unknown = [765, 573, 0]
# self.Items.append(("flipnote", link, tmb, stars, unknown1, unknown2))
# continue
# elif subtype == 100: pass
# elif subtype == 101: pass
# elif subtype == 102: pass
# elif subtype == 104:#list item? like mails and announcements
# label = b64decode(i[3]).decode("UTF-16LE")
# unknown = i[4]
# num = int(i[5])#only seen as 0
# # self.Items.append(("list item?", link, label, unknown, num))
# # continue
# pass
# elif subtype == 115:#Labeled button link(size of a flipnote thumbnail, commonly "next page")
# label = b64decode(i[3]).decode("UTF-16LE")
# unknown = i[4:6]
# self.Items.append(("thumbnail link", link, label, unknown))
# continue
# elif subtype == 117: pass
self.Items.append(("button", trait, label, link, other, embedded))
continue
#if not recognized:
self.Items.append(("unknown", i))
print "Unknown UGO item discovered:", i
self.Items.append(("unknown", list(entry)))
print("Unknown UGO item discovered:", entry)
self.Loaded = True
return self
def WriteFile(self, path):
if self.Loaded:
out = self.Pack()
if out:
f = open(path, "wb")
f.write(out)
f.close()
return True
else:
if not self.Loaded:
return False
out = self.Pack()
if out is False:
return False
with open(path, "wb") as f:
f.write(out)
return True
def Pack(self):
if not self.Loaded: return False
if not self.Loaded:
return False
Header = ["UGAR", None]
TableOfContents = []
ExtraData = []
table_rows = []
extra_parts = []
#Encode data:
for i in self.Items:
if i[0] == "unknown":
TableOfContents.append("\t".join(i[1]))
elif i[0] == "layout":#0
TableOfContents.append("\t".join(["0"] + map(str, list(i[1]))))
elif i[0] == "topscreen text":#1
labels, num = i[1:]
num = str(num)
for i in xrange(5):
labels[i] = b64encode(labels[i].encode("UTF-16LE"))
TableOfContents.append("\t".join(("1", num, labels[0], labels[1], labels[2], labels[3], labels[4])))
elif i[0] == "category":#2
link, label, selected = i[1:]
label = b64encode(label.encode("UTF-16LE"))
selected = str(1*selected)
TableOfContents.append("\t".join(("2", link, label, selected)))
elif i[0] == "post":#3
link, label = i[1:]
label = b64encode(label.encode("UTF-16LE"))
TableOfContents.append("\t".join(("3", link, label)))
elif i[0] == "button":#4
trait, label, link, other, file = i[1:]
trait = str(trait)
label = b64encode(label.encode("UTF-16LE"))
TableOfContents.append("\t".join(["4", link, trait, label] + list(other)))
if file:
ExtraData.append(file[1])
for item in self.Items:
kind = item[0]
if kind == "unknown":
table_rows.append("\t".join(map(str, item[1])))
elif kind == "layout":
table_rows.append("\t".join(["0"] + [str(value) for value in item[1]]))
elif kind == "topscreen text":
labels, num = item[1:]
encoded_labels = [b64encode(label.encode("UTF-16LE")).decode("ascii") for label in labels]
# The format expects exactly five labels.
encoded_labels = (encoded_labels + [""] * 5)[:5]
table_rows.append("\t".join(["1", str(num)] + encoded_labels))
elif kind == "category":
link, label, selected = item[1:]
encoded = b64encode(label.encode("UTF-16LE")).decode("ascii")
table_rows.append("\t".join(("2", link, encoded, str(int(bool(selected))))))
elif kind == "post":
link, label = item[1:]
encoded = b64encode(label.encode("UTF-16LE")).decode("ascii")
table_rows.append("\t".join(("3", link, encoded)))
elif kind == "button":
trait, label, link, other, embedded = item[1:]
encoded = b64encode(label.encode("UTF-16LE")).decode("ascii")
table_rows.append("\t".join(["4", link, str(trait), encoded] + [str(v) for v in other]))
if embedded:
extra_parts.append(bytes(embedded[1]))
else:
print "Unrecognized entry in self.Items:", i
TableOfContents = "\n".join(TableOfContents)
ExtraData = "".join(ExtraData)
print("Unrecognized entry in self.Items:", item)
#Format data:
Sections = 0
if TableOfContents:
Sections += 1
Header.append(DecAsc(len(TableOfContents), 4, True))
table = "\n".join(table_rows).encode("utf-8")
extra = b"".join(extra_parts)
#padding/zipaligning
if len(TableOfContents) % 4:
TableOfContents += "\0" * (4 - len(TableOfContents) % 4)
if ExtraData:
Sections += 1
Header.append(DecAsc(len(ExtraData), 4, True))
sections = 0
lengths = []
if table:
sections += 1
lengths.append(len(table))
if extra:
sections += 1
lengths.append(len(extra))
#padding/zipaligning
if len(ExtraData) % 4:
ExtraData += "\0" * (4 - len(ExtraData) % 4)
Header[1] = DecAsc(Sections, 4, True)
header = b"UGAR" + DecAsc(sections, 4, True) + b"".join(DecAsc(length, 4, True) for length in lengths)
if table:
table += b"\0" * ((-len(table)) % 4)
if extra:
extra += b"\0" * ((-len(extra)) % 4)
return header + table + extra
#Zip up and send the file:
return "".join(Header) + TableOfContents + ExtraData
#XML
def WriteXML(self, xmlname="content.ugoxml", folder="content.ugoxml embedded"):#WIP
if not self.Loaded: return False
def WriteXML(self, xmlname="content.ugoxml", folder="content.ugoxml embedded"):
if not self.Loaded:
return False
path, xmlname = os.path.split(xmlname)
path, xml_basename = os.path.split(xmlname)
ugo_xml = ET.Element("ugo_xml")
files = []
for i in self.Items:
if i[0] == "unknown":
elem = ET.SubElement(ugo_xml, "raw", type=i[1][0])
for value in i[1][1:]:
ET.SubElement(elem, "value").text = value
continue
elif i[0] == "layout":#0
elem = ET.SubElement(ugo_xml, "layout")
for value in i[1]:
for item in self.Items:
kind = item[0]
if kind == "unknown":
elem = ET.SubElement(ugo_xml, "raw", type=str(item[1][0]))
for value in item[1][1:]:
ET.SubElement(elem, "value").text = str(value)
continue
elif i[0] == "topscreen text":#1
elif kind == "layout":
elem = ET.SubElement(ugo_xml, "layout")
for value in item[1]:
ET.SubElement(elem, "value").text = str(value)
elif kind == "topscreen text":
elem = ET.SubElement(ugo_xml, "title")
labels, num = i[1:]
labels, num = item[1:]
for label in labels:
ET.SubElement(elem, "label").text = label
ET.SubElement(elem, "num").text = str(num)
continue
elif i[0] == "category":#2
elif kind == "category":
elem = ET.SubElement(ugo_xml, "category")
link, label, selected = i[1:]
link, label, selected = item[1:]
ET.SubElement(elem, "label").text = label
ET.SubElement(elem, "address").text = link
ET.SubElement(elem, "selected").text = str(selected).lower()
elif i[0] == "post":#3
ET.SubElement(elem, "selected").text = str(bool(selected)).lower()
elif kind == "post":
elem = ET.SubElement(ugo_xml, "post")
link, label = i[1:]
link, label = item[1:]
ET.SubElement(elem, "label").text = label
ET.SubElement(elem, "address").text = link
elif i[0] == "button":#4
elif kind == "button":
elem = ET.SubElement(ugo_xml, "button")
trait, label, link, other, file = i[1:]
trait, label, link, other, embedded = item[1:]
ET.SubElement(elem, "label").text = label
ET.SubElement(elem, "address").text = link
ET.SubElement(elem, "trait").text = str(trait)#todo: add names
ET.SubElement(elem, "trait").text = str(trait)
for n, value in enumerate(other):
entry = ET.SubElement(elem, "value")
entry.text = value
entry.text = str(value)
if n == 0 and trait == 3:
entry.attrib["tip"] = "stars"
if embedded:
ET.SubElement(elem, "embedded_file").text = os.path.join(folder, embedded[0])
files.append((os.path.join(folder, embedded[0]), bytes(embedded[1])))
if file:
ET.SubElement(elem, "embedded_file").text = os.path.join(folder, file[0])
files.append((os.path.join(folder, file[0]), file[1]))
#intend
indentXML(ugo_xml)
files.append((xml_basename, ET.tostring(ugo_xml, encoding="UTF-8")))
#pack xml
files.append((xmlname, ET.tostring(ugo_xml, encoding="UTF-8")))
#write files
if not os.path.isdir(os.path.join(path, folder)):
os.mkdir(os.path.join(path, folder))
folder_path = os.path.join(path, folder)
if folder and not os.path.isdir(folder_path):
os.makedirs(folder_path, exist_ok=True)
for name, data in files:
f = open(os.path.join(path, name), "wb")
full_path = os.path.join(path, name)
parent = os.path.dirname(full_path)
if parent:
os.makedirs(parent, exist_ok=True)
with open(full_path, "wb") as f:
f.write(data)
f.close()
return True
def ReadXML(self, xmlfile, silent=True):
ugo_xml = ET.parse(xmlfile).getroot()
xmlpath = os.path.split(xmlfile)[0]
Items = []
items = []
for elem in ugo_xml:
if elem.tag == "raw":
if "type" not in elem.attrib:
if not silent: print "Invalid formatting. <raw> without \"type\" attribute"
if not silent:
print('Invalid formatting. <raw> without "type" attribute')
return False
values = [elem.attrib["type"]]
for value in elem:
if value.tag <> "value":
if not silent: print "Invalid formatting. <%s> found within <unknown>" % value.tag
if value.tag != "value":
if not silent:
print("Invalid formatting. <%s> found within <unknown>" % value.tag)
return False
values.append(value.text if value.text else "")
values.append(value.text or "")
items.append(("unknown", values))
Items.append(("unknown", values))
elif elem.tag == "layout":#0
elif elem.tag == "layout":
values = []
for value in elem:
if value.tag <> "value":
if not silent: print "Invalid formatting. <%s> found within <layout>" % value.tag
if value.tag != "value":
if not silent:
print("Invalid formatting. <%s> found within <layout>" % value.tag)
return False
if not value.text.isdigit():
if not silent: print "Invalid entry. <value> in <layout> is not a number" % value.tag
if not (value.text or "").isdigit():
if not silent:
print("Invalid entry. <value> in <layout> is not a number")
return False
values.append(int(value.text))
Items.append(("layout", values))
elif elem.tag == "title":#1
items.append(("layout", values))
elif elem.tag == "title":
labels = ["", "", "", "", ""]
num = 0
pos = 0
numset = False
for value in elem:
if value.tag not in ("label", "num"):
if not silent: print "Invalid formatting. <%s> found within <title>" % value.tag
if not silent:
print("Invalid formatting. <%s> found within <title>" % value.tag)
return False
if value.tag == "label":
if pos >= 5:
if not silent: print "Invalid formatting. More than 5 <labels> in <title>"
if not silent:
print("Invalid formatting. More than 5 <labels> in <title>")
return False
if value.text: labels[pos] = value.text
labels[pos] = value.text or ""
pos += 1
elif value.tag == "num":
else:
if numset:
if not silent: print "Invalid formatting. Multible <num> in <title>"
if not silent:
print("Invalid formatting. Multiple <num> in <title>")
return False
if not value.text.isdigit():
if not silent: print "Invalid entry. <num> in <title> is not a number!"
if not (value.text or "").isdigit():
if not silent:
print("Invalid entry. <num> in <title> is not a number!")
return False
num = int(value.text)
numset = True
items.append(("topscreen text", labels, num))
Items.append(("topscreen text", labels, num))
elif elem.tag == "category":#2
link = None
label = None
selected = None
elif elem.tag == "category":
link = label = selected = None
for value in elem:
if value.tag not in ("label", "address", "selected"):
if not silent: print "Invalid formatting. <%s> found within <category>" % value.tag
if not silent:
print("Invalid formatting. <%s> found within <category>" % value.tag)
return False
if value.tag == "address":
if isinstance(link, str):
if not silent: print "Invalid formatting. multible <address> within <category>"
if link is not None:
return False
link = value.text if value.text else ""
link = value.text or ""
elif value.tag == "label":
if isinstance(label, str):
if not silent: print "Invalid formatting. multible <label> within <category>"
if label is not None:
return False
label = value.text if value.text else ""
elif value.tag == "selected":
if selected in (True, False):
if not silent: print "Invalid formatting. multible <selected> within <category>"
label = value.text or ""
else:
if selected is not None:
return False
selected = value.text[0].lower() in "t1"
Items.append(("category", link, label, selected))
elif elem.tag == "post":#3
label = None
link = None
selected = bool(value.text) and value.text[0].lower() in "t1"
if link is None or label is None or selected is None:
return False
items.append(("category", link, label, selected))
elif elem.tag == "post":
label = link = None
for value in elem:
if value.tag == "label":
if isinstance(label, str):
if not silent: print "Invalid formatting. Multible <label> within <post>"
if label is not None:
return False
label = value.text if value.text else ""
label = value.text or ""
elif value.tag == "address":
if isinstance(link, str):
if not silent: print "Invalid formatting. Multible <address> within <post>"
if link is not None:
return False
link = value.text if value.text else ""
if None in (link, label):
if not silent: print "Invalid formatting. <button> lacks either a <address> or <label>"
link = value.text or ""
else:
if not silent:
print("Invalid formatting. <%s> found within <post>" % value.tag)
return False
if link is None or label is None:
return False
items.append(("post", link, label))
Items.append(("post", link, label))
elif elem.tag == "button":#4
trait = None#todo: add names
label = None
link = None
elif elem.tag == "button":
trait = label = link = None
other = []
file = None
embedded = None
for value in elem:
if value.tag not in ("label", "address", "trait", "value", "embedded_file"):
if not silent: print "Invalid formatting. <%s> found within <button>" % value.tag
if not silent:
print("Invalid formatting. <%s> found within <button>" % value.tag)
return False
if value.tag == "label":
if isinstance(label, str):
if not silent: print "Invalid formatting. Multible <label> within <button>"
if label is not None:
return False
label = value.text if value.text else ""
label = value.text or ""
elif value.tag == "address":
if isinstance(link, str):
if not silent: print "Invalid formatting. Multible <address> within <button>"
if link is not None:
return False
link = value.text if value.text else ""
elif value.tag == "trait":#todo: add names
if isinstance(trait, str):
if not silent: print "Invalid formatting. Multible <trait> within <button>"
return False
if not value.text.isdigit():
if not silent: print "Invalid entry. <trait> in <button> is not a number"
link = value.text or ""
elif value.tag == "trait":
if trait is not None or not (value.text or "").isdigit():
return False
trait = int(value.text)
elif value.tag == "value":
other.append(value.text if value.text else "")
elif value.tag == "embedded_file":
if file <> None:
if not silent: print "Invalid formatting. Multible <embedded_file> within <button>"
other.append(value.text or "")
else:
if embedded is not None:
return False
path = os.path.join(xmlpath, value.text)
if not os.path.isfile(path):
if not silent: print "Invalid entry. Embedded file \"%s\" not found!" % value.text
print path
if not value.text:
return False
f = open(path, "rb")
file = (os.path.split(value.text)[1], f.read())
f.close()
if None in (trait, label, link):
if not silent: print "Invalid formatting. <button> lacks either a <trait>, a <address> or a <label>"
embedded_path = os.path.join(xmlpath, value.text)
if not os.path.isfile(embedded_path):
if not silent:
print('Invalid entry. Embedded file "%s" not found!' % value.text)
return False
with open(embedded_path, "rb") as f:
embedded = (os.path.split(value.text)[1], f.read())
if trait is None or label is None or link is None:
return False
items.append(("button", trait, label, link, other, embedded))
Items.append(("button", trait, label, link, other, file))
else:
if not silent:
print "Invalid formatting: <%s> found within <ugo_xml>" % elem.tag
print("Invalid formatting: <%s> found within <ugo_xml>" % elem.tag)
return False
self.Items = Items
self.Items = items
self.Loaded = True
return self
if __name__ == "__main__":
print " == UGO.py =="
print " == by pbsds =="
print " == v0.93 =="
print
def _main():
print(" == UGO.py ==")
print(" == by pbsds ==")
print(" == v0.93 ==")
print()
if len(sys.argv) < 2:
print "Usage:"
print " UGO.py [<mode>] <input> [<output> [<foldername>]]"
print ""
print " <Mode>:"
print " -d: Converts the UGO file in <input> to a UGOXML file with the same"
print " name, unless <output> is specified. Any embedded files will be"
print " written to a folder called UGOXML-filename + \" embedded\" unless"
print " <foldername> is given."
print " <foldername> is relative to the XML."
print " -e: Converts the UGOXML file in <input> to a UGO file with the same"
print " name, unless <output> is specified."
print " If mode is not specified, it will try to find out for itself"
sys.exit()
print("Usage:")
print(" UGO.py [<mode>] <input> [<output> [<foldername>]]")
print()
print(" <Mode>:")
print(" -d: Converts the UGO file in <input> to a UGOXML file with the same")
print(" name, unless <output> is specified. Any embedded files will be")
print(' written to a folder called UGOXML-filename + " embedded" unless')
print(" <foldername> is given.")
print(" <foldername> is relative to the XML.")
print(" -e: Converts the UGOXML file in <input> to a UGO file with the same")
print(" name, unless <output> is specified.")
print(" If mode is not specified, it will try to find out for itself")
return 0
mode = sys.argv[1]
if mode not in ("-d", "-e"):
if os.path.exists(mode):#the mode is actually the file
f = open(mode, "rb")
if os.path.exists(mode):
with open(mode, "rb") as f:
magic = f.read(4)
f.close()
if magic == "UGAR":
mode = "-d"
print "No mode specified. UGO -> UGOXML chosen"
else:
mode = "-e"
print "No mode specified. UGOXML -> UGO chosen"
mode = "-d" if magic == b"UGAR" else "-e"
print("No mode specified. %s chosen" % ("UGO -> UGOXML" if mode == "-d" else "UGOXML -> UGO"))
sys.argv.insert(1, mode)
else:
print "Invalid <mode> given!"
sys.exit()
print("Invalid <mode> given!")
return 1
input_path = sys.argv[2]
if mode == "-d":
input = sys.argv[2]
output = sys.argv[3] if len(sys.argv) >= 4 else sys.argv[2]+"xml"
output = sys.argv[3] if len(sys.argv) >= 4 else input_path + "xml"
foldername = sys.argv[4] if len(sys.argv) >= 5 else os.path.split(output)[1] + " embedded"
print "Reading %s..." % os.path.split(input)[1]
ugo = UGO().ReadFile(input)
print("Reading %s..." % os.path.split(input_path)[1])
ugo = UGO().ReadFile(input_path)
if not ugo:
print "Error!\n The given file is not a UGO file!"
sys.exit()
print "Done!"
print "Writing XML..."
print("Error!\n The given file is not a UGO file!")
return 1
print("Writing XML...")
ugo.WriteXML(output, foldername)
print("Done!")
return 0
print "Done!\n\nHave a nice day!"
if mode == "-e":
input = sys.argv[2]
output = sys.argv[3] if len(sys.argv) >= 4 else ".".join(sys.argv[2].split(".")[:-1]) + ".ugo"
print "Reading %s..." % os.path.split(input)[1]
output = sys.argv[3] if len(sys.argv) >= 4 else ".".join(input_path.split(".")[:-1]) + ".ugo"
print("Reading %s..." % os.path.split(input_path)[1])
try:
ugo = UGO().ReadXML(input, False)
except EL.ParseError:
print "Error!\nThe given file is not in the XML format!"
ugo = False
ugo = UGO().ReadXML(input_path, False)
except ET.ParseError:
print("Error!\nThe given file is not in the XML format!")
return 1
if not ugo:
#it prints sufficient errormessages
#print "Error!\n The given file is not a UGO file!"
sys.exit()
print "Done!"
print "Writing UGO..."
return 1
print("Writing UGO...")
ugo.WriteFile(output)
print "Done"
print("Done")
return 0
if __name__ == "__main__":
raise SystemExit(_main())
__init__.py
+15
View File
@@ -0,0 +1,15 @@
"""Hatenatools3 - Python 3 port of Hatenatools, optimised for hatena-server-py3.
Original Hatenatools by pbsds. Python 3 port by RekuNote aka rekushiiiiii.
The package directory remains named ``Hatenatools`` for drop-in server compatibility.
"""
PROJECT_NAME = "Hatenatools3"
PORT_AUTHOR = "RekuNote aka rekushiiiiii"
TARGET_PROJECT = "https://github.com/RexiMemo-org/hatena-server-py3"
from .PPM import PPM, TMB
from .UGO import UGO
from .NTFT import NTFT
__all__ = ["PPM", "TMB", "UGO", "NTFT"]
compile.bat
+4 -3
View File
@@ -1,6 +1,7 @@
pyinstaller --onefile -n "UGO" --console "UGO.py"
pyinstaller --onefile -n "PPM" --console "PPM.py"
pyinstaller --onefile -n "NTFT" --console "NTFT.py"
@echo off
py -3 -m PyInstaller --onefile -n "UGO" --console "UGO.py"
py -3 -m PyInstaller --onefile -n "PPM" --console "PPM.py"
py -3 -m PyInstaller --onefile -n "NTFT" --console "NTFT.py"
move "dist\PPM.exe" "PPM.exe"
move "dist\UGO.exe" "UGO.exe"
move "dist\NTFT.exe" "NTFT.exe"
hatenatools project page.url
+5
View File
@@ -0,0 +1,5 @@
[{000214A0-0000-0000-C000-000000000046}]
Prop3=19,2
[InternetShortcut]
URL=http://pbsds.net/projects/hatenatools
IDList=