Initial release of Hatenatools3

Python 3 port of Hatenatools, optimised for compatibility with RexiMemo-org/hatena-server-py3.

Includes Python 3 compatibility fixes, restored functionality from the original Hatenatools, and additional porting/behavior fixes.
This commit is contained in:
Rekushi authored and GitHub committed 2026-09-02 23:50:33 +01:00
1 parent d5bb128b76
commit c0b38d0d9a
8 files changed
+1078 -1137

No files matched your search

+240 -215
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env python
#PPM.py by pbsds for python 2.7
#!/usr/bin/env python3
#PPM.py by pbsds; Hatenatools3 Python 3 port by RekuNote aka rekushiiiiii
#AGPL3 licensed
#
#Numpy is required
@@ -12,7 +12,7 @@
# - Jsafive for supplying .tmb files
# - Austin Burk, Midmad on hatena haiku and WDLmaster on hcs64.com for determining the sound codec
#
import sys, wave, audioop, re, os #needs os and time aswell in CMD mode (UPDATE 2018/07/04: os used to get devnull for discarding subprocess output)
import sys, wave, re, os, struct # Python 3: audioop was removed in 3.13
import numpy as np
import subprocess # used for ffmpeg
import tempfile, shutil # temporary directory whilst exporting, shutil to clean up after
@@ -21,7 +21,7 @@ try:
from PIL import Image
hasPIL = True
except ImportError:
print "Warning: PIL not found, image extraction won't work!"
print("Warning: PIL not found, image extraction won't work!")
hasPIL = False
# Test for ffmpeg by executing ffmpeg -h; an OSError indicates ffmpeg is not installed
@@ -32,39 +32,65 @@ try:
stderr=null)
hasffmpeg = True
except OSError:
print "Warning: ffmpeg not found, video export unavailable. Please make sure ffmepg is installed and can be accessed on your path."
print("Warning: ffmpeg not found, video export unavailable. Please make sure ffmepg is installed and can be accessed on your path.")
hasffmpeg = False
#helpers:
def AscDec(ascii, LittleEndian=False):#Converts a ascii string into a decimal
ret = 0
l = map(ord, ascii)
if LittleEndian: l.reverse()
for i in l:
ret = (ret<<8) | i
return ret
def DecAsc(dec, length=None, LittleEndian=False):#Converts a decimal into an ascii string of chosen length
out = []
while dec <> 0:
out.insert(0, dec&0xFF)
dec >>= 8
#"".join(map(chr, out))
if length:
if len(out) > length:
#return "".join(map(chr, out[-length:]))
out = out[-length:]
if len(out) < length:
#return "".join(map(chr, [0]*(length-len(out)) + out))
out = [0]*(length-len(out)) + out
if LittleEndian: out.reverse()
return "".join(map(chr, out))
def AscDec(data, LittleEndian=False):#Converts a byte string into a decimal
if isinstance(data, str):
data = data.encode("latin-1")
return int.from_bytes(bytes(data), "little" if LittleEndian else "big")
def DecAsc(dec, length=None, LittleEndian=False):#Converts a decimal into bytes of chosen length
if dec < 0:
raise ValueError("DecAsc does not support negative values")
if length is None:
length = max(1, (dec.bit_length() + 7) // 8) if dec else 0
if length == 0:
return b""
dec &= (1 << (length * 8)) - 1
return dec.to_bytes(length, "little" if LittleEndian else "big")
def AddPadding(i,pad = 0x10):#used mainly for zipaligning offsets
if i % pad <> 0:
if i % pad != 0:
return i + pad - (i % pad)
return i
# Python 3.13 removed audioop. This is the IMA/DVI ADPCM decoder used by
# audioop.adpcm2lin(..., 2, None), retaining the original initial state.
_IMA_INDEX_TABLE = (-1, -1, -1, -1, 2, 4, 6, 8)
_IMA_STEP_TABLE = (
7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34,
37, 41, 45, 50, 55, 60, 66, 73, 80, 88, 97, 107, 118, 130, 143, 157,
173, 190, 209, 230, 253, 279, 307, 337, 371, 408, 449, 494, 544,
598, 658, 724, 796, 876, 963, 1060, 1166, 1282, 1411, 1552, 1707,
1878, 2066, 2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871,
5358, 5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635,
13899, 15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794,
32767,
)
def _decode_ima_adpcm(data):
valpred = 0
index = 0
out = bytearray()
for byte in data:
for delta in (byte >> 4, byte & 0x0F):
step = _IMA_STEP_TABLE[index]
vpdiff = step >> 3
if delta & 4: vpdiff += step
if delta & 2: vpdiff += step >> 1
if delta & 1: vpdiff += step >> 2
if delta & 8:
valpred -= vpdiff
else:
valpred += vpdiff
valpred = max(-32768, min(32767, valpred))
index += _IMA_INDEX_TABLE[delta & 7]
index = max(0, min(88, index))
out.extend(struct.pack("<h", valpred))
return bytes(out)
#palettes:
FramePalette = [0xFFFFFFFF,0x000000FF,0xFF0000FF,0x0000FFFF]
ThumbPalette = (0xFEFEFEFF,#0
@@ -98,9 +124,9 @@ ThumbPalette = (0xFEFEFEFF,#0
# - instance.FrameCount: The number of frames this flipnote consists of
# - instance.Locked: A boolean telling us wether it's locked or not
# - instance.ThumbnailFrameIndex: Which frame the thumbnail is(starting at 0)
# - instance.OriginalAuthorName: The name of the original author in unicode
# - instance.EditorAuthorName: The name of the (previous?) editing author in unicode
# - instance.Username: The username in unicode
# - instance.OriginalAuthorName: The name of the original author in str
# - instance.EditorAuthorName: The name of the (previous?) editing author in str
# - instance.Username: The username in str
# - instance.OriginalAuthorID: The ID of the original author
# - instance.EditorAuthorID: The ID of the last editor, the last user to save the file
# - instance.PreviousEditAuthorID: The ID of the previous editor
@@ -135,7 +161,7 @@ class PPM:
f.close()
return ret
def Read(self, data, DecodeThumbnail=False, ReadFrames=True, ReadSound=False):#Load: (thumbnail, frames, sound)
if data[:4] <> "PARA" or len(data) <= 0x6a0:
if data[:4] != b"PARA" or len(data) <= 0x6a0:
return False
@@ -144,22 +170,22 @@ class PPM:
AudioLenght = AscDec(data[8:12], True)
self.FrameCount = AscDec(data[12:14], True) + 1
self.Locked = ord(data[0x10]) & 0x01 == 1
self.Locked = data[0x10] & 0x01 == 1
self.ThumbnailFrameIndex = AscDec(data[0x12:0x14], True)#which frame the thumbnnail represents
self.OriginalAuthorName = data[0x14:0x2A].decode("UTF-16LE").split(u"\0")[0]
self.EditorAuthorName = data[0x2A:0x40].decode("UTF-16LE").split(u"\0")[0]
self.Username = data[0x40:0x56].decode("UTF-16LE").split(u"\0")[0]
self.OriginalAuthorID = data[0x56:0x5e][::-1].encode("HEX").upper()
self.EditorAuthorID = data[0x5E:0x66][::-1].encode("HEX").upper()#the last user to save the file
self.OriginalAuthorID = data[0x56:0x5e][::-1].hex().upper()
self.EditorAuthorID = data[0x5E:0x66][::-1].hex().upper()#the last user to save the file
self.OriginalFilenameC = data[0x66:0x78]#compressed
self.CurrentFilenameC = data[0x78:0x8a]#compressed
self.OriginalFilename = "%s_%s_%s.ppm" % (self.OriginalFilenameC[:3].encode("HEX").upper(), self.OriginalFilenameC[3:-2], str(AscDec(self.OriginalFilenameC[-2:], True)).zfill(3))
self.CurrentFilename = "%s_%s_%s.ppm" % (self.CurrentFilenameC[:3].encode("HEX").upper(), self.CurrentFilenameC[3:-2], str(AscDec(self.CurrentFilenameC[-2:], True)).zfill(3))
self.OriginalFilename = "%s_%s_%s.ppm" % (self.OriginalFilenameC[:3].hex().upper(), self.OriginalFilenameC[3:-2].decode("ascii", "replace"), str(AscDec(self.OriginalFilenameC[-2:], True)).zfill(3))
self.CurrentFilename = "%s_%s_%s.ppm" % (self.CurrentFilenameC[:3].hex().upper(), self.CurrentFilenameC[3:-2].decode("ascii", "replace"), str(AscDec(self.CurrentFilenameC[-2:], True)).zfill(3))
self.PreviousEditAuthorID = data[0x8a:0x92][::-1].encode("HEX").upper()#don't know what this really is
self.PreviousEditAuthorID = data[0x8a:0x92][::-1].hex().upper()#don't know what this really is
#self.PartialFilenameC = data[0x92:0x9a]#compressed
@@ -172,21 +198,21 @@ class PPM:
self.Loaded[0] = True
#read the animation sequence header:
self.Looped = ord(data[0x06A6]) >> 1 & 0x01 == 1#Unverified?
self.Looped = data[0x06A6] >> 1 & 0x01 == 1#Unverified?
AnimationOffset = 0x6a8 + AscDec(data[0x6a0:0x6a4], True)
FrameOffsets = [AnimationOffset + AscDec(data[0x06a8+i*4:0x06a8+i*4+4], True) for i in xrange(self.FrameCount)]
FrameOffsets = [AnimationOffset + AscDec(data[0x06a8+i*4:0x06a8+i*4+4], True) for i in range(self.FrameCount)]
#Read the audio header:
self.SFXUsage = [(i&0x1<>0, i&0x2<>0, i&0x4<>0) for i in map(ord, data[AudioOffset:AudioOffset+self.FrameCount])]#SFXUsage[frame] = (sfx1, sfx2, sfx2) shere sfxX is either 0 or 1
self.SFXUsage = [(i&0x1!=0, i&0x2!=0, i&0x4!=0) for i in data[AudioOffset:AudioOffset+self.FrameCount]]#SFXUsage[frame] = (sfx1, sfx2, sfx2) shere sfxX is either 0 or 1
SoundSize =(AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4) :AddPadding(AudioOffset+self.FrameCount, 4)+ 4], True),#BG music
AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4)+ 4:AddPadding(AudioOffset+self.FrameCount, 4)+ 8], True),#SFX1
AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4)+ 8:AddPadding(AudioOffset+self.FrameCount, 4)+12], True),#SFX2
AscDec(data[AddPadding(AudioOffset+self.FrameCount, 4)+12:AddPadding(AudioOffset+self.FrameCount, 4)+16], True))#SFX3
if self.forced_speed == None:
self.Framespeed = 8 - ord(data[AddPadding(AudioOffset+self.FrameCount, 4) + 16])
self.Framespeed = 8 - data[AddPadding(AudioOffset+self.FrameCount, 4) + 16]
else:
self.Framespeed = self.forced_speed
self.BGMFramespeed = 8 - ord(data[AddPadding(AudioOffset+self.FrameCount, 4) + 17])#framespeed when the bgm was recorded
self.BGMFramespeed = 8 - data[AddPadding(AudioOffset+self.FrameCount, 4) + 17]#framespeed when the bgm was recorded
## self.BGMFramespeed = self.forced_speed
@@ -195,11 +221,11 @@ class PPM:
self.Frames = []#self.Frames[frame] = [inverted(bool), (color1(0-2), color2(0-2)), frame[layer, x, y] = bool]
for i, offset in enumerate(FrameOffsets):
#Read frame header:
Inverted = ord(data[offset]) & 0x01 == 0
Inverted = data[offset] & 0x01 == 0
#Reads which color that will be used:
Colors = (ord(data[offset]) >> 1 & 0x03,
ord(data[offset]) >> 3 & 0x03)
Colors = (data[offset] >> 1 & 0x03,
data[offset] >> 3 & 0x03)
Frame = self.ExtractFrame(data, offset, self.Frames[i-1][2] if i else None)
@@ -214,7 +240,7 @@ class PPM:
if ReadSound:
self.SoundData = []
pos = AddPadding(AudioOffset+self.FrameCount+32, 4)
for i in xrange(4):
for i in range(4):
self.SoundData.append(data[pos:pos+SoundSize[i]])
pos += SoundSize[i]
@@ -264,7 +290,7 @@ class PPM:
for y in range(8):
for x in range(0,8,2):
#two colors stored in each byte:
byte = ord(self.RawThumbnail[(ty*512+tx*64+y*8+x)/2])
byte = self.RawThumbnail[(ty*512+tx*64+y*8+x)//2]
out[x+tx*8 , y+ty*8] = palette[byte & 0xF]
out[x+tx*8+1, y+ty*8] = palette[byte >> 4]
@@ -275,12 +301,12 @@ class PPM:
if self.SoundData[index]:
#reverse nibbles:
data = []
for i in map(ord,self.SoundData[index]):
data.append(chr((i&0xF)<< 4 | (i>>4)))
data = "".join(data)
for i in self.SoundData[index]:
data.append((i&0xF)<< 4 | (i>>4))
data = bytes(data)
#4bit ADPCM decode
decoded = audioop.adpcm2lin(data, 2, None)[0]
decoded = _decode_ima_adpcm(data)
#write to wav:
if outputpath:
@@ -307,19 +333,19 @@ class PPM:
#Color2Frame = [[0 for i in range(192)] for i in range(256)]
Encoding = [[], []]
#Frame = [[[False for _ in xrange(192)] for _ in xrange(256)] for _ in xrange(2)]
#Frame = [[[False for _ in range(192)] for _ in range(256)] for _ in range(2)]
Frame = np.zeros((2, 256, 192), dtype=np.bool_)
#Read tags:
NewFrame = ord(data[offset]) & 0x80 <> 0
Unknown = ord(data[offset]) >> 5 & 0x03
NewFrame = data[offset] & 0x80 != 0
Unknown = data[offset] >> 5 & 0x03
offset += 1
#WIP - framemove:
FrameMove = [0,0]
if Unknown & 0x2:#doesn't work 100%...
print "FrameMove at offset ",offset-1
print("FrameMove at offset ",offset-1)
move_x = AscDec(data[offset+0:offset+1], True)
move_y = AscDec(data[offset+1:offset+2], True)
@@ -327,12 +353,12 @@ class PPM:
FrameMove[1] = move_y if move_y <= 127 else move_y-256
offset += 2
elif Unknown:
print "Unknown tags:",Unknown,"at offset ",offset-1
print("Unknown tags:",Unknown,"at offset ",offset-1)
#read the line encoding of the layers:
for layer in xrange(2):
for byte in map(ord, data[offset:offset+48]):
for layer in range(2):
for byte in data[offset:offset+48]:
Encoding[layer].append(byte & 0x03)
Encoding[layer].append(byte >> 2 & 0x03)
Encoding[layer].append(byte >> 4 & 0x03)
@@ -340,8 +366,8 @@ class PPM:
offset += 48
#read layers:
for layer in xrange(2):
for y in xrange(192):
for layer in range(2):
for y in range(192):
if Encoding[layer][y] == 0:#Nothing
pass
elif Encoding[layer][y] == 1:#Normal
@@ -350,9 +376,9 @@ class PPM:
x = 0
while UseByte & 0xFFFFFFFF:
if UseByte & 0x80000000:
byte = ord(data[offset])
byte = data[offset]
offset += 1
for _ in xrange(8):
for _ in range(8):
if byte & 0x01:
Frame[layer][x][y] = True
x += 1
@@ -366,7 +392,7 @@ class PPM:
x = 0
while UseByte&0xFFFFFFFF:
if UseByte & 0x80000000:
byte = ord(data[offset])
byte = data[offset]
offset += 1
for _ in range(8):
if not byte & 0x01:
@@ -382,7 +408,7 @@ class PPM:
elif Encoding[layer][y] == 3:#Raw/full
x = 0
for _ in range(32):
byte = ord(data[offset])
byte = data[offset]
offset += 1
for _ in range(8):
if byte & 0x01:
@@ -391,7 +417,7 @@ class PPM:
byte >>= 1
#Merges this frame with the previous frame if NewFrame isn't true:
if not NewFrame and PrevFrame.all() <> None:#maybe optimize this better for numpy...
if not NewFrame and PrevFrame is not None:#maybe optimize this better for numpy...
if FrameMove[0] or FrameMove[1]:#Moves the previous frame if specified:
NewPrevFrame = np.zeros((2, 256, 192), dtype=np.bool_)
@@ -406,7 +432,7 @@ class PPM:
PrevFrame = NewPrevFrame
#merge the frames:
Frame = Frame <> PrevFrame
Frame = Frame != PrevFrame
return Frame
@@ -422,9 +448,9 @@ class PPM:
# - instance.FrameCount: The number of frames this flipnote consists of
# - instance.Locked: A boolean telling us wether it's locked or not
# - instance.ThumbnailFrameIndex: Which frame the thumbnail is(starting at 0)
# - instance.OriginalAuthorName: The name of the original author in unicode
# - instance.EditorAuthorName: The name of the (previous?) editing author in unicode
# - instance.Username: The username in unicode
# - instance.OriginalAuthorName: The name of the original author in str
# - instance.EditorAuthorName: The name of the (previous?) editing author in str
# - instance.Username: The username in str
# - instance.OriginalAuthorID: The ID of the original author
# - instance.EditorAuthorID: The ID of the last editor, the last user to save the file
# - instance.PreviousEditAuthorID: The ID of the previous editor
@@ -443,7 +469,7 @@ class TMB:
f.close()
return ret
def Read(self, data, DecodeThumbnail=False):
if data[:4] <> "PARA" or len(data) < 0x6a0:
if data[:4] != b"PARA" or len(data) < 0x6a0:
return False
#Read the header:
@@ -451,25 +477,25 @@ class TMB:
self.AudioLenght = AscDec(data[8:12], True)#only stored for self.Pack()
self.FrameCount = AscDec(data[12:14], True) + 1
self.Locked = ord(data[0x10]) & 0x01 == 1
self.Locked = data[0x10] & 0x01 == 1
self.ThumbnailFrameIndex = AscDec(data[0x12:0x14], True)#which frame is in the thumbnnail
#self.OriginalAuthorName = u"".join(unichr(AscDec(data[0x14+i*2:0x14+i*2+2], True)) for i in xrange(11)).split(u"\0")[0]
#self.EditorAuthorName = u"".join(unichr(AscDec(data[0x2A+i*2:0x2A+i*2+2], True)) for i in xrange(11)).split(u"\0")[0]
#self.Username = u"".join(unichr(AscDec(data[0x40+i*2:0x40+i*2+2], True)) for i in xrange(11)).split(u"\0")[0]
#self.OriginalAuthorName = u"".join(chr(AscDec(data[0x14+i*2:0x14+i*2+2], True)) for i in range(11)).split(u"\0")[0]
#self.EditorAuthorName = u"".join(chr(AscDec(data[0x2A+i*2:0x2A+i*2+2], True)) for i in range(11)).split(u"\0")[0]
#self.Username = u"".join(chr(AscDec(data[0x40+i*2:0x40+i*2+2], True)) for i in range(11)).split(u"\0")[0]
self.OriginalAuthorName = data[0x14:0x2A].decode("UTF-16LE").split(u"\0")[0]
self.EditorAuthorName = data[0x2A:0x40].decode("UTF-16LE").split(u"\0")[0]
self.Username = data[0x40:0x56].decode("UTF-16LE").split(u"\0")[0]
self.OriginalAuthorID = data[0x56:0x5e][::-1].encode("HEX").upper()
self.EditorAuthorID = data[0x5E:0x66][::-1].encode("HEX").upper()#the last user to save the file
self.OriginalAuthorID = data[0x56:0x5e][::-1].hex().upper()
self.EditorAuthorID = data[0x5E:0x66][::-1].hex().upper()#the last user to save the file
self.OriginalFilenameC = data[0x66:0x78]#compressed
self.CurrentFilenameC = data[0x78:0x8a]#compressed
self.OriginalFilename = "%s_%s_%s.tmb" % (self.OriginalFilenameC[:3].encode("HEX").upper(), self.OriginalFilenameC[3:-2], str(AscDec(self.OriginalFilenameC[-2:], True)).zfill(3))
self.CurrentFilename = "%s_%s_%s.tmb" % (self.CurrentFilenameC[:3].encode("HEX").upper(), self.CurrentFilenameC[3:-2], str(AscDec(self.CurrentFilenameC[-2:], True)).zfill(3))
self.OriginalFilename = "%s_%s_%s.tmb" % (self.OriginalFilenameC[:3].hex().upper(), self.OriginalFilenameC[3:-2].decode("ascii", "replace"), str(AscDec(self.OriginalFilenameC[-2:], True)).zfill(3))
self.CurrentFilename = "%s_%s_%s.tmb" % (self.CurrentFilenameC[:3].hex().upper(), self.CurrentFilenameC[3:-2].decode("ascii", "replace"), str(AscDec(self.CurrentFilenameC[-2:], True)).zfill(3))
self.PreviousEditAuthorID = data[0x8a:0x92][::-1].encode("HEX").upper()#don't know what this really is
self.PreviousEditAuthorID = data[0x8a:0x92][::-1].hex().upper()#don't know what this really is
self.PartialFilenameC = data[0x92:0x9a]#compressed
@@ -497,27 +523,27 @@ class TMB:
#realself = self
#if ppm: self = ppm
out = ["PARA",#magic
out = [b"PARA",#magic
DecAsc(self.AudioOffset-0x6a0, 4, True),#animation data size
DecAsc(self.AudioLenght, 4, True),#audio data size
DecAsc(self.FrameCount-1, 2, True),#frame count
"\x24\x00",#unknown
chr(self.Locked), "\0",#locked
b"\x24\x00",#unknown
bytes((int(bool(self.Locked)),)), b"\0",#locked
DecAsc(self.ThumbnailFrameIndex, 2, True),#which frame is in the thumbnnail
self.OriginalAuthorName.encode("UTF-16LE") + "\0\0"*(11-len(self.OriginalAuthorName)),#Original Author Name
self.EditorAuthorName.encode("UTF-16LE") + "\0\0"*(11-len(self.EditorAuthorName)),#Editor Author Name
self.Username.encode("UTF-16LE") + "\0\0"*(11-len(self.Username)),#Username
self.OriginalAuthorID.decode("HEX")[::-1],#OriginalAuthorID
self.EditorAuthorID.decode("HEX")[::-1],#EditorAuthorID
self.OriginalAuthorName.encode("UTF-16LE") + b"\0\0"*(11-len(self.OriginalAuthorName)),#Original Author Name
self.EditorAuthorName.encode("UTF-16LE") + b"\0\0"*(11-len(self.EditorAuthorName)),#Editor Author Name
self.Username.encode("UTF-16LE") + b"\0\0"*(11-len(self.Username)),#Username
bytes.fromhex(self.OriginalAuthorID)[::-1],#OriginalAuthorID
bytes.fromhex(self.EditorAuthorID)[::-1],#EditorAuthorID
self.OriginalFilenameC,#OriginalFilename
self.CurrentFilenameC,#CurrentFilename
self.PreviousEditAuthorID.decode("HEX")[::-1],#EditorAuthorID
bytes.fromhex(self.PreviousEditAuthorID)[::-1],#EditorAuthorID
self.PartialFilenameC,#PartialFilename
DecAsc(self.Date, 4, True),#Date in seconds
"\0\0",#padding
b"\0\0",#padding
self.PackThumbnail()]#thumbnail
return "".join(out)
return b"".join(out)
def GetThumbnail(self, force=False):
if (self.Thumbnail is None or force):# and self.RawThumbnail:
global ThumbPalette
@@ -535,7 +561,7 @@ class TMB:
for y in range(8):
for x in range(0,8,2):
#two colors stored in each byte:
byte = ord(self.RawThumbnail[(ty*512+tx*64+y*8+x)/2])
byte = self.RawThumbnail[(ty*512+tx*64+y*8+x)//2]
out[x+tx*8 , y+ty*8] = palette[byte & 0xF]
out[x+tx*8+1, y+ty*8] = palette[byte >> 4]
@@ -560,7 +586,7 @@ class TMB:
0x00FF00FF)#F-
if not self.Thumbnail:
if self.Thumbnail is None:
return self.RawThumbnail
else:
if Exact:
@@ -572,12 +598,12 @@ class TMB:
for y in range(8):
for x in range(0,8,2):
#two colors stored in each byte:
#pos = 0xa0+(ty*512+tx*64+y*8+x)/2
#pos = 0xa0+(ty*512+tx*64+y*8+x)//2
p1 = palette.index(int(self.Thumbnail[x+tx*8 , y+ty*8]))
p2 = palette.index(int(self.Thumbnail[x+tx*8+1, y+ty*8]))
out.append(chr(p2<<4 | p1))
out.append(p2<<4 | p1)
self.RawThumbnail = "".join(out)
self.RawThumbnail = bytes(out)
return self.RawThumbnail
else:
#not yet implented
@@ -591,7 +617,7 @@ class TMB:
# This function requires the PIl imaging module
def WriteImage(image, outputPath, scale=1):
if not hasPIL:
print "Error: PIL not found!"
print("Error: PIL not found!")
return False
#if not image: return False
@@ -603,7 +629,7 @@ def WriteImage(image, outputPath, scale=1):
else:
newImg = image
out = newImg.tostring("F")
out = newImg.tobytes(order="F")
out = Image.frombytes("RGBA", (len(newImg), len(newImg[0])), out)
filetype = outputPath[outputPath.rfind(".")+1:]
@@ -611,7 +637,9 @@ def WriteImage(image, outputPath, scale=1):
return True
def get_metadata(flipnote):
def get_metadata(flipnote, filetype=None):
if filetype is None:
filetype = "ppm" if isinstance(flipnote, PPM) else "tmb"
epoch = time.mktime(time.struct_time([2000, 1, 1, 0, 0, 0, 5, 1, -1]))
meta = {
@@ -636,8 +664,8 @@ def get_metadata(flipnote):
return meta
def DumpFrames(flipnote,directory,scale=1):
for i in xrange(flipnote.FrameCount):
print "Dumping frame #%i of %i..." % (i+1, flipnote.FrameCount)
for i in range(flipnote.FrameCount):
print("Dumping frame #%i of %i..." % (i+1, flipnote.FrameCount))
WriteImage(flipnote.GetFrame(i), os.path.join(directory, "frame %s.png" % str(i+1).zfill(3)), scale)
def DumpSoundFiles(flipnote,directory,raw=False):
@@ -657,161 +685,159 @@ def DumpSFXUsage(flipnote,directory):
f.write("Frame %i:%s%s%s\n" % (i, " SFX1"*s1, " SFX2"*s2, " SFX3"*s3))
if __name__ == '__main__':
print " == PPM.py =="
print " == by pbsds =="
print " == v1.3 =="
print
print(" == PPM.py ==")
print(" == by pbsds ==")
print(" == v1.3 ==")
print()
if len(sys.argv) < 3:
print "Usage:"
print " PPM.py <Mode> <Input> [<Output>] [<Frame>] [<Option>]"
print ""
print " <Mode>:"
print " -t: Extracts the thumbnail to the file <Output>"
print " -f: Extracts the frame(s) to <Output>"
print " -s: Dumps the sound files to the folder <Output>"
print " -S: Same as mode -s, but will also dump the raw sound data files"
print " -e: Exports the flipnote to an MKV with minimal transcoding"
print " -E: Same as -e, but will encode the video as lossless H.265"
print " -m: Prints out the metadata. Can also write it to <output> which also"
print " supports unicode charactes."
print " -oa: Seach a directory for an original author that matches the RegEx"
print " Mode -t and -m supports TMB files aswell"
print " <Frame>"
print " Only used in mode -f"
print " Set this to the exact frame you want to extract(starting at 1) and it"
print " will be saved as a file to <Output>."
print " If not specified, it will extract all frames to the folder <Output>"
print " <Option>"
print " --speed N: Only used in mode -e. Set this to force a specific"
print " flipnote speed (1 to 8)."
print " --scale N: Only used in modes -e and -f. Set this to upscale the"
print " frames N times."
print("Usage:")
print(" PPM.py <Mode> <Input> [<Output>] [<Frame>] [<Option>]")
print("")
print(" <Mode>:")
print(" -t: Extracts the thumbnail to the file <Output>")
print(" -f: Extracts the frame(s) to <Output>")
print(" -s: Dumps the sound files to the folder <Output>")
print(" -S: Same as mode -s, but will also dump the raw sound data files")
print(" -e: Exports the flipnote to an MKV with minimal transcoding")
print(" -E: Same as -e, but will encode the video as lossless H.265")
print(" -m: Prints out the metadata. Can also write it to <output> which also")
print(" supports str charactes.")
print(" -oa: Seach a directory for an original author that matches the RegEx")
print(" Mode -t and -m supports TMB files aswell")
print(" <Frame>")
print(" Only used in mode -f")
print(" Set this to the exact frame you want to extract(starting at 1) and it")
print(" will be saved as a file to <Output>.")
print(" If not specified, it will extract all frames to the folder <Output>")
print(" <Option>")
print(" --speed N: Only used in mode -e. Set this to force a specific")
print(" flipnote speed (1 to 8).")
print(" --scale N: Only used in modes -e and -f. Set this to upscale the")
print(" frames N times.")
sys.exit()
import os, time
if sys.argv[1] == "-t":
print "Reading the flipnote file...",
print("Reading the flipnote file...", end=" ")
if not os.path.isfile(sys.argv[2]):
print "Error!\nSpecified file doesn't exist!"
print("Error!\nSpecified file doesn't exist!")
sys.exit()
flipnote = TMB().ReadFile(sys.argv[2], True)
if not flipnote:
print "Error!"
print "The given file is not a Flipnote PPM file or TMB file!"
print("Error!")
print("The given file is not a Flipnote PPM file or TMB file!")
sys.exit()
print "Done!"
print("Done!")
print "Dumping the thumbnail...",
print("Dumping the thumbnail...", end=" ")
WriteImage(flipnote.GetThumbnail(), sys.argv[3])
print "Done!"
print("Done!")
elif sys.argv[1] == "-f":
if len(sys.argv) < 4:
print "Error!"
print "<Output> not specified!"
print("Error!")
print("<Output> not specified!")
sys.exit()
try:
scale = int(sys.argv[sys.argv.index("--scale")+1])
print "Using frame scaling "+str(scale)
print("Using frame scaling "+str(scale))
except (IndexError,ValueError):
scale = 1
print "Reading the flipnote file...",
print("Reading the flipnote file...", end=" ")
if not os.path.isfile(sys.argv[2]):
print "Error!\nSpecified file doesn't exist!"
print("Error!\nSpecified file doesn't exist!")
sys.exit()
flipnote = PPM().ReadFile(sys.argv[2])
if not flipnote:
print "Error!\nThe given file is not a Flipnote PPM file."
print("Error!\nThe given file is not a Flipnote PPM file.")
sys.exit()
print "Done!"
print("Done!")
if len(sys.argv) < 5:
if not os.path.isdir(sys.argv[3]):
print "Error!\nThe specified directory doesn't exist!"
print("Error!\nThe specified directory doesn't exist!")
sys.exit()
DumpFrames(flipnote,sys.argv[3], scale)
print "Done!"
print("Done!")
else:
try:
int(sys.argv[4])
except:
print "Error!\nInvalid <Frame>!"
print("Error!\nInvalid <Frame>!")
sys.exit()
if not (0 <= int(sys.argv[4])-1 < flipnote.FrameCount):
print "Error!\n<Frame> is out of bounds!"
print("Error!\n<Frame> is out of bounds!")
sys.exit()
print "Dumping frame #%i..." % int(sys.argv[4]),
print("Dumping frame #%i..." % int(sys.argv[4]), end=" ")
WriteImage(flipnote.GetFrame(int(sys.argv[4])-1), sys.argv[3], scale)
print "Done!"
print("Done!")
elif sys.argv[1] in ("-s", "-S"):
if len(sys.argv) < 4:
print "Error!"
print "<Output> not specified!"
print("Error!")
print("<Output> not specified!")
sys.exit()
print "Reading the flipnote file...",
print("Reading the flipnote file...", end=" ")
if not os.path.isfile(sys.argv[2]):
print "Error!\nSpecified file doesn't exist!"
print("Error!\nSpecified file doesn't exist!")
sys.exit()
flipnote = PPM().ReadFile(sys.argv[2], ReadFrames=False, ReadSound=True)
if not flipnote:
print "Error!\nThe given file is not a Flipnote PPM file."
print("Error!\nThe given file is not a Flipnote PPM file.")
sys.exit()
print "Done!"
print("Done!")
if not os.path.isdir(sys.argv[3]):
print "Error!\nThe specified output directory doesn't exist!"
print("Error!\nThe specified output directory doesn't exist!")
sys.exit()
print "Converting the sound files...",
print("Converting the sound files...", end=" ")
DumpSoundFiles(flipnote,sys.argv[3],raw=(sys.argv[1]=="-S"))
print "Done!"
print("Done!")
print "Dumping the sound effect usage...",
print("Dumping the sound effect usage...", end=" ")
DumpSFXUsage(flipnote,sys.argv[3])
print "Done!"
print("Done!")
elif sys.argv[1] == "-m":
if not os.path.isfile(sys.argv[2]):
print "Error!\nSpecified file doesn't exist!"
print("Error!\nSpecified file doesn't exist!")
sys.exit()
filetype = "ppm" if sys.argv[2][-3:] == "ppm" else "tmb"
flipnote = TMB().ReadFile(sys.argv[2]) if filetype == "tmb" else PPM().ReadFile(sys.argv[2], ReadFrames=False)
if not flipnote:
print "Error!\nThe given file is not a Flipnote PPM file or TMB file."
print("Error!\nThe given file is not a Flipnote PPM file or TMB file.")
if len(sys.argv) >= 4:
f = open(sys.argv, "wb")
f.write("Error!\nThe given file is not a Flipnote PPM file or TMB file.")
f.close()
with open(sys.argv[3], "w", encoding="UTF-8") as f:
f.write("Error!\nThe given file is not a Flipnote PPM file or TMB file.\n")
sys.exit()
meta = get_metadata(flipnote)
meta = get_metadata(flipnote, filetype)
newline = "\n"
if sys.platform in ("win32", "cygwin"): newline = "\r\n"
elif sys.platform in ("darwin"): newline = "\r"
print newline.join(["\t".join([unicode(char) for char in line]) for line in meta.items()]).encode('ascii', 'ignore')
metadata_text = newline.join(f"{key}\t{value}" for key, value in meta.items())
print(metadata_text)
if len(sys.argv) >= 4:
f = open(sys.argv[3], "wb")
f.write(newline.join(meta).encode("UTF-8"))
f.close()
with open(sys.argv[3], "w", encoding="UTF-8", newline="") as f:
f.write(metadata_text + newline)
elif sys.argv[1] == "-oa":
regex = re.compile(sys.argv[2])
os.chdir(sys.argv[3])
for filename in os.listdir("."):
epoch = time.mktime(time.struct_time([2000, 1, 1, 0, 0, 0, 5, 1, -1]))
if not os.path.isfile(filename):
print "Error!\nSpecified file doesn't exist!"
print("Error!\nSpecified file doesn't exist!")
sys.exit()
filetype = "ppm" if filename[-3:].lower() == "ppm" else "tmb"
@@ -821,11 +847,11 @@ if __name__ == '__main__':
meta = get_metadata(flipnote)
if regex.match(meta["Original author"]):
print filename
print(filename)
elif sys.argv[1] == "-e" or sys.argv[1] == "-E":
elif sys.argv[1] in ("-e", "-E"):
if not hasffmpeg:
print "Error!\nffmpeg is not installed."
print("Error!\nffmpeg is not installed.")
sys.exit()
in_file = sys.argv[2]
out_file = sys.argv[3]
@@ -833,30 +859,30 @@ if __name__ == '__main__':
sleep_time = int(sys.argv[sys.argv.index("--sleep")+1])
except (IndexError,ValueError):
sleep_time = 0
print "Not sleeping."
print("Not sleeping.")
try:
forced_speed = int(sys.argv[sys.argv.index("--speed")+1])
print "Using forced speed "+str(forced_speed)
print("Using forced speed "+str(forced_speed))
except (IndexError,ValueError):
forced_speed = None
try:
scale = int(sys.argv[sys.argv.index("--scale")+1])
print "Using frame scaling "+str(scale)
print("Using frame scaling "+str(scale))
except (IndexError,ValueError):
scale = 1
if out_file.lower()[-4:] != ".mkv":
out_file += ".mkv"
if not os.path.isfile(in_file):
print "Error!\nSpecified file doesn't exist!"
print("Error!\nSpecified file doesn't exist!")
sys.exit()
if os.path.isfile(out_file):
print "Overwrite existing file?"
print("Overwrite existing file?")
overwrite = ""
while overwrite != "y" and overwrite != "n":
overwrite = raw_input("(Y/N) ").lower()
overwrite = input("(Y/N) ").lower()
if overwrite == "n":
print "Not overwriting; exiting."
print("Not overwriting; exiting.")
sys.exit()
filetype = "ppm" if in_file[-3:].lower() == "ppm" else "tmb"
flipnote = TMB().ReadFile(in_file) if filetype == "tmb" else PPM(forced_speed).ReadFile(in_file, ReadFrames=True, ReadSound=True)
@@ -864,63 +890,62 @@ if __name__ == '__main__':
# Make temp dir and dump the frames and sound here
tempdir = tempfile.mkdtemp()
os.mkdir(tempdir+"/sounds")
print "Dumping the frames..."
print("Dumping the frames...")
DumpFrames(flipnote,tempdir,scale)
print "Done!"
print "Dumping the sounds..."
print("Done!")
print("Dumping the sounds...")
DumpSoundFiles(flipnote,tempdir+"/sounds")
print "Done!"
print "Dumping SFX usage..."
print("Done!")
print("Dumping SFX usage...")
DumpSFXUsage(flipnote,tempdir+"/sounds")
print "Done!"
print("Done!")
# Now we need the metadata so we can look up the FPS
SPEEDS = [None,0.5,1,2,4,6,12,20,30]
print "Getting metadata..."
print("Getting metadata...")
metadata = get_metadata(flipnote)
print "Done!"
print("Done!")
speed = int(metadata["Frame speed"])
fps = SPEEDS[speed]
duration = float(metadata["Number of frames"])/float(fps)
print "Flipnote is speed {speed}, so {fps} FPS for {dur} seconds".format(speed=speed,fps=fps,dur=duration)
print("Flipnote is speed {speed}, so {fps} FPS for {dur} seconds".format(speed=speed,fps=fps,dur=duration))
# Now to make the video in ffmpeg
print "Exporting video with ffmpeg..."
print("Exporting video with ffmpeg...")
export_command = ["ffmpeg","-framerate",str(fps),"-start_number","1","-i","{path}/frame %03d.png".format(path=tempdir),"-i","{path}/sounds/BGM.wav".format(path=tempdir),"-c:v","copy","-c:a","copy","-t","{dur}".format(dur=duration),"-y",out_file]
if not os.path.isfile(tempdir+"/sounds/BGM.wav"):
print "No background music. Adding silent track..."
print("No background music. Adding silent track...")
#has_bgm = False
export_command = ["ffmpeg","-framerate",str(fps),"-start_number","1","-i","{path}/frame %03d.png".format(path=tempdir),"-f","lavfi","-i","anullsrc=r=8192:cl=mono","-c:v","copy","-c:a","pcm_s16le","-shortest","-y",out_file]
else:
#has_bgm = True
pass
# use libx265 in lossless mode, rather than MPNG
# Latest master also supports a lossless H.265 export mode.
if sys.argv[1] == "-E":
# insert the necessary arguments into the existing command
codec_index = export_command.index("-c:v")
export_command[codec_index+1] = "libx265"
export_command.insert(codec_index+2, "-x265-params")
export_command.insert(codec_index+3, "lossless=1")
export_command.insert(codec_index+2, "-preset")
export_command.insert(codec_index+3, "veryslow")
print " ".join(export_command)
with open(os.devnull) as null:
export_command[codec_index + 1] = "libx265"
export_command[codec_index + 2:codec_index + 2] = [
"-preset", "veryslow", "-x265-params", "lossless=1"
]
print(" ".join(export_command))
with open(os.devnull, "w") as null:
subprocess.call(export_command,stdout=null,stderr=null)
print "Done!"
print("Done!")
# If the audio has been sped up, we have to do it again manually
bgm_speed = int(metadata["BGM Frame speed"])
if bgm_speed != speed:
print "Background music speed must be modified!"
print("Background music speed must be modified!")
original_rate = 8192
newrate = 8192*(float(fps)/SPEEDS[bgm_speed])
print "Using new rate: "+str(newrate)
print("Using new rate: "+str(newrate))
speed_change_command = ["ffmpeg","-i","{path}/sounds/BGM.wav".format(path=tempdir),"-filter_complex","asetrate={rate}".format(rate=newrate),"-i",out_file,"-map","0:a","-map","1:v","-vcodec","copy","-acodec","pcm_s16le","{path}/temp_out.mkv".format(path=tempdir)]
with open(os.devnull,"w") as null:
subprocess.call(speed_change_command,stdout=null,stderr=null)
os.remove(out_file)
shutil.move("{path}/temp_out.mkv".format(path=tempdir),out_file)
print "Done!"
print("Done!")
# These are the ffmpeg commands I need for each sound effect
# The first generates a silent track, with variable length. This gets concatenated to the front of the sound effect.
@@ -938,10 +963,10 @@ if __name__ == '__main__':
## else:
# Read in the sound effect usage data
print "Reading sound effect usage..."
print("Reading sound effect usage...")
with open("{path}/sounds/SFX usage.txt".format(path=tempdir),"r") as sfx_usage_file:
sfx_usage = sfx_usage_file.read().split("\n")
print "Done!"
print("Done!")
# Iterate through the frames, checking if sound effects need to be added
for frame in range(len(sfx_usage)):
@@ -950,7 +975,7 @@ if __name__ == '__main__':
sfx = line.split(":")[1].strip() if line.strip() != "" else ""
if sfx != "": # If a sound effect must be played...
length = frame/float(fps)
print "Adding "+sfx+" at {length} seconds into the video.".format(length=length)
print("Adding "+sfx+" at {length} seconds into the video.".format(length=length))
# ...run each command in series with the correct arguments
with open(os.devnull,"w") as null:
subprocess.call([i.format(path=tempdir,sfx=sfx,length=length) for i in silence_command],stdout=null,stderr=null)
@@ -958,7 +983,7 @@ if __name__ == '__main__':
subprocess.call([i.format(path=tempdir,video_length=fps*len(sfx_usage)) for i in merge_command],stdout=null,stderr=null)
os.remove(out_file)
shutil.move("{path}/temp_out.mkv".format(path=tempdir),out_file)
print "Done!"
print("Done!")
time.sleep(sleep_time) # optional sleep -- in case you want to slow things down for HDD strain or reliability or something
## subprocess.call(normalise_command)
@@ -967,9 +992,9 @@ if __name__ == '__main__':
# Remove the temp dir and all files in it
print "Removing temporary directory..."
print("Removing temporary directory...")
shutil.rmtree(tempdir)
print "Done!"
print("Done!")
else:
print "Error!\nThere's no such mode."
print("Error!\nThere's no such mode.")