Rewrote Downloader (It No Longer Downloads Duplicates)

This commit is contained in:
Glitched Panda
2025-01-06 15:37:39 +00:00
parent f66b698f0d
commit e3961bf1bf
3 changed files with 189 additions and 216 deletions

View File

@@ -1,16 +1,35 @@
# DLC-Downloader # DLC-Downloader
DLC Downloader for The Simpsons: Tapped Out. Thank you to [schdub](https://github.com/schdub/dlcsync) for (most) of the code. I decided to modify his, as i want to get this done before Jan. 24 when the game shuts down. DLC Downloader for The Simpsons: Tapped Out. It takes a while to download everything, so be patient.
## Usage ## Usage
You will need about 350GB of free storage, and [python3](https://www.python.org) installed. You will need about 30GB of free storage, and [python3](https://www.python.org) installed.
``` ```
python3 dlcDownloader.py python3 dlcDownloader.py
``` ```
## Configuration ## Configuration
If you want to for example change the language of dlcs the script downloads, you can do so at the top of the script. If you want to for example change the language of dlcs the script downloads, you can do so at the top of the script.
``` ```python
verbose = False # display some debug messages OUT_DIR = "./dlc" # Directory where the dlc files will be downloaded
lang = [ 'all', 'en' ] # en,fr,it,de,es,ko,zh,cn,pt,ru,tc,da,sv,no,nl,tr,th LANGUAGE = [
tier = [ 'all', '25', '50', '100', 'retina', 'iphone', 'ipad', 'ipad3' ] # 25,50,100,retina,iphone,ipad,ipad3 "all",
"en"
] # en,fr,it,de,es,ko,zh,cn,pt,ru,tc,da,sv,no,nl,tr,th
TIER = [
"all",
"25",
"50",
"100",
"retina",
"iphone",
"ipad",
"ipad3",
"mp3",
"caf",
"wav",
] # 25,50,100,retina,iphone,ipad,ipad3,mp3,caf,wav
ALL_LANGUAGES = True # Download the dlcs in every languages
ALL_TIERS = True # Download the dlcs in every tier
``` ```

View File

@@ -1,66 +0,0 @@
thoh2013-3a3f8d.tstodlc.eamobile.com
thoh2013-4-5-1-9rj43.tstodlc.eamobile.com
xmas2013-4-6-0-3x9flx.tstodlc.eamobile.com
xmas2013-4-6-1-89rrjf.tstodlc.eamobile.com
feb2014-4-7-0-49fxs0.tstodlc.eamobile.com
feb2014-4-7-1-ut90jr.tstodlc.eamobile.com
apr2014-4-8-0-4kf0at.tstodlc.eamobile.com
apr2014-4-8-1-k39fxt.tstodlc.eamobile.com
may2014-4-8-2-2b33bc.tstodlc.eamobile.com
jun2014-4-9-0-3jf0xc.tstodlc.eamobile.com
jun2014-4-9-3-8ca2bb.tstodlc.eamobile.com
jun2014-4-9-5-016374.tstodlc.eamobile.com
aug2014-4-10-0-33r8k6.tstodlc.eamobile.com
aug2014-4-10-2-a6c40f.tstodlc.eamobile.com
oct2014-4-11-0-97pes0.tstodlc.eamobile.com
oct2014-4-11-1-5rofxt.tstodlc.eamobile.com
oct2014-4-11-5-393fjf.tstodlc.eamobile.com
xmas2014-4-12-0-q58ubc.tstodlc.eamobile.com
xmas2014td-4-12-5-uhtg34.tstodlc.eamobile.com
superheroes-4-13-0-nv98q4.tstodlc.eamobile.com
superheroes-takedown-4-13-5-93faz8.tstodlc.eamobile.com
apr2015-4-14-0-39fajo9a.tstodlc.eamobile.com
apr2015-4-14-5-48fuecxa.tstodlc.eamobile.com
jun2015-4-15-0-jg84gs3f.tstodlc.eamobile.com
jun2015-4-15-5-nvoiw8efj.tstodlc.eamobile.com
aug2015-4-16-0-xzi394jf.tstodlc.eamobile.com
aug2015-4-16-5-bv0e3fia.tstodlc.eamobile.com
oct2015-4-17-0-fj39afaed.tstodlc.eamobile.com
oct2015-4-17-5-bnc9a93.tstodlc.eamobile.com
dec2015-4-18-0-ghe83hf.tstodlc.eamobile.com
dec2015-4-18-5-bid9234.tstodlc.eamobile.com
feb2016-4-19-0-i02ifj.tstodlc.eamobile.com
feb2016-4-19-5-b83ifoa.tstodlc.eamobile.com
apr2016-4-20-0-38faho8a.tstodlc.eamobile.com
apr2016-4-20-5-w03ifjal.tstodlc.eamobile.com
jun2016-4-21-0-tlj3zug8.tstodlc.eamobile.com
july2016-4-21-5-5qs2b3jh.tstodlc.eamobile.com
august2016-4-22-0-4urau4h8.tstodlc.eamobile.com
september2016-4-22-5-st0pny01.tstodlc.eamobile.com
oct2016-4-23-0-vrj28z.tstodlc.eamobile.com
nov2016-4-23-5-l3osc5.tstodlc.eamobile.com
dec2016-4-24-0-jccburm.tstodlc.eamobile.com
jan2017-4-24-5-5j6kds4q.tstodlc.eamobile.com
jan2017-4-25-0-ztk6mia7.tstodlc.eamobile.com
march2017-4-25-5-n57dpzu3.tstodlc.eamobile.com
march2017-4-25-6-g491pnkr.tstodlc.eamobile.com
may2017-4-26-5-t7w59cqc.tstodlc.eamobile.com
mar2017-4-26-0-s4hur7t7.tstodlc.eamobile.com
jun2017-4-27-0-8ep9eykc.tstodlc.eamobile.com
jul2017-4-27-5-ikp7ncq5.tstodlc.eamobile.com
aug2017-4-28-0-5euxc0v8.tstodlc.eamobile.com
aug2017-4-28-5-uzki2ogt.tstodlc.eamobile.com
oct2017-4-29-0-2k8p9riy.tstodlc.eamobile.com
nov2017-4-29-5-giai591j.tstodlc.eamobile.com
dec2017-4-30-0-ph621tki.tstodlc.eamobile.com
jan2018-4-31-0-zp9u3cog.tstodlc.eamobile.com
mar2018-4-31-5-3dpu6dnx.tstodlc.eamobile.com
mar2018-4-32-0-xk2u4fm5.tstodlc.eamobile.com
mar2018-4-32-1-ie5c5aq9.tstodlc.eamobile.com
may2018-4-32-5-9nz9wk4w.tstodlc.eamobile.com
may2018-4-33-0-ibr0qhd8.tstodlc.eamobile.com
jun2018-4-33-1-s7utav9k.tstodlc.eamobile.com
july2018-4-33-5-ffd6uju9.tstodlc.eamobile.com
july2018-4-34-0-xpb2jvat.tstodlc.eamobile.com
aug2018-4-34-5-a98qp6kzcu.tstodlc.eamobile.com
oct2018-4-35-0-uam5h44a.tstodlc.eamobile.com

View File

@@ -1,163 +1,183 @@
#!/usr/bin/python3
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
import binascii
import urllib.request
import zipfile
import io
import os
verbose = False # display some debug messages from requests import get
lang = [ 'all', 'en' ] # en,fr,it,de,es,ko,zh,cn,pt,ru,tc,da,sv,no,nl,tr,th
tier = [ 'all', '25', '50', '100', 'retina', 'iphone', 'ipad', 'ipad3' ] # 25,50,100,retina,iphone,ipad,ipad3
LOCAL_DLC_DIR = './dlcs-'+ lang[1] + '/' # directory where DLC will be loaded
URL_DLC_BASE = 'http://oct2018-4-35-0-uam5h44a.tstodlc.eamobile.com/netstorage/gameasset/direct/simpsons/'
CURRENT_LINE = '' from colorama import Fore
def LOG(msg): from time import sleep
if verbose: print(msg)
def crc32ForFile(filename): import os, io, zipfile
with open(filename,'rb') as f:
buf = (binascii.crc32(f.read()) & 0xFFFFFFFF)
return "%d" % buf
def getZippedXml(url): OUT_DIR = "./dlc" # Directory where the dlc files will be downloaded
r = urllib.request.urlopen(url) LANGUAGE = [
data = r.read() "all",
r.close() "en"
LOG("downloaded %d bytes" % len(data)) ] # en,fr,it,de,es,ko,zh,cn,pt,ru,tc,da,sv,no,nl,tr,th
with open("./dlcs-" + lang[1] + "/" + CURRENT_LINE + "/DLCIndex.zip", "ab+") as file: TIER = [
file.write(data) "all",
file.close() "25",
"50",
"100",
"retina",
"iphone",
"ipad",
"ipad3",
"mp3",
"caf",
"wav",
] # 25,50,100,retina,iphone,ipad,ipad3,mp3,caf,wav
with zipfile.ZipFile(io.BytesIO(data)) as z: ALL_LANGUAGES = True # Download the dlcs in every languages
data = z.read(z.infolist()[0]) ALL_TIERS = True # Download the dlcs in every tier
LOG("unzipped %d bytes" % len(data))
return data
def getDlcIndex(): BASE_URL = "http://oct2018-4-35-0-uam5h44a.tstodlc.eamobile.com/netstorage/gameasset/direct/simpsons/"
tree = ET.fromstring(getZippedXml(URL_DLC_BASE + 'dlc/DLCIndex.zip'))
lst = tree.findall('./IndexFile')
return lst[0].get('index').replace(':', '/')
def getRest(dlFile, fromUrl): DOWNLOAD_QUEUE = [] # [ Url, Filename, Folder ]
existSize = 0 DOWNLOADED = []
req = urllib.request.Request(fromUrl)
if os.path.exists(dlFile):
outputFile = open(dlFile, "ab")
existSize = os.path.getsize(dlFile)
# if the file exists, then download only the remainder
req.headers['Range'] = 'bytes=%s-' % (existSize)
else:
outputFile = open(dlFile,"wb")
webPage = urllib.request.urlopen(req)
if verbose:
for k, v in webPage.headers.items():
LOG("%s=%s" % (k, v))
# if we already have the whole file, there is no need to download it again
ok = False
numBytes = 0
webSize = int(webPage.headers['Content-Length'])
if webSize == existSize:
LOG("File (%s) was already downloaded from URL (%s)" % (dlFile, fromUrl))
ok = True
else:
#LOG("Downloading %d more bytes" % (webSize-existSize))
while 1:
data = webPage.read(8192)
if not data: break
outputFile.write(data)
numBytes = numBytes + len(data)
ok = numBytes == webSize
LOG("downloaded %d bytes from %d" % (numBytes, webSize))
webPage.close()
outputFile.close()
return ok
def doDownload(fn):
print('./dlcs-' + lang[1] + '/' + CURRENT_LINE + '/' + fn)
tempFileName = fn.replace('/', '#') def log(severity: int, message: str):
bytesCount = getRest('./dlcs-' + lang[1] + '/'+ CURRENT_LINE + '/' + tempFileName, URL_DLC_BASE + fn) if severity == 0:
print(Fore.BLUE + "[i] " + Fore.WHITE + message)
elif severity == 1:
print(Fore.YELLOW + "[!] " + Fore.WHITE + message)
elif severity == 2:
print(Fore.RED + "[!] " + Fore.WHITE + message)
else:
print(Fore.WHITE + message)
class DlcIndexParser:
ignorePackage = True
# called for each opening tag.
def start(self, tag, attrib):
#LOG("tag='%s' attrib='%s'" % (tag, attrib))
if (tag == 'Package'):
# initilize variables for each Package
self.ignorePackage = False
self.LocalDir = ''
self.FileSize = ''
self.UncompressedFileSize = ''
self.IndexFileCRC = ''
self.IndexFileSig = ''
self.Version = ''
self.FileName = ''
self.Language = ''
if attrib['ignore'] == 'true' or attrib['tier'] not in tier: def downloadFile(url: str, filename: str):
self.ignorePackage = True os.makedirs(OUT_DIR, exist_ok=True)
# ignore? response = get(url)
if (self.ignorePackage): if not response.status_code == 200:
return log(1, f"Non 200 response ({response.status_code}). Skipping... ({url})")
# parse sub-tag for Package return
if (tag == 'LocalDir'):
self.LocalDir = attrib['name']
elif (tag == 'FileSize'):
self.FileSize = attrib['val']
elif (tag == 'UncompressedFileSize'):
self.UncompressedFileSize = attrib['val']
elif (tag == 'IndexFileCRC'):
self.IndexFileCRC = attrib['val']
elif (tag == 'IndexFileSig'):
self.IndexFileSig = attrib['val']
elif (tag == 'Version'):
self.Version = attrib['val']
elif (tag == 'FileName'):
self.FileName = attrib['val']
elif (tag == 'Language'):
self.Language = attrib['val']
# called for each closing tag.
def end(self, tag):
if (tag == 'Package'):
if (not self.ignorePackage and self.Language in lang):
need2Download = True
fn = self.FileName.replace(':', '/')
zeroFile = LOCAL_DLC_DIR + fn[:-4] + '/0'
# check crc32 of local 0 file
if os.path.exists(zeroFile):
crc32 = crc32ForFile(zeroFile)
need2Download = crc32 != self.IndexFileCRC
if need2Download:
print("crc mismatch actual=%s expected=%s." % (crc32, self.IndexFileCRC))
# now download it
if need2Download:
doDownload(fn)
self.ignorePackage = True
def data(self, data): pass
def close(self): pass
if __name__ == '__main__': data = response.content
with open("dlcs.txt", "r") as dlcs:
log(0, f"Downloaded {filename} ({len(data)} bytes).")
with open(OUT_DIR + f"/{filename}", "wb+") as outFile:
outFile.write(data)
return data # So it can be used by other functions, but still be saved to disk
def getDLCIndexXml(url: str, filename: str):
zippedFileData = downloadFile(url, filename)
if not zippedFileData:
return
with zipfile.ZipFile(io.BytesIO(zippedFileData)) as z:
data = z.read(z.infolist()[0])
return data
def getDLCIndexes():
log(0, "Getting DLC Indexes...")
try:
os.makedirs(OUT_DIR + "/dlc", exist_ok=True)
masterIndex = getDLCIndexXml(BASE_URL + "dlc/DLCIndex.zip", "dlc/DLCIndex.zip")
if not masterIndex:
return []
tree = ET.fromstring(masterIndex)
lst = tree.findall("./IndexFile")
return [item.get("index").replace(":", "/") for item in lst]
except ET.ParseError as e:
log(2, f"Failed to parse XML: {e}")
return []
class DLCIndexParser(ET.XMLParser):
def start(self, tag, attrs):
if tag == "Package":
self.tier = attrs["tier"]
self.LocalDir = ""
self.FileSize = ""
self.UncompressedFileSize = ""
self.IndexFileCRC = ""
self.IndexFileSig = ""
self.Version = ""
self.FileName = ""
self.Language = ""
if tag == "LocalDir":
self.LocalDir = attrs["name"]
elif tag == "FileSize":
self.FileSize = attrs["val"]
elif tag == "UncompressedFileSize":
self.UncompressedFileSize = attrs["val"]
elif tag == "IndexFileCRC":
self.IndexFileCRC = attrs["val"]
elif tag == "IndexFileSig":
self.IndexFileSig = attrs["val"]
elif tag == "Version":
self.Version = attrs["val"]
elif tag == "FileName":
self.FileName = attrs["val"]
elif tag == "Language":
self.Language = attrs["val"]
def end(self, tag):
if tag == "Package":
return
if self.tier == "" or self.Language == "":
return
if self.Language not in LANGUAGE and not ALL_LANGUAGES:
return
if self.tier not in TIER and not ALL_TIERS:
return
DOWNLOAD_QUEUE.append(
[
BASE_URL + self.FileName.replace(":", "/"),
self.FileName.split(":")[-1],
self.FileName.split(":")[0],
]
) # So i can download them later
def data(self, data):
pass
def close(self):
pass
if __name__ == "__main__":
indexes = getDLCIndexes()
# Process Data (Get Urls)
for index in indexes:
try: try:
os.mkdir("./dlcs-" + lang[1]) dlcIndexXml = getDLCIndexXml(BASE_URL + index, "dlc/" + index.split("/")[1])
except: if not dlcIndexXml:
a = "" # Ignore continue
for line in dlcs: parser = ET.XMLParser(target=DLCIndexParser())
CURRENT_LINE = line.strip() parser.feed(dlcIndexXml)
os.mkdir("./dlcs-" + lang[1] + "/" + line.strip())
URL_DLC_BASE = 'https://' + line.strip() + '/netstorage/gameasset/direct/simpsons/'
LOCAL_DLC_DIR = os.path.expanduser(LOCAL_DLC_DIR)
if LOCAL_DLC_DIR[-1] != '/': LOCAL_DLC_DIR = LOCAL_DLC_DIR + '/'
index = getDlcIndex()
parser = ET.XMLParser(target=DlcIndexParser())
parser.feed(getZippedXml(URL_DLC_BASE + index))
parser.close() parser.close()
log(0, f"Processed {index}")
except ET.ParseError as e:
log(2, f"Failed to parse XML for index {index}: {e}")
# Download Dlcs
for download in DOWNLOAD_QUEUE:
if download[0] in DOWNLOADED:
continue
os.makedirs(
OUT_DIR + "/" + download[2], exist_ok=True
) # Make dlc subdirectory if it doesn't exist
downloadFile(download[0], download[2] + "/" + download[1])
DOWNLOADED.append(
download[0]
) # So it doesn't download the same file multiple times