From e3961bf1bfb5ea1c26b3eac24f9f5409a415d93d Mon Sep 17 00:00:00 2001 From: Glitched Panda <82221876+GlitchedPanda@users.noreply.github.com> Date: Mon, 6 Jan 2025 15:37:39 +0000 Subject: [PATCH] Rewrote Downloader (It No Longer Downloads Duplicates) --- README.md | 31 ++++- dlcs.txt | 66 ----------- downloadDlcs.py | 308 ++++++++++++++++++++++++++---------------------- 3 files changed, 189 insertions(+), 216 deletions(-) delete mode 100644 dlcs.txt diff --git a/README.md b/README.md index bc6e412..aaeabf4 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,35 @@ # DLC-Downloader -DLC Downloader for The Simpsons: Tapped Out. Thank you to [schdub](https://github.com/schdub/dlcsync) for (most) of the code. I decided to modify his, as i want to get this done before Jan. 24 when the game shuts down. +DLC Downloader for The Simpsons: Tapped Out. It takes a while to download everything, so be patient. ## Usage -You will need about 350GB of free storage, and [python3](https://www.python.org) installed. +You will need about 30GB of free storage, and [python3](https://www.python.org) installed. ``` python3 dlcDownloader.py ``` ## Configuration If you want to for example change the language of dlcs the script downloads, you can do so at the top of the script. -``` -verbose = False # display some debug messages -lang = [ 'all', 'en' ] # en,fr,it,de,es,ko,zh,cn,pt,ru,tc,da,sv,no,nl,tr,th -tier = [ 'all', '25', '50', '100', 'retina', 'iphone', 'ipad', 'ipad3' ] # 25,50,100,retina,iphone,ipad,ipad3 +```python +OUT_DIR = "./dlc" # Directory where the dlc files will be downloaded +LANGUAGE = [ + "all", + "en" +] # en,fr,it,de,es,ko,zh,cn,pt,ru,tc,da,sv,no,nl,tr,th + +TIER = [ + "all", + "25", + "50", + "100", + "retina", + "iphone", + "ipad", + "ipad3", + "mp3", + "caf", + "wav", +] # 25,50,100,retina,iphone,ipad,ipad3,mp3,caf,wav + +ALL_LANGUAGES = True # Download the dlcs in every languages +ALL_TIERS = True # Download the dlcs in every tier ``` diff --git a/dlcs.txt b/dlcs.txt deleted file mode 100644 index d06db8f..0000000 --- a/dlcs.txt +++ /dev/null @@ -1,66 +0,0 @@ -thoh2013-3a3f8d.tstodlc.eamobile.com -thoh2013-4-5-1-9rj43.tstodlc.eamobile.com -xmas2013-4-6-0-3x9flx.tstodlc.eamobile.com -xmas2013-4-6-1-89rrjf.tstodlc.eamobile.com -feb2014-4-7-0-49fxs0.tstodlc.eamobile.com -feb2014-4-7-1-ut90jr.tstodlc.eamobile.com -apr2014-4-8-0-4kf0at.tstodlc.eamobile.com -apr2014-4-8-1-k39fxt.tstodlc.eamobile.com -may2014-4-8-2-2b33bc.tstodlc.eamobile.com -jun2014-4-9-0-3jf0xc.tstodlc.eamobile.com -jun2014-4-9-3-8ca2bb.tstodlc.eamobile.com -jun2014-4-9-5-016374.tstodlc.eamobile.com -aug2014-4-10-0-33r8k6.tstodlc.eamobile.com -aug2014-4-10-2-a6c40f.tstodlc.eamobile.com -oct2014-4-11-0-97pes0.tstodlc.eamobile.com -oct2014-4-11-1-5rofxt.tstodlc.eamobile.com -oct2014-4-11-5-393fjf.tstodlc.eamobile.com -xmas2014-4-12-0-q58ubc.tstodlc.eamobile.com -xmas2014td-4-12-5-uhtg34.tstodlc.eamobile.com -superheroes-4-13-0-nv98q4.tstodlc.eamobile.com -superheroes-takedown-4-13-5-93faz8.tstodlc.eamobile.com -apr2015-4-14-0-39fajo9a.tstodlc.eamobile.com -apr2015-4-14-5-48fuecxa.tstodlc.eamobile.com -jun2015-4-15-0-jg84gs3f.tstodlc.eamobile.com -jun2015-4-15-5-nvoiw8efj.tstodlc.eamobile.com -aug2015-4-16-0-xzi394jf.tstodlc.eamobile.com -aug2015-4-16-5-bv0e3fia.tstodlc.eamobile.com -oct2015-4-17-0-fj39afaed.tstodlc.eamobile.com -oct2015-4-17-5-bnc9a93.tstodlc.eamobile.com -dec2015-4-18-0-ghe83hf.tstodlc.eamobile.com -dec2015-4-18-5-bid9234.tstodlc.eamobile.com -feb2016-4-19-0-i02ifj.tstodlc.eamobile.com -feb2016-4-19-5-b83ifoa.tstodlc.eamobile.com -apr2016-4-20-0-38faho8a.tstodlc.eamobile.com -apr2016-4-20-5-w03ifjal.tstodlc.eamobile.com -jun2016-4-21-0-tlj3zug8.tstodlc.eamobile.com -july2016-4-21-5-5qs2b3jh.tstodlc.eamobile.com -august2016-4-22-0-4urau4h8.tstodlc.eamobile.com -september2016-4-22-5-st0pny01.tstodlc.eamobile.com -oct2016-4-23-0-vrj28z.tstodlc.eamobile.com -nov2016-4-23-5-l3osc5.tstodlc.eamobile.com -dec2016-4-24-0-jccburm.tstodlc.eamobile.com -jan2017-4-24-5-5j6kds4q.tstodlc.eamobile.com -jan2017-4-25-0-ztk6mia7.tstodlc.eamobile.com -march2017-4-25-5-n57dpzu3.tstodlc.eamobile.com -march2017-4-25-6-g491pnkr.tstodlc.eamobile.com -may2017-4-26-5-t7w59cqc.tstodlc.eamobile.com -mar2017-4-26-0-s4hur7t7.tstodlc.eamobile.com -jun2017-4-27-0-8ep9eykc.tstodlc.eamobile.com -jul2017-4-27-5-ikp7ncq5.tstodlc.eamobile.com -aug2017-4-28-0-5euxc0v8.tstodlc.eamobile.com -aug2017-4-28-5-uzki2ogt.tstodlc.eamobile.com -oct2017-4-29-0-2k8p9riy.tstodlc.eamobile.com -nov2017-4-29-5-giai591j.tstodlc.eamobile.com -dec2017-4-30-0-ph621tki.tstodlc.eamobile.com -jan2018-4-31-0-zp9u3cog.tstodlc.eamobile.com -mar2018-4-31-5-3dpu6dnx.tstodlc.eamobile.com -mar2018-4-32-0-xk2u4fm5.tstodlc.eamobile.com -mar2018-4-32-1-ie5c5aq9.tstodlc.eamobile.com -may2018-4-32-5-9nz9wk4w.tstodlc.eamobile.com -may2018-4-33-0-ibr0qhd8.tstodlc.eamobile.com -jun2018-4-33-1-s7utav9k.tstodlc.eamobile.com -july2018-4-33-5-ffd6uju9.tstodlc.eamobile.com -july2018-4-34-0-xpb2jvat.tstodlc.eamobile.com -aug2018-4-34-5-a98qp6kzcu.tstodlc.eamobile.com -oct2018-4-35-0-uam5h44a.tstodlc.eamobile.com diff --git a/downloadDlcs.py b/downloadDlcs.py index 9b08605..b5546fd 100644 --- a/downloadDlcs.py +++ b/downloadDlcs.py @@ -1,163 +1,183 @@ -#!/usr/bin/python3 - import xml.etree.ElementTree as ET -import binascii -import urllib.request -import zipfile -import io -import os -verbose = False # display some debug messages -lang = [ 'all', 'en' ] # en,fr,it,de,es,ko,zh,cn,pt,ru,tc,da,sv,no,nl,tr,th -tier = [ 'all', '25', '50', '100', 'retina', 'iphone', 'ipad', 'ipad3' ] # 25,50,100,retina,iphone,ipad,ipad3 -LOCAL_DLC_DIR = './dlcs-'+ lang[1] + '/' # directory where DLC will be loaded -URL_DLC_BASE = 'http://oct2018-4-35-0-uam5h44a.tstodlc.eamobile.com/netstorage/gameasset/direct/simpsons/' +from requests import get -CURRENT_LINE = '' +from colorama import Fore -def LOG(msg): - if verbose: print(msg) +from time import sleep -def crc32ForFile(filename): - with open(filename,'rb') as f: - buf = (binascii.crc32(f.read()) & 0xFFFFFFFF) - return "%d" % buf +import os, io, zipfile -def getZippedXml(url): - r = urllib.request.urlopen(url) - data = r.read() - r.close() - LOG("downloaded %d bytes" % len(data)) +OUT_DIR = "./dlc" # Directory where the dlc files will be downloaded +LANGUAGE = [ + "all", + "en" +] # en,fr,it,de,es,ko,zh,cn,pt,ru,tc,da,sv,no,nl,tr,th - with open("./dlcs-" + lang[1] + "/" + CURRENT_LINE + "/DLCIndex.zip", "ab+") as file: - file.write(data) - file.close() +TIER = [ + "all", + "25", + "50", + "100", + "retina", + "iphone", + "ipad", + "ipad3", + "mp3", + "caf", + "wav", +] # 25,50,100,retina,iphone,ipad,ipad3,mp3,caf,wav - with zipfile.ZipFile(io.BytesIO(data)) as z: - data = z.read(z.infolist()[0]) - LOG("unzipped %d bytes" % len(data)) - return data +ALL_LANGUAGES = True # Download the dlcs in every languages +ALL_TIERS = True # Download the dlcs in every tier -def getDlcIndex(): - tree = ET.fromstring(getZippedXml(URL_DLC_BASE + 'dlc/DLCIndex.zip')) - lst = tree.findall('./IndexFile') - return lst[0].get('index').replace(':', '/') +BASE_URL = "http://oct2018-4-35-0-uam5h44a.tstodlc.eamobile.com/netstorage/gameasset/direct/simpsons/" -def getRest(dlFile, fromUrl): - existSize = 0 - req = urllib.request.Request(fromUrl) - if os.path.exists(dlFile): - outputFile = open(dlFile, "ab") - existSize = os.path.getsize(dlFile) - # if the file exists, then download only the remainder - req.headers['Range'] = 'bytes=%s-' % (existSize) - else: - outputFile = open(dlFile,"wb") - webPage = urllib.request.urlopen(req) - if verbose: - for k, v in webPage.headers.items(): - LOG("%s=%s" % (k, v)) - # if we already have the whole file, there is no need to download it again - ok = False - numBytes = 0 - webSize = int(webPage.headers['Content-Length']) - if webSize == existSize: - LOG("File (%s) was already downloaded from URL (%s)" % (dlFile, fromUrl)) - ok = True - else: - #LOG("Downloading %d more bytes" % (webSize-existSize)) - while 1: - data = webPage.read(8192) - if not data: break - outputFile.write(data) - numBytes = numBytes + len(data) - ok = numBytes == webSize - LOG("downloaded %d bytes from %d" % (numBytes, webSize)) - webPage.close() - outputFile.close() - return ok +DOWNLOAD_QUEUE = [] # [ Url, Filename, Folder ] +DOWNLOADED = [] -def doDownload(fn): - print('./dlcs-' + lang[1] + '/' + CURRENT_LINE + '/' + fn) - tempFileName = fn.replace('/', '#') - bytesCount = getRest('./dlcs-' + lang[1] + '/'+ CURRENT_LINE + '/' + tempFileName, URL_DLC_BASE + fn) +def log(severity: int, message: str): + if severity == 0: + print(Fore.BLUE + "[i] " + Fore.WHITE + message) + elif severity == 1: + print(Fore.YELLOW + "[!] " + Fore.WHITE + message) + elif severity == 2: + print(Fore.RED + "[!] " + Fore.WHITE + message) + else: + print(Fore.WHITE + message) -class DlcIndexParser: - ignorePackage = True - # called for each opening tag. - def start(self, tag, attrib): - #LOG("tag='%s' attrib='%s'" % (tag, attrib)) - if (tag == 'Package'): - # initilize variables for each Package - self.ignorePackage = False - self.LocalDir = '' - self.FileSize = '' - self.UncompressedFileSize = '' - self.IndexFileCRC = '' - self.IndexFileSig = '' - self.Version = '' - self.FileName = '' - self.Language = '' - if attrib['ignore'] == 'true' or attrib['tier'] not in tier: - self.ignorePackage = True +def downloadFile(url: str, filename: str): + os.makedirs(OUT_DIR, exist_ok=True) - # ignore? - if (self.ignorePackage): - return - # parse sub-tag for Package - if (tag == 'LocalDir'): - self.LocalDir = attrib['name'] - elif (tag == 'FileSize'): - self.FileSize = attrib['val'] - elif (tag == 'UncompressedFileSize'): - self.UncompressedFileSize = attrib['val'] - elif (tag == 'IndexFileCRC'): - self.IndexFileCRC = attrib['val'] - elif (tag == 'IndexFileSig'): - self.IndexFileSig = attrib['val'] - elif (tag == 'Version'): - self.Version = attrib['val'] - elif (tag == 'FileName'): - self.FileName = attrib['val'] - elif (tag == 'Language'): - self.Language = attrib['val'] - # called for each closing tag. - def end(self, tag): - if (tag == 'Package'): - if (not self.ignorePackage and self.Language in lang): - need2Download = True - fn = self.FileName.replace(':', '/') - zeroFile = LOCAL_DLC_DIR + fn[:-4] + '/0' - # check crc32 of local 0 file - if os.path.exists(zeroFile): - crc32 = crc32ForFile(zeroFile) - need2Download = crc32 != self.IndexFileCRC - if need2Download: - print("crc mismatch actual=%s expected=%s." % (crc32, self.IndexFileCRC)) - # now download it - if need2Download: - doDownload(fn) - self.ignorePackage = True - def data(self, data): pass - def close(self): pass + response = get(url) + if not response.status_code == 200: + log(1, f"Non 200 response ({response.status_code}). Skipping... ({url})") + return -if __name__ == '__main__': - with open("dlcs.txt", "r") as dlcs: + data = response.content + + log(0, f"Downloaded {filename} ({len(data)} bytes).") + with open(OUT_DIR + f"/{filename}", "wb+") as outFile: + outFile.write(data) + + return data # So it can be used by other functions, but still be saved to disk + + +def getDLCIndexXml(url: str, filename: str): + zippedFileData = downloadFile(url, filename) + if not zippedFileData: + return + + with zipfile.ZipFile(io.BytesIO(zippedFileData)) as z: + data = z.read(z.infolist()[0]) + return data + + +def getDLCIndexes(): + log(0, "Getting DLC Indexes...") + try: + os.makedirs(OUT_DIR + "/dlc", exist_ok=True) + masterIndex = getDLCIndexXml(BASE_URL + "dlc/DLCIndex.zip", "dlc/DLCIndex.zip") + if not masterIndex: + return [] + + tree = ET.fromstring(masterIndex) + lst = tree.findall("./IndexFile") + return [item.get("index").replace(":", "/") for item in lst] + + except ET.ParseError as e: + log(2, f"Failed to parse XML: {e}") + return [] + + +class DLCIndexParser(ET.XMLParser): + def start(self, tag, attrs): + if tag == "Package": + self.tier = attrs["tier"] + + self.LocalDir = "" + self.FileSize = "" + self.UncompressedFileSize = "" + self.IndexFileCRC = "" + self.IndexFileSig = "" + self.Version = "" + self.FileName = "" + self.Language = "" + + if tag == "LocalDir": + self.LocalDir = attrs["name"] + elif tag == "FileSize": + self.FileSize = attrs["val"] + elif tag == "UncompressedFileSize": + self.UncompressedFileSize = attrs["val"] + elif tag == "IndexFileCRC": + self.IndexFileCRC = attrs["val"] + elif tag == "IndexFileSig": + self.IndexFileSig = attrs["val"] + elif tag == "Version": + self.Version = attrs["val"] + elif tag == "FileName": + self.FileName = attrs["val"] + elif tag == "Language": + self.Language = attrs["val"] + + def end(self, tag): + if tag == "Package": + return + + if self.tier == "" or self.Language == "": + return + + if self.Language not in LANGUAGE and not ALL_LANGUAGES: + return + if self.tier not in TIER and not ALL_TIERS: + return + + DOWNLOAD_QUEUE.append( + [ + BASE_URL + self.FileName.replace(":", "/"), + self.FileName.split(":")[-1], + self.FileName.split(":")[0], + ] + ) # So i can download them later + + def data(self, data): + pass + + def close(self): + pass + + +if __name__ == "__main__": + indexes = getDLCIndexes() + + # Process Data (Get Urls) + for index in indexes: try: - os.mkdir("./dlcs-" + lang[1]) - except: - a = "" # Ignore + dlcIndexXml = getDLCIndexXml(BASE_URL + index, "dlc/" + index.split("/")[1]) + if not dlcIndexXml: + continue - for line in dlcs: - CURRENT_LINE = line.strip() - os.mkdir("./dlcs-" + lang[1] + "/" + line.strip()) - - URL_DLC_BASE = 'https://' + line.strip() + '/netstorage/gameasset/direct/simpsons/' - LOCAL_DLC_DIR = os.path.expanduser(LOCAL_DLC_DIR) - if LOCAL_DLC_DIR[-1] != '/': LOCAL_DLC_DIR = LOCAL_DLC_DIR + '/' - index = getDlcIndex() - parser = ET.XMLParser(target=DlcIndexParser()) - parser.feed(getZippedXml(URL_DLC_BASE + index)) + parser = ET.XMLParser(target=DLCIndexParser()) + parser.feed(dlcIndexXml) parser.close() + + log(0, f"Processed {index}") + except ET.ParseError as e: + log(2, f"Failed to parse XML for index {index}: {e}") + + # Download Dlcs + for download in DOWNLOAD_QUEUE: + if download[0] in DOWNLOADED: + continue + + os.makedirs( + OUT_DIR + "/" + download[2], exist_ok=True + ) # Make dlc subdirectory if it doesn't exist + downloadFile(download[0], download[2] + "/" + download[1]) + + DOWNLOADED.append( + download[0] + ) # So it doesn't download the same file multiple times