Rewrote Downloader (It No Longer Downloads Duplicates)
This commit is contained in:
Glitched Panda
31
README.md
31
README.md
@@ -1,16 +1,35 @@
|
||||
# DLC-Downloader
|
||||
DLC Downloader for The Simpsons: Tapped Out. Thank you to [schdub](https://github.com/schdub/dlcsync) for (most) of the code. I decided to modify his, as i want to get this done before Jan. 24 when the game shuts down.
|
||||
DLC Downloader for The Simpsons: Tapped Out. It takes a while to download everything, so be patient.
|
||||
|
||||
## Usage
|
||||
You will need about 350GB of free storage, and [python3](https://www.python.org) installed.
|
||||
You will need about 30GB of free storage, and [python3](https://www.python.org) installed.
|
||||
```
|
||||
python3 dlcDownloader.py
|
||||
```
|
||||
|
||||
## Configuration
|
||||
If you want to for example change the language of dlcs the script downloads, you can do so at the top of the script.
|
||||
```
|
||||
verbose = False # display some debug messages
|
||||
lang = [ 'all', 'en' ] # en,fr,it,de,es,ko,zh,cn,pt,ru,tc,da,sv,no,nl,tr,th
|
||||
tier = [ 'all', '25', '50', '100', 'retina', 'iphone', 'ipad', 'ipad3' ] # 25,50,100,retina,iphone,ipad,ipad3
|
||||
```python
|
||||
OUT_DIR = "./dlc" # Directory where the dlc files will be downloaded
|
||||
LANGUAGE = [
|
||||
"all",
|
||||
"en"
|
||||
] # en,fr,it,de,es,ko,zh,cn,pt,ru,tc,da,sv,no,nl,tr,th
|
||||
|
||||
TIER = [
|
||||
"all",
|
||||
"25",
|
||||
"50",
|
||||
"100",
|
||||
"retina",
|
||||
"iphone",
|
||||
"ipad",
|
||||
"ipad3",
|
||||
"mp3",
|
||||
"caf",
|
||||
"wav",
|
||||
] # 25,50,100,retina,iphone,ipad,ipad3,mp3,caf,wav
|
||||
|
||||
ALL_LANGUAGES = True # Download the dlcs in every languages
|
||||
ALL_TIERS = True # Download the dlcs in every tier
|
||||
```
|
||||
|
||||
66
dlcs.txt
66
dlcs.txt
@@ -1,66 +0,0 @@
|
||||
thoh2013-3a3f8d.tstodlc.eamobile.com
|
||||
thoh2013-4-5-1-9rj43.tstodlc.eamobile.com
|
||||
xmas2013-4-6-0-3x9flx.tstodlc.eamobile.com
|
||||
xmas2013-4-6-1-89rrjf.tstodlc.eamobile.com
|
||||
feb2014-4-7-0-49fxs0.tstodlc.eamobile.com
|
||||
feb2014-4-7-1-ut90jr.tstodlc.eamobile.com
|
||||
apr2014-4-8-0-4kf0at.tstodlc.eamobile.com
|
||||
apr2014-4-8-1-k39fxt.tstodlc.eamobile.com
|
||||
may2014-4-8-2-2b33bc.tstodlc.eamobile.com
|
||||
jun2014-4-9-0-3jf0xc.tstodlc.eamobile.com
|
||||
jun2014-4-9-3-8ca2bb.tstodlc.eamobile.com
|
||||
jun2014-4-9-5-016374.tstodlc.eamobile.com
|
||||
aug2014-4-10-0-33r8k6.tstodlc.eamobile.com
|
||||
aug2014-4-10-2-a6c40f.tstodlc.eamobile.com
|
||||
oct2014-4-11-0-97pes0.tstodlc.eamobile.com
|
||||
oct2014-4-11-1-5rofxt.tstodlc.eamobile.com
|
||||
oct2014-4-11-5-393fjf.tstodlc.eamobile.com
|
||||
xmas2014-4-12-0-q58ubc.tstodlc.eamobile.com
|
||||
xmas2014td-4-12-5-uhtg34.tstodlc.eamobile.com
|
||||
superheroes-4-13-0-nv98q4.tstodlc.eamobile.com
|
||||
superheroes-takedown-4-13-5-93faz8.tstodlc.eamobile.com
|
||||
apr2015-4-14-0-39fajo9a.tstodlc.eamobile.com
|
||||
apr2015-4-14-5-48fuecxa.tstodlc.eamobile.com
|
||||
jun2015-4-15-0-jg84gs3f.tstodlc.eamobile.com
|
||||
jun2015-4-15-5-nvoiw8efj.tstodlc.eamobile.com
|
||||
aug2015-4-16-0-xzi394jf.tstodlc.eamobile.com
|
||||
aug2015-4-16-5-bv0e3fia.tstodlc.eamobile.com
|
||||
oct2015-4-17-0-fj39afaed.tstodlc.eamobile.com
|
||||
oct2015-4-17-5-bnc9a93.tstodlc.eamobile.com
|
||||
dec2015-4-18-0-ghe83hf.tstodlc.eamobile.com
|
||||
dec2015-4-18-5-bid9234.tstodlc.eamobile.com
|
||||
feb2016-4-19-0-i02ifj.tstodlc.eamobile.com
|
||||
feb2016-4-19-5-b83ifoa.tstodlc.eamobile.com
|
||||
apr2016-4-20-0-38faho8a.tstodlc.eamobile.com
|
||||
apr2016-4-20-5-w03ifjal.tstodlc.eamobile.com
|
||||
jun2016-4-21-0-tlj3zug8.tstodlc.eamobile.com
|
||||
july2016-4-21-5-5qs2b3jh.tstodlc.eamobile.com
|
||||
august2016-4-22-0-4urau4h8.tstodlc.eamobile.com
|
||||
september2016-4-22-5-st0pny01.tstodlc.eamobile.com
|
||||
oct2016-4-23-0-vrj28z.tstodlc.eamobile.com
|
||||
nov2016-4-23-5-l3osc5.tstodlc.eamobile.com
|
||||
dec2016-4-24-0-jccburm.tstodlc.eamobile.com
|
||||
jan2017-4-24-5-5j6kds4q.tstodlc.eamobile.com
|
||||
jan2017-4-25-0-ztk6mia7.tstodlc.eamobile.com
|
||||
march2017-4-25-5-n57dpzu3.tstodlc.eamobile.com
|
||||
march2017-4-25-6-g491pnkr.tstodlc.eamobile.com
|
||||
may2017-4-26-5-t7w59cqc.tstodlc.eamobile.com
|
||||
mar2017-4-26-0-s4hur7t7.tstodlc.eamobile.com
|
||||
jun2017-4-27-0-8ep9eykc.tstodlc.eamobile.com
|
||||
jul2017-4-27-5-ikp7ncq5.tstodlc.eamobile.com
|
||||
aug2017-4-28-0-5euxc0v8.tstodlc.eamobile.com
|
||||
aug2017-4-28-5-uzki2ogt.tstodlc.eamobile.com
|
||||
oct2017-4-29-0-2k8p9riy.tstodlc.eamobile.com
|
||||
nov2017-4-29-5-giai591j.tstodlc.eamobile.com
|
||||
dec2017-4-30-0-ph621tki.tstodlc.eamobile.com
|
||||
jan2018-4-31-0-zp9u3cog.tstodlc.eamobile.com
|
||||
mar2018-4-31-5-3dpu6dnx.tstodlc.eamobile.com
|
||||
mar2018-4-32-0-xk2u4fm5.tstodlc.eamobile.com
|
||||
mar2018-4-32-1-ie5c5aq9.tstodlc.eamobile.com
|
||||
may2018-4-32-5-9nz9wk4w.tstodlc.eamobile.com
|
||||
may2018-4-33-0-ibr0qhd8.tstodlc.eamobile.com
|
||||
jun2018-4-33-1-s7utav9k.tstodlc.eamobile.com
|
||||
july2018-4-33-5-ffd6uju9.tstodlc.eamobile.com
|
||||
july2018-4-34-0-xpb2jvat.tstodlc.eamobile.com
|
||||
aug2018-4-34-5-a98qp6kzcu.tstodlc.eamobile.com
|
||||
oct2018-4-35-0-uam5h44a.tstodlc.eamobile.com
|
||||
308
downloadDlcs.py
308
downloadDlcs.py
@@ -1,163 +1,183 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
import binascii
|
||||
import urllib.request
|
||||
import zipfile
|
||||
import io
|
||||
import os
|
||||
|
||||
verbose = False # display some debug messages
|
||||
lang = [ 'all', 'en' ] # en,fr,it,de,es,ko,zh,cn,pt,ru,tc,da,sv,no,nl,tr,th
|
||||
tier = [ 'all', '25', '50', '100', 'retina', 'iphone', 'ipad', 'ipad3' ] # 25,50,100,retina,iphone,ipad,ipad3
|
||||
LOCAL_DLC_DIR = './dlcs-'+ lang[1] + '/' # directory where DLC will be loaded
|
||||
URL_DLC_BASE = 'http://oct2018-4-35-0-uam5h44a.tstodlc.eamobile.com/netstorage/gameasset/direct/simpsons/'
|
||||
from requests import get
|
||||
|
||||
CURRENT_LINE = ''
|
||||
from colorama import Fore
|
||||
|
||||
def LOG(msg):
|
||||
if verbose: print(msg)
|
||||
from time import sleep
|
||||
|
||||
def crc32ForFile(filename):
|
||||
with open(filename,'rb') as f:
|
||||
buf = (binascii.crc32(f.read()) & 0xFFFFFFFF)
|
||||
return "%d" % buf
|
||||
import os, io, zipfile
|
||||
|
||||
def getZippedXml(url):
|
||||
r = urllib.request.urlopen(url)
|
||||
data = r.read()
|
||||
r.close()
|
||||
LOG("downloaded %d bytes" % len(data))
|
||||
OUT_DIR = "./dlc" # Directory where the dlc files will be downloaded
|
||||
LANGUAGE = [
|
||||
"all",
|
||||
"en"
|
||||
] # en,fr,it,de,es,ko,zh,cn,pt,ru,tc,da,sv,no,nl,tr,th
|
||||
|
||||
with open("./dlcs-" + lang[1] + "/" + CURRENT_LINE + "/DLCIndex.zip", "ab+") as file:
|
||||
file.write(data)
|
||||
file.close()
|
||||
TIER = [
|
||||
"all",
|
||||
"25",
|
||||
"50",
|
||||
"100",
|
||||
"retina",
|
||||
"iphone",
|
||||
"ipad",
|
||||
"ipad3",
|
||||
"mp3",
|
||||
"caf",
|
||||
"wav",
|
||||
] # 25,50,100,retina,iphone,ipad,ipad3,mp3,caf,wav
|
||||
|
||||
with zipfile.ZipFile(io.BytesIO(data)) as z:
|
||||
data = z.read(z.infolist()[0])
|
||||
LOG("unzipped %d bytes" % len(data))
|
||||
return data
|
||||
ALL_LANGUAGES = True # Download the dlcs in every languages
|
||||
ALL_TIERS = True # Download the dlcs in every tier
|
||||
|
||||
def getDlcIndex():
|
||||
tree = ET.fromstring(getZippedXml(URL_DLC_BASE + 'dlc/DLCIndex.zip'))
|
||||
lst = tree.findall('./IndexFile')
|
||||
return lst[0].get('index').replace(':', '/')
|
||||
BASE_URL = "http://oct2018-4-35-0-uam5h44a.tstodlc.eamobile.com/netstorage/gameasset/direct/simpsons/"
|
||||
|
||||
def getRest(dlFile, fromUrl):
|
||||
existSize = 0
|
||||
req = urllib.request.Request(fromUrl)
|
||||
if os.path.exists(dlFile):
|
||||
outputFile = open(dlFile, "ab")
|
||||
existSize = os.path.getsize(dlFile)
|
||||
# if the file exists, then download only the remainder
|
||||
req.headers['Range'] = 'bytes=%s-' % (existSize)
|
||||
else:
|
||||
outputFile = open(dlFile,"wb")
|
||||
webPage = urllib.request.urlopen(req)
|
||||
if verbose:
|
||||
for k, v in webPage.headers.items():
|
||||
LOG("%s=%s" % (k, v))
|
||||
# if we already have the whole file, there is no need to download it again
|
||||
ok = False
|
||||
numBytes = 0
|
||||
webSize = int(webPage.headers['Content-Length'])
|
||||
if webSize == existSize:
|
||||
LOG("File (%s) was already downloaded from URL (%s)" % (dlFile, fromUrl))
|
||||
ok = True
|
||||
else:
|
||||
#LOG("Downloading %d more bytes" % (webSize-existSize))
|
||||
while 1:
|
||||
data = webPage.read(8192)
|
||||
if not data: break
|
||||
outputFile.write(data)
|
||||
numBytes = numBytes + len(data)
|
||||
ok = numBytes == webSize
|
||||
LOG("downloaded %d bytes from %d" % (numBytes, webSize))
|
||||
webPage.close()
|
||||
outputFile.close()
|
||||
return ok
|
||||
DOWNLOAD_QUEUE = [] # [ Url, Filename, Folder ]
|
||||
DOWNLOADED = []
|
||||
|
||||
def doDownload(fn):
|
||||
print('./dlcs-' + lang[1] + '/' + CURRENT_LINE + '/' + fn)
|
||||
|
||||
tempFileName = fn.replace('/', '#')
|
||||
bytesCount = getRest('./dlcs-' + lang[1] + '/'+ CURRENT_LINE + '/' + tempFileName, URL_DLC_BASE + fn)
|
||||
def log(severity: int, message: str):
|
||||
if severity == 0:
|
||||
print(Fore.BLUE + "[i] " + Fore.WHITE + message)
|
||||
elif severity == 1:
|
||||
print(Fore.YELLOW + "[!] " + Fore.WHITE + message)
|
||||
elif severity == 2:
|
||||
print(Fore.RED + "[!] " + Fore.WHITE + message)
|
||||
else:
|
||||
print(Fore.WHITE + message)
|
||||
|
||||
class DlcIndexParser:
|
||||
ignorePackage = True
|
||||
# called for each opening tag.
|
||||
def start(self, tag, attrib):
|
||||
#LOG("tag='%s' attrib='%s'" % (tag, attrib))
|
||||
if (tag == 'Package'):
|
||||
# initilize variables for each Package
|
||||
self.ignorePackage = False
|
||||
self.LocalDir = ''
|
||||
self.FileSize = ''
|
||||
self.UncompressedFileSize = ''
|
||||
self.IndexFileCRC = ''
|
||||
self.IndexFileSig = ''
|
||||
self.Version = ''
|
||||
self.FileName = ''
|
||||
self.Language = ''
|
||||
|
||||
if attrib['ignore'] == 'true' or attrib['tier'] not in tier:
|
||||
self.ignorePackage = True
|
||||
def downloadFile(url: str, filename: str):
|
||||
os.makedirs(OUT_DIR, exist_ok=True)
|
||||
|
||||
# ignore?
|
||||
if (self.ignorePackage):
|
||||
return
|
||||
# parse sub-tag for Package
|
||||
if (tag == 'LocalDir'):
|
||||
self.LocalDir = attrib['name']
|
||||
elif (tag == 'FileSize'):
|
||||
self.FileSize = attrib['val']
|
||||
elif (tag == 'UncompressedFileSize'):
|
||||
self.UncompressedFileSize = attrib['val']
|
||||
elif (tag == 'IndexFileCRC'):
|
||||
self.IndexFileCRC = attrib['val']
|
||||
elif (tag == 'IndexFileSig'):
|
||||
self.IndexFileSig = attrib['val']
|
||||
elif (tag == 'Version'):
|
||||
self.Version = attrib['val']
|
||||
elif (tag == 'FileName'):
|
||||
self.FileName = attrib['val']
|
||||
elif (tag == 'Language'):
|
||||
self.Language = attrib['val']
|
||||
# called for each closing tag.
|
||||
def end(self, tag):
|
||||
if (tag == 'Package'):
|
||||
if (not self.ignorePackage and self.Language in lang):
|
||||
need2Download = True
|
||||
fn = self.FileName.replace(':', '/')
|
||||
zeroFile = LOCAL_DLC_DIR + fn[:-4] + '/0'
|
||||
# check crc32 of local 0 file
|
||||
if os.path.exists(zeroFile):
|
||||
crc32 = crc32ForFile(zeroFile)
|
||||
need2Download = crc32 != self.IndexFileCRC
|
||||
if need2Download:
|
||||
print("crc mismatch actual=%s expected=%s." % (crc32, self.IndexFileCRC))
|
||||
# now download it
|
||||
if need2Download:
|
||||
doDownload(fn)
|
||||
self.ignorePackage = True
|
||||
def data(self, data): pass
|
||||
def close(self): pass
|
||||
response = get(url)
|
||||
if not response.status_code == 200:
|
||||
log(1, f"Non 200 response ({response.status_code}). Skipping... ({url})")
|
||||
return
|
||||
|
||||
if __name__ == '__main__':
|
||||
with open("dlcs.txt", "r") as dlcs:
|
||||
data = response.content
|
||||
|
||||
log(0, f"Downloaded {filename} ({len(data)} bytes).")
|
||||
with open(OUT_DIR + f"/{filename}", "wb+") as outFile:
|
||||
outFile.write(data)
|
||||
|
||||
return data # So it can be used by other functions, but still be saved to disk
|
||||
|
||||
|
||||
def getDLCIndexXml(url: str, filename: str):
|
||||
zippedFileData = downloadFile(url, filename)
|
||||
if not zippedFileData:
|
||||
return
|
||||
|
||||
with zipfile.ZipFile(io.BytesIO(zippedFileData)) as z:
|
||||
data = z.read(z.infolist()[0])
|
||||
return data
|
||||
|
||||
|
||||
def getDLCIndexes():
|
||||
log(0, "Getting DLC Indexes...")
|
||||
try:
|
||||
os.makedirs(OUT_DIR + "/dlc", exist_ok=True)
|
||||
masterIndex = getDLCIndexXml(BASE_URL + "dlc/DLCIndex.zip", "dlc/DLCIndex.zip")
|
||||
if not masterIndex:
|
||||
return []
|
||||
|
||||
tree = ET.fromstring(masterIndex)
|
||||
lst = tree.findall("./IndexFile")
|
||||
return [item.get("index").replace(":", "/") for item in lst]
|
||||
|
||||
except ET.ParseError as e:
|
||||
log(2, f"Failed to parse XML: {e}")
|
||||
return []
|
||||
|
||||
|
||||
class DLCIndexParser(ET.XMLParser):
|
||||
def start(self, tag, attrs):
|
||||
if tag == "Package":
|
||||
self.tier = attrs["tier"]
|
||||
|
||||
self.LocalDir = ""
|
||||
self.FileSize = ""
|
||||
self.UncompressedFileSize = ""
|
||||
self.IndexFileCRC = ""
|
||||
self.IndexFileSig = ""
|
||||
self.Version = ""
|
||||
self.FileName = ""
|
||||
self.Language = ""
|
||||
|
||||
if tag == "LocalDir":
|
||||
self.LocalDir = attrs["name"]
|
||||
elif tag == "FileSize":
|
||||
self.FileSize = attrs["val"]
|
||||
elif tag == "UncompressedFileSize":
|
||||
self.UncompressedFileSize = attrs["val"]
|
||||
elif tag == "IndexFileCRC":
|
||||
self.IndexFileCRC = attrs["val"]
|
||||
elif tag == "IndexFileSig":
|
||||
self.IndexFileSig = attrs["val"]
|
||||
elif tag == "Version":
|
||||
self.Version = attrs["val"]
|
||||
elif tag == "FileName":
|
||||
self.FileName = attrs["val"]
|
||||
elif tag == "Language":
|
||||
self.Language = attrs["val"]
|
||||
|
||||
def end(self, tag):
|
||||
if tag == "Package":
|
||||
return
|
||||
|
||||
if self.tier == "" or self.Language == "":
|
||||
return
|
||||
|
||||
if self.Language not in LANGUAGE and not ALL_LANGUAGES:
|
||||
return
|
||||
if self.tier not in TIER and not ALL_TIERS:
|
||||
return
|
||||
|
||||
DOWNLOAD_QUEUE.append(
|
||||
[
|
||||
BASE_URL + self.FileName.replace(":", "/"),
|
||||
self.FileName.split(":")[-1],
|
||||
self.FileName.split(":")[0],
|
||||
]
|
||||
) # So i can download them later
|
||||
|
||||
def data(self, data):
|
||||
pass
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
indexes = getDLCIndexes()
|
||||
|
||||
# Process Data (Get Urls)
|
||||
for index in indexes:
|
||||
try:
|
||||
os.mkdir("./dlcs-" + lang[1])
|
||||
except:
|
||||
a = "" # Ignore
|
||||
dlcIndexXml = getDLCIndexXml(BASE_URL + index, "dlc/" + index.split("/")[1])
|
||||
if not dlcIndexXml:
|
||||
continue
|
||||
|
||||
for line in dlcs:
|
||||
CURRENT_LINE = line.strip()
|
||||
os.mkdir("./dlcs-" + lang[1] + "/" + line.strip())
|
||||
|
||||
URL_DLC_BASE = 'https://' + line.strip() + '/netstorage/gameasset/direct/simpsons/'
|
||||
LOCAL_DLC_DIR = os.path.expanduser(LOCAL_DLC_DIR)
|
||||
if LOCAL_DLC_DIR[-1] != '/': LOCAL_DLC_DIR = LOCAL_DLC_DIR + '/'
|
||||
index = getDlcIndex()
|
||||
parser = ET.XMLParser(target=DlcIndexParser())
|
||||
parser.feed(getZippedXml(URL_DLC_BASE + index))
|
||||
parser = ET.XMLParser(target=DLCIndexParser())
|
||||
parser.feed(dlcIndexXml)
|
||||
parser.close()
|
||||
|
||||
log(0, f"Processed {index}")
|
||||
except ET.ParseError as e:
|
||||
log(2, f"Failed to parse XML for index {index}: {e}")
|
||||
|
||||
# Download Dlcs
|
||||
for download in DOWNLOAD_QUEUE:
|
||||
if download[0] in DOWNLOADED:
|
||||
continue
|
||||
|
||||
os.makedirs(
|
||||
OUT_DIR + "/" + download[2], exist_ok=True
|
||||
) # Make dlc subdirectory if it doesn't exist
|
||||
downloadFile(download[0], download[2] + "/" + download[1])
|
||||
|
||||
DOWNLOADED.append(
|
||||
download[0]
|
||||
) # So it doesn't download the same file multiple times
|
||||
|
||||
Reference in New Issue
Block a user