Pberndt V4
Direkt zum Inhalt springen
Quellcode rsDownload.py
Sourcecode
import cgi
import urllib2
import urllib
import urlparse
import re
import os
import threading
import sys
import random
import time
import getopt
import cookielib
import optparse
import json, json.decoder
import termios
import tty
import fcntl
import select
import hashlib
import tempfile
import subprocess
from htmlentitydefs import name2codepoint
def restartRouter():
"""
Restart router to get a new IP
Hint: Add a conditional branch to your routerReset-Script which
tests if other rsDownload-instances are running and only continues
if they also started routerReset. i.e.:
while true; do
RSD_HAS_NO_ROUTERRESET_PENDING=0
for PID in $(pgrep -f rsDownload); do
if ! pgrep -f routerReset -P $PID > /dev/null; then
RSD_HAS_NO_ROUTERRESET_PENDING=1
fi
done
[ "$RSD_HAS_NO_ROUTERRESET_PENDING" == "0" ] && break
sleep 2
done
"""
os.system("aplay -q $(ls /usr/share/sounds/*/*.wav 2>/dev/null | head -n3 | tail -n1) 2>&1 >/dev/null &")
return os.system("routerReset") == 0
def doneHook(flags):
"""
Called when a download is finished
See handler.HANDLER_* for values of flags
"""
return
if not flags & handler.HANDLER_WAS_REDIRECTED:
os.system("aplay -q $(ls /usr/share/sounds/*/*.wav 2>/dev/null | head -n3 | tail -n1) 2>&1 >/dev/null &")
def infoSound():
"""
Play a sound if a URL is detected in URL detection mode or a question is
being asked
"""
os.system("aplay -q $(ls /usr/share/sounds/*/*.wav 2>/dev/null | head -n3 | tail -n1) 2>&1 >/dev/null &")
def errorHook():
"""
Called when an error occurred
"""
os.system("aplay -q $(ls /usr/share/sounds/*.wav 2>/dev/null | head -n3 | tail -n1) 2>&1 >/dev/null &")
def captchaHook(data, caller):
"""
Called when a captcha is to be deciphered
data holds the raw data of the image!
Use caller to check which handler is calling this
Return False or the code
"""
# Anticaptcha does not work ATM
if False and caller == "rapidshare" and os.system("which anticaptcha 2>&1 >/dev/null") == 0:
# Call anticaptcha
fileName = os.path.abspath(impFilename("captcha.jpg"))
file = open(fileName, "w")
file.write(data)
file.close()
try:
captcha = os.popen("anticaptcha %r --method 22a" % fileName).read().strip()
except:
os.unlink(fileName)
return False
os.unlink(fileName)
if len(captcha) != 4:
return False
ui.info("Anticaptcha detected captcha successfully: %s" % captcha)
return captcha
else:
# Play sound to inform the user that interaction is needed
os.system("aplay -q $(ls /usr/share/sounds/*.wav | head -n1) 2>&1 >/dev/null &")
return False
configFile = os.path.expanduser("~/.rsDownload.config.py")
if os.access(configFile, os.R_OK):
exec open(configFile).read() in globals()
RSDOWNLOD_VERSION = "1.4"
useReferer = ""
class HTTPCookieAndRefererProcessor(urllib2.BaseHandler):
def __init__(self, cookiejar=None):
import cookielib
if cookiejar is None:
cookiejar = cookielib.CookieJar()
self.cookiejar = cookiejar
def http_request(self, request):
global useReferer
if useReferer and not request.has_header("Referer"):
request.add_header("Referer", useReferer)
self.cookiejar.add_cookie_header(request)
return request
def http_response(self, request, response):
self.cookiejar.extract_cookies(response, request)
return response
def redirect_request(self, req, fp, code, msg, headers, newurl):
global useReferer
useReferer = "tmp#" + newurl
https_request = http_request
https_response = http_response
cookieJar = cookielib.CookieJar()
opener = urllib2.build_opener(HTTPCookieAndRefererProcessor(cookieJar))
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
urllib2.install_opener(opener)
fileList = []
doneList = []
class ui(object):
_COLOR = {
"error": "\033[00;31m",
"info": "\033[00;32m",
"warn": "\033[00;33m",
"info2": "\033[00;34m",
"def": "\033[0m"
}
hasTermWidth = True
@staticmethod
def _getTerminalWidth():
tsize = 80
if ui.hasTermWidth and sys.stdin.isatty():
oldSettings = termios.tcgetattr(sys.stdin.fileno())
tty.setraw(sys.stdin.fileno())
mfcntl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
sys.stdin.flush()
sys.stdout.flush()
sys.stdout.write("\033[18t")
sys.stdout.flush()
s, l, l = select.select([ sys.stdin ], [], [], 1)
if not s:
ui.hasTermWidth = False
fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, mfcntl)
termios.tcsetattr(sys.stdin.fileno(), termios.TCSADRAIN, oldSettings)
return tsize
fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, os.O_NONBLOCK | mfcntl)
size = sys.stdin.read(20)
if "t" not in size[1:]:
ui.hasTermWidth = False
fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, mfcntl)
termios.tcsetattr(sys.stdin.fileno(), termios.TCSADRAIN, oldSettings)
return tsize
try:
tsize = int(re.sub('[^0-9]', '', size.split(";")[-1]))
except:
tsize = 80
fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, mfcntl)
termios.tcsetattr(sys.stdin.fileno(), termios.TCSADRAIN, oldSettings)
return tsize
@staticmethod
def endProgress():
"""
Erase the progress bar from the terminal
(End progress())
"""
sys.stdout.write("\0338\033[J\r") # Restore cursor and erase till end of screen
ui.setTitle()
sys.stdout.flush()
_progressData = {}
@staticmethod
def startProgress():
"""
Start progress()
"""
ui._progressData = {
'start': time.time(),
'last': time.time(),
'data': 0,
'lastDsp': time.time() - 10
}
sys.stdout.write("\0337") # Store cursor position
@staticmethod
def _formatSize(size):
units = [ "b", "Kb", "Mb", "Gb" ]
ret = units.pop(0)
while size > 1024 and len(units) > 0:
size /= 1024.0
ret = units.pop(0)
return "%03.2f%s" % (size, ret)
@staticmethod
def _formatTime(seconds):
hours = int(seconds / 3600)
seconds %= 3600
minutes = int(seconds / 60)
seconds %= 60
return "%02d:%02d:%02d" % (hours, minutes, seconds)
@staticmethod
def progress(done, length):
"""
Diplay a progress message
"""
timeNow = time.time()
timeElapsed = timeNow - ui._progressData["start"]
if timeElapsed == 0:
avgSpeed = 0
else:
avgSpeed = done * 1.0 / timeElapsed
timeLast = ui._progressData["last"]
if timeNow - timeLast == 0:
speed = 0
else:
speed = (done - ui._progressData["data"]) * 1.0 / (timeNow - timeLast)
ui._progressData["last"] = time.time()
ui._progressData["data"] = done
if length > 0:
pdone = done * 1.0 / length
timeToGo = 0
if pdone > 0:
timeToGo = (timeElapsed / pdone) - timeElapsed
info = " %02d%% %s/%s %8s/s %s" % (pdone * 100, ui._formatSize(done),
ui._formatSize(length), ui._formatSize(speed),
ui._formatTime(timeToGo))
if sys.stdout.isatty():
if ui._progressData["lastDsp"] + 0.5 > timeNow:
return
ui._progressData["lastDsp"] = timeNow
sys.stdout.write("\0338\033[J\r") # Restore cursor and erase till end of screen
if length > 0:
barWidth = ui._getTerminalWidth() - len(info) - 5 - 2
barFilled = int(barWidth * pdone)
bar = ""
if barFilled > 0:
bar = "-" * (barFilled - 1) + ">"
bar += " " * (barWidth - len(bar))
bar = "[%s]" % bar
print " %s%s" % (bar, info),
else:
print " %s downloaded, %8s/s" % (ui._formatSize(done),
ui._formatSize(speed)),
else:
if ui._progressData["lastDsp"] + 5 < timeNow:
ui._progressData["lastDsp"] = timeNow
if length > 0:
print info
else:
print " %s downloaded, %8s/s" % (ui._formatSize(done),
ui._formatSize(speed))
sys.stdout.flush()
@staticmethod
def _gt(color):
"""
Return a colored ">>>" string
"""
if sys.stdout.isatty():
return "%s>>>%s" % (ui._COLOR[color], ui._COLOR["def"])
else:
return ">>>"
@staticmethod
def error(message):
"""
Output error message
"""
print >> sys.stderr, ui._gt("error"), message
@staticmethod
def info(message, type=1):
"""
Output info message
Don't use the type parameter, it's meant for use in
this script's main function only!
"""
if type == 1:
print ui._gt("info"), message
else:
print ui._gt("info2"), message
@staticmethod
def warn(message):
"""
Output warning message
"""
print ui._gt("warn"), message
@staticmethod
def ask(question):
"""
Ask a question
"""
ui.setTitle("Question pending - interaction required")
infoSound()
try:
import gtk
assert(gtk.gdk.display_get_default() != None)
hasGtk = True
except:
hasGtk = False
if hasGtk:
wnd = gtk.Dialog("rsDownload - Question", buttons=(gtk.STOCK_YES, gtk.RESPONSE_YES, gtk.STOCK_NO, gtk.RESPONSE_NO))
lbl = gtk.Label(question)
wnd.set_resizable(False)
lbl.set_padding(0, 10)
wnd.get_content_area().add(lbl)
wnd.show_all()
answer = wnd.run() == gtk.RESPONSE_YES
wnd.hide()
while gtk.events_pending():
gtk.main_iteration_do()
return answer
else:
print ui._gt("info"), question, " [y|n]: ",
savedSettings = termios.tcgetattr(sys.stdin.fileno())
tty.setraw(sys.stdin.fileno())
sys.stdin.flush()
answer = ""
while answer.lower() not in ("y", "n"):
answer = sys.stdin.read(1)
termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, savedSettings)
print answer
ui.setTitle()
return answer == "y"
@staticmethod
def input(question):
"""
Ask the user to input something (password)
"""
ui.setTitle("Question pending - interaction required")
infoSound()
try:
import gtk
assert(gtk.gdk.display_get_default() != None)
hasGtk = True
except:
hasGtk = False
if hasGtk:
wnd = gtk.Dialog("rsDownload - Question", buttons=(gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL))
wnd.set_resizable(False)
btn = wnd.add_button(gtk.STOCK_OK, gtk.RESPONSE_OK)
btn.set_flags(gtk.CAN_DEFAULT)
btn.grab_default()
entry = gtk.Entry()
lbl = gtk.Label(question)
lbl.set_padding(0, 10)
wnd.get_content_area().add(lbl)
wnd.get_content_area().add(entry)
wnd.show_all()
wnd.set_focus(entry)
answer = ""
if wnd.run() == gtk.RESPONSE_OK:
answer = entry.get_text()
wnd.hide()
while gtk.events_pending():
gtk.main_iteration_do()
return answer
else:
print ui._gt("info"), question + ": ",
ret = raw_input()
ui.setTitle()
return ret.strip()
@staticmethod
def wait(message, seconds):
"""
Wait for something. Message should contain a %d (which
will be replaced by the remaining seconds)
"""
until = round(time.time() + seconds)
ui.setTitle("Waiting")
sys.stdout.write("\0337") # Store cursor position
if sys.stdout.isatty():
while time.time() < until:
sys.stdout.write("\0338\0337\033[J\r") # Restore cursor and erase till end of screen
print ui._gt("warn"), message % round(until - time.time()),
sys.stdout.flush()
seconds -= 1
time.sleep(1)
sys.stdout.write("\0338\033[J\r") # Restore cursor and erase till end of screen
else:
print ui._gt("warn"), message % seconds
sys.stdout.flush()
while seconds > 0:
seconds -= 1
time.sleep(1)
ui.setTitle()
sys.stdout.flush()
titleCache = []
@staticmethod
def setTitle(title = None):
if not title:
if len(ui.titleCache) > 1:
ui.titleCache.pop()
title = ui.titleCache[-1]
else:
if len(ui.titleCache) == 1:
ui.titleCache.pop()
title = "Idle"
else:
ui.titleCache.append(title)
title = "rsDownload - " + title
sys.stdout.write("\033]0;%s\007" % title)
sys.stdout.flush()
@staticmethod
def basicCaptchaImage(url):
"""
Return a GTK image containing the image from the given
URL. For special captcha methods
"""
try:
dataObject = urllib2.urlopen(url)
data = dataObject.read()
except:
ui.error("Failed to load captcha ")
return False
try:
import gtk
assert(gtk.gdk.display_get_default() != None)
except:
ui.error("This operation requires pyGtk")
return False
loader = gtk.gdk.PixbufLoader()
try:
loader.write(data)
loader.close()
except:
ui.error("Failed to load captcha file")
ui.setTitle("Captcha - interaction required")
return False
image = gtk.Image()
image.set_from_animation(loader.get_animation())
return image
@staticmethod
def getCaptcha(url, caller=None, returnObject=False):
"""
Decipher the captcha from the given URL.
Will execute the function captchaHook or, if it fails,
display a GTK+ dialog, asking for the code (with
console / imagemagick as a fallback)
"""
try:
dataObject = urllib2.urlopen(url)
data = dataObject.read()
except:
ui.error("Failed to load captcha ")
return False
ui.setTitle("Captcha - interaction required")
# Try hook first
response = captchaHook(data, caller)
if response:
return response
# Then try GTK+
noGtk = False
try:
import gtk
assert(gtk.gdk.display_get_default() != None)
except:
noGtk = True
if not noGtk:
try:
loader = gtk.gdk.PixbufLoader()
try:
loader.write(data)
loader.close()
except:
ui.error("Failed to load captcha file")
ui.setTitle("Captcha - interaction required")
return False
dialog = gtk.Dialog(u"Get captcha", buttons=(gtk.STOCK_OK, gtk.RESPONSE_ACCEPT))
image = gtk.Image()
image.set_from_animation(loader.get_animation())
#image.set_from_pixbuf(loader.get_pixbuf())
dialog.vbox.add(image)
entry = gtk.Entry()
def returnOnReturn(widget, event):
if event.keyval == gtk.keysyms.Return:
dialog.response(gtk.RESPONSE_ACCEPT)
entry.connect("key-press-event", returnOnReturn)
dialog.vbox.add(entry)
dialog.show_all()
dialog.run()
dialog.hide()
while gtk.events_pending():
gtk.main_iteration_do()
ui.setTitle()
if entry.get_text() == "":
return False
return entry.get_text()
except:
ui.setTitle()
return False
# Try console finally
ui.info("Please enter the captcha from the following image")
display = os.popen("display -", "w")
display.write(data)
display.close()
ui.setTitle()
return raw_input(" Captcha: ").strip()
class handler(object):
HANDLER_WAS_REDIRECTED = 1
options = {} # Will be set by __main__
SIMULTANEOUS_POSSIBLE = False
NAME = "Base handler"
@staticmethod
def canHandle(url):
"""
Return if this handler is able to handle
this url
"""
pass
@staticmethod
def handle(url):
"""
Handles an URL
May append files to the global download list
Returns either
- True: Successful download
- False: Download failed
- bool, bitmask:
Additional information, see constants above
"""
pass
def matchGet(rex, string): # {{{
"""
Match regular expresion rex on string
Returns
- False if rex does not match anywhere in string
- The matched part of string if rex does not contain
parentheses
- The first group if only one group exists
- A touple with all group values elsewise
"""
if type(string) != str and type(string) != buffer:
ui.info("Debug: Matched against non-string. Propably a HTTP request failed.")
return False
match = re.search(rex, string)
if match:
if len(match.groups()) == 0:
return string[match.span()[0]:match.span()[1]]
if len(match.groups()) == 1:
return match.groups()[0]
else:
return match.groups()
return False
class WouldDispositException(Exception):
pass
def GET(url, changeReferer=False, dispositOk=False): # {{{
"""
Place a GET request on URL, return data
changeReferer will store the URL and use it as a referer header
for all further requests (during this download)
"""
try:
dataObject = urllib2.urlopen(url)
if "Content-Disposition" in dataObject.info() and not dispositOk:
raise WouldDispositException()
if changeReferer:
global useReferer
useReferer = url
return dataObject.read()
except WouldDispositException:
raise WouldDispositException()
except:
ui.error("Failed to download %s" % url)
return False
# }}}
def POST(url, data, changeReferer=False, headers=False): # {{{
"""
Place a POST request with data on URL,
return data
changeReferer will store the URL and use it as a referer header
for all further requests (during this download)
"""
try:
if changeReferer:
global useReferer
useReferer = url
if headers != False:
return urllib2.urlopen(urllib2.Request(url, data, headers)).read()
return urllib2.urlopen(url, data).read()
except:
ui.error("Failed to download %s" % url)
return False
# }}}
def addDownload(url, redirected=False): # {{{
"""
Add a download to the download list
"""
global fileList
fileList.insert(0, url)
if not redirected:
ui.info("Added new download %s" % url)
# }}}
def download(url, postData = None, targetFileName = None): # {{{
"""
Download url to targetFileName (will be extracted from url /
the content-disposition header if omitted)
Returns False on failure and the file name elsewise.
"""
filename = False
targetFile = None
try:
# Escape characters, space only atm
for char in " !":
url = url.replace(char, "%%%2x" % ord(char))
request = urllib2.Request(url)
if postData:
request.add_data(postData)
urlobject = urllib2.urlopen(request)
headers = urlobject.info()
if "Content-Disposition" in headers and not targetFileName:
dispositionHeader = headers["Content-Disposition"]
targetFileName = matchGet("filename=(.)(.+)\\1$", dispositionHeader)
if targetFileName:
targetFileName = targetFileName[1]
if not targetFileName:
targetFileName = os.path.basename(urlobject.geturl())
if "?" in targetFileName:
targetFileName = targetFileName[:targetFileName.index("?")]
if not targetFileName:
ui.warn("Failed to obtain correct filename for download.")
targetFileName = "download_%s_%d" % (urlparse.urlparse(urlobject.geturl())[1], int(time.time()))
targetFileName = impFilename(targetFileName)
targetFile = open(targetFileName, "w")
if "Content-Length" in headers:
fileLength = int(headers["Content-Length"])
else:
fileLength = -1
ui.info("Downloading to %s" % targetFileName)
ui.setTitle("Downloading %s" % targetFileName)
ui.startProgress()
size = 0
while True:
data = urlobject.read(1024 * 8)
if not data:
break
size += len(data)
targetFile.write(data)
ui.progress(size, fileLength)
ui.endProgress()
del targetFile
ui.setTitle()
return targetFileName
except:
try:
if targetFile:
del targetFile
if targetFileName and os.access(targetFileName, os.F_OK):
os.unlink(targetFileName)
except:
pass
print
ui.error("Download failed: %s" % sys.exc_info()[1])
return False
# }}}
def impFilename(fileName): # {{{
"""
Return fileName or, if this file does already exists,
fileName~nnn.
"""
i = 0
if os.access(fileName, os.F_OK):
fileName = "%s~%%d" % fileName
while os.access(fileName % i, os.F_OK):
i += 1
fileName = fileName % i
return fileName
# }}}
def getFileSize(name):
return os.stat(name)[6]
def htmlEntityDecode(text):
"""
Decode HTML entities
"""
def entSubst(arg):
if arg.group(1) == "#":
return unichr(int(arg.group(2)))
else:
cp = name2codepoint.get(arg.group(2))
if cp:
cp = unichr(cp)
else:
cp = arg.group(2)
return cp
return re.sub("&(#?)(\d{1,5}|\w{1,8});", entSubst, text)
def getScriptPath():
return os.path.abspath(os.path.dirname(sys.argv[0]))
def testAndSetIfInProgress(file, downloader, finished=False): # {{{
"""
Stores download information into a file so that
two instances of the script won't try downloading
the same file
Returns TRUE if it's ok to download the file
This IS a race condition, but still better than nothing...
"""
if handler.options.ignoreActive:
return True, 0
if not os.access("/proc/%d/" % os.getpid(), os.F_OK):
return True, 0
if downloader:
downloader = re.sub("\s", "_", downloader)
if os.access(".rsDownloadProgress", os.R_OK):
inProgress = dict(
map(lambda x: ( x[1], [ x[0], x[2] ], ),
filter(lambda x: type(x) == tuple and len(x) == 3,
[ matchGet("^(.+?)\s+(.+)\s+([0-9]+)$", x) for x in open(".rsDownloadProgress").readlines() ])))
else:
inProgress = {}
if finished:
try: del inProgress[file]
except: pass
else:
if file in inProgress:
path = "/proc/%s/cmdline" % inProgress[file][1]
if os.access(path, os.F_OK) and os.path.basename(sys.argv[0]) in open(path).read():
return False, 0
dlerToPid = dict(inProgress.values())
if downloader in dlerToPid:
path = "/proc/%s/cmdline" % dlerToPid[downloader]
if os.access(path, os.F_OK) and os.path.basename(sys.argv[0]) in open(path).read():
return False, 1
inProgress[file] = [ downloader, str(os.getpid()) ]
if len(inProgress) == 0:
if os.access(".rsDownloadProgress", os.F_OK | os.W_OK):
os.unlink(".rsDownloadProgress")
else:
try: open(".rsDownloadProgress", "w").write("\n".join([ "%s %s %s" % (x[1][0], x[0], x[1][1]) for x in inProgress.items() ]))
except: pass
return True, 0
# }}}
class js: # {{{
"""
Run javascript from python
"""
executable = ""
scriptSourceCache = {}
@staticmethod
def _getExecutable():
candidates = []
for directory in os.environ["PATH"].split(":"):
try:
candidates += filter(lambda x: "jsscript" in x or "smjs" in x or x == "rhino", os.listdir(directory))
except:
pass
if len(candidates) > 0:
js.executable = candidates[0]
else:
js.executable = "-"
# Test if a sandbox is available
if js.executable != "-" and os.access("/usr/bin/sandbox", os.X_OK):
js.executable = "/usr/bin/sandbox " + js.executable
@staticmethod
def isAvailable():
"""
Returns TRUE if javascript is available
"""
if js.executable == "":
js._getExecutable()
if js.executable != "-":
return True
ui.warn("I could handle this file if rhino/spidermonkey was installed")
return False
@staticmethod
def run(url, expression=None, sha1=None):
"""
Load script from url, execute the given parameter
and return it's output. If url is no url it's interpreted
as javascript directly
If sha1 is given the source code is checked against
the given hash. (SHOULD be used)
"""
if not js.isAvailable():
ui.error("Bad coding style detected. isAvailable should be called earlier!")
os._exit(1)
if url not in js.scriptSourceCache:
if url[0:6].lower() == "http://":
scriptContent = GET(url)
else:
scriptContent = url
js.scriptSourceCache[url] = scriptContent
else:
scriptContent = js.scriptSourceCache[url]
if not scriptContent:
ui.warn("Failed to download javascript file")
return False
if sha1 and sha1 != hashlib.sha1(scriptContent).hexdigest():
ui.error("Consistency check failed for %s!" % os.path.basename(url))
return False
scriptFileHandle, scriptFile = tempfile.mkstemp(".js")
os.write(scriptFileHandle, "var document = { write: function(t) { print(t); } };\n");
os.write(scriptFileHandle, scriptContent)
if expression:
os.write(scriptFileHandle, "\nprint(" + expression + ");");
os.close(scriptFileHandle)
proc = subprocess.Popen([js.executable.split()[0]] + js.executable.split()[1:] + [scriptFile], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if proc.wait() != 0:
ui.warn("Failed to run javascript: %s" % proc.stderr.read())
#os.unlink(scriptFile)
return False
retval = proc.stdout.read()
os.unlink(scriptFile)
return retval
# }}}
def captcha_recaptcha(data):
captchaURL = matchGet('http://api.recaptcha.net/noscript\?k=[^"]+', data)
if not captchaURL:
captchaURL = matchGet('/recaptcha/api/challenge\?k=([^"\']+)', data)
if captchaURL: captchaURL = "http://api.recaptcha.net/noscript?k=" + captchaURL
if not captchaURL:
ui.error("Failed to find Captcha URL.")
return False
captchaURL = captchaURL.replace("http://api.recaptcha.net/noscript", "http://www.google.com/recaptcha/api/noscript")
if not captchaURL:
ui.error("Failed to find ReCaptcha URL")
return False
captchaPage = GET(captchaURL)
captchaImage = matchGet('src="(/?image[^"]+)', captchaPage)
captchaChallenge = matchGet('id="recaptcha_challenge_field"\s+value="([^"]+)"', captchaPage)
captchaResponse = ui.getCaptcha("http://api.recaptcha.net/" + captchaImage) # recaptcha_response_field
data = urllib.urlencode({ "recaptcha_challenge_field": captchaChallenge, "recaptcha_response_field": captchaResponse, "submit": "I'm a human" })
captchaResponse = POST(captchaURL, data)
if "Your answer was correct" not in captchaResponse:
ui.error("Captcha code is wrong")
return False
captchaAnswer = matchGet("<textarea[^>]+>([^<]+)<", captchaResponse)
return { "recaptcha_challenge_field": captchaAnswer, "recaptcha_response_field": "manual_challenge" }
def captcha_recaptcha_api(key):
data = GET("http://www.google.com/recaptcha/api/challenge?k=" + key + "&ajax=1&cachestop=0.27076800501623777")
captchaChallenge = matchGet("challenge\s*:\s*'([^']+)", data)
captchaImage = "http://www.google.com/recaptcha/api/image?c=" + captchaChallenge
captchaResponse = ui.getCaptcha(captchaImage)
return { "recaptcha_challenge_field": captchaChallenge, "recaptcha_response_field": captchaResponse }
class anonymTo(handler):
NAME = "anonym.to Redirector"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www\.)?anonym.to", url) != False
@staticmethod
def handle(url):
match = matchGet("^http://(?:www\.)?anonym.to/\?(.+)$", url)
if match:
addDownload(urllib.unquote(match), True)
return True, handler.HANDLER_WAS_REDIRECTED
return False
class binloadTo(handler):
NAME = "binload.to"
@staticmethod
def canHandle(url):
return matchGet("^http://(www\.)?binload.to/file/[a-z0-9]+", url) != False
@staticmethod
def handle(url):
data = GET(url)
if not data:
ui.error("Failed to load %s" % url)
return False
urlData = matchGet('action="(http://[^.]+.binload.to/download.php[^"]+)"', data)
dlSession = matchGet('dlSession"\s+value="([^"]+)"', data)
fileName = matchGet('Download: (.+?)<\/h3>', data)
if not urlData or not fileName or not dlSession:
ui.error("Failed to get download link from binload.to page")
return False
wait = matchGet('cdLength = ([0-9]+)', data)
if wait:
# Ugly but needed - netload adds some time to the time given in the JS oO
ui.wait("Waiting %d seconds for download ticket.", int(wait) * 2)
postData = "dlSession=%s&down=Download&login_nickname=&login_passwort=" % dlSession
downloadedFile = download(urlData, postData, fileName)
if not downloadedFile:
return False
if getFileSize(downloadedFile) < 1024 ** 2:
data = open(downloadedFile).read()
msg = matchGet('^<script[^(]+\("([^"]+)"\);', data)
if msg:
ui.error("Error while downloading: %s" % msg)
os.unlink(downloadedFile)
return False
return True
class blogRedirector(handler):
NAME = "Blog Redirector Service"
@staticmethod
def canHandle(url):
return matchGet("^http://[^/]+/(?:nl|ff|rc|ul).+", url) != False
@staticmethod
def handle(url):
page = GET(url)
frame = matchGet('src="(.+?)"', page)
try:
redirector = urllib2.urlopen(urlparse.urljoin(url, frame))
redirectUrl = redirector.geturl()
redirector.close()
if "error_traffic_exceeded_free" in redirectUrl:
ui.warn("Download limit exceeded. Restarting router...")
restartRouter()
return blogRedirector.handle(url)
except:
return False
if url:
addDownload(redirectUrl, True)
return True, handler.HANDLER_WAS_REDIRECTED
return False
class cryptIt(handler):
NAME = "crypt-it.com Redirector"
@staticmethod
def canHandle(url):
if not matchGet("^http://crypt-it.com/s/[A-Z0-9]+(?: [^ ]+)?$", url):
return False
try: from Crypto.Cipher import AES
except:
ui.warn("I could handle this URL if I had PyCrypto")
return False
return True
@staticmethod
def handle(url):
from Crypto.Cipher import AES
# Support for password protected folders
password = ""
urldata = url.split()
if len(urldata) > 1:
url, password = urldata
ui.info("Using password %s" % password)
page = POST(url, urllib.urlencode({ "pw": password, "a": "pw" }))
else:
page = GET(url)
if matchGet('id="folder_pw" name="pw"', page):
ui.error("This download requires a password. Please append it to the url, separated by a whitespace")
return False
id = matchGet("^http://crypt-it.com/s/([A-Z0-9]+)$", url)
# Crypt-it is kind of complicated. It uses a flash file to
# download the file list in an internal flash format (amf, see
# http://osflash.org/documentation/amf) This file contains the
# download urls, crypted using AES with a 24 byte cipher
#
# There's no need to understand the AMF format, how to obtain
# the answer and understanding where the URLs are stored can be
# done using wireshark
#
# Getting the key is more complicated,
# http://crypt-it.com/loaderv2.swf should contain it (Have a look
# at flasm) I, however, simply googled for it.
#
# Request URL list
post = "\0\0\0\0\0\1\0\x11cryptit2.getFiles\0\2/1\0\0\0\x11\x0a\0\0\0\2\2\0%s%s\2\0%s%s" % (chr(len(id)), id, chr(len(password)), password)
request = urllib2.Request("http://crypt-it.com/engine/", post)
request.add_header("Content-Type", "application/x-amf")
request.add_header("Referer", "http://crypt-it.com/mainv2.swf")
try:
data = urllib2.urlopen(request).read()
except:
ui.warn("Failed to download CCF")
return False
urlAdded = False
for codedUrl in re.findall("url\2\0\x80(.{128})", data):
# Interpret URL as ascii-hex encoded bytes
abytes = map(lambda z: int("".join(z), 16),
reduce(lambda x, y: x[:-1] + [(x[-1], y)] if len(x) > 0 and type(x[-1]) != tuple else x + [y], list(codedUrl), []))
cbytes = "".join(map(chr, abytes))
# Decrypt URL
key = "so5sxNsPKfNSDDZHayr32520"
url = filter(lambda x: x != "\0", AES.new(key).decrypt(cbytes))
addDownload(url)
urlAdded = True
if urlAdded:
return True, handler.HANDLER_WAS_REDIRECTED
else:
return False
class datenschleuderCC(handler):
NAME = "datenschleuder.cc Redirector"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www\.)?datenschleuder.cc", url) != False
@staticmethod
def handle(url):
data = GET(url)
if not data:
ui.error("Failed to load %s" % url)
return False
mirrors = [ 'http://(?:www\.)?rapidshare.com/[^"]+',
'http://(?:www\.)?uploaded.to/[^"]+',
'http://(?:www\.)?netload.in/[^"]+' ]
redir = matchGet('http://www.datenschleuder.cc/redir.php\?\S+mirror=(%s)[^"]+' % "|".join(mirrors),
data)
if not redir:
ui.error("Failed to get download frame link")
return False
data = GET(redir)
if not data:
ui.error("Failed to load %s" % redir)
return False
redir = matchGet(dlUrl, data)
if not redir:
ui.error("Failed to get download link")
return False
addDownload(redir)
return True, handler.HANDLER_WAS_REDIRECTED
class depositfilesCom(handler):
NAME = "depositfiles.com"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www\.)?depositfiles.com", url) != False
@staticmethod
def handle(url):
data = GET(url)
target = matchGet('<form[^>]+action="(/(?:[^/]+/)?files/[a-zA-Z0-9]+)"', data)
if not target:
ui.error('Failed to get download form')
return False
data = POST("http://depositfiles.com" + target, "gateway_result=1")
if not data or "The site is temporarily unavailable for we are making some important upgrades to its parts" in data:
ui.wait("Page offline. Trying again in %d seconds", 60)
return depositfilesCom.handle(url)
if "is already downloading a file from our system" in data or \
"You used up your limit for file" in data or \
"Bitte versuchen Sie noch mal nach" in data:
wait = int(matchGet("in\s+([0-9]+)\s+minute", data))
ui.warn("Download limit reached.")
if not restartRouter():
if wait:
ui.wait("Router restart failed. Waiting %d seconds", wait * 60)
else:
ui.warn("Router restart failed failed.")
return False
return depositfilesCom.handle(url)
wait = int(matchGet('id="download_waiter_remain">([^<]+)<\/', data))
url_part = matchGet('/get_file.php\?[^\']+', data)
if not url_part:
ui.error("The site changed it's design, can't find download url")
return False
ui.wait("Waiting %d seconds for download", wait + 2)
data = GET("http://depositfiles.com" + url_part)
downloadUrl = matchGet('action="(http://[^"]+)"[^>]+download_started', data)
return download(downloadUrl)
class easyShareCom(handler):
NAME = "easy-share.com"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www\.|w[0-9]+\.)?easy-share.com/[0-9]+", url) != False
@staticmethod
def handle(url):
data = GET(url, changeReferer=True)
if not 'src="/kaptcha' in data:
wait = matchGet("w='([0-9]+)'", data)
captchaAjax = matchGet("u='([^']+)'", data)
if not wait or not captchaAjax:
print data
ui.error("Failed to get download information")
return False
ui.wait("Waiting %d seconds for download", int(wait))
data = GET(urlparse.urljoin(url, captchaAjax))
downloadUrl = matchGet('action="([^"]+)"', data)
captcha = matchGet('img src="(/kaptcha[^"]+)', data)
if not downloadUrl or not captcha:
ui.error("Failed to get download information from AJAX form")
return False
captcha = urlparse.urljoin(url, captcha)
print captcha
form = { 'id': matchGet('name="id" value="([^"]+)"', data) }
form["captcha"] = ui.getCaptcha(captcha)
fileName = download(downloadUrl, urllib.urlencode(form))
if fileName and getFileSize(fileName) < 1024*25:
content = open(fileName).read()
if "and earn money.</title>" in content:
ui.error("Download failed. Captcha wrong?")
os.unlink(fileName)
return False
return fileName
class fastloadNet(handler):
NAME = "fast-load.net"
@staticmethod
def canHandle(url):
return matchGet("^http://(www\.)?fast-load.net/", url) != False
@staticmethod
def handle(url):
data = GET(url)
if not data:
ui.error("Failed to load %s" % url)
return False
fid = matchGet('name="fid"\s+value="([^"]+)"', data)
file = matchGet('<font style="font-color:grey; font-size:8pt;">([^<]+)<\/font>', data)
if not fid:
ui.error("Failed to get download link")
return False
captcha = ui.getCaptcha("http://www.fast-load.net//includes/captcha.php", "fastloadNet")
if not captcha:
return False
downloadedFile = download("http://www.fast-load.net//download.php", "fid=%s&captcha_code=%s" % (fid, captcha), file)
if not downloadedFile:
return False
if getFileSize(downloadedFile) < 1024 ** 2:
data = open(downloadedFile).read()
msg = matchGet('^wrong captcha', data)
ui.error("Wrong captcha")
return False
return True
class filesAg(handler):
NAME = "files.ag"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www\.)?files.ag/files/[0-9]+/.+", url) != False
@staticmethod
def handle(url):
data = POST(url, "start=Free")
downloadUrl = matchGet('action="([^"]+)"', data)
if not downloadUrl:
ui.error("Failed to get download url")
return False
wait = matchGet('download-timeout">([^<]+)<', data)
if wait:
ui.wait("Waiting %d seconds for download slot", int(wait))
return download("http://files.ag" + downloadUrl)
class fileFactory(handler):
NAME = "filefactory.com"
@staticmethod
def canHandle(url):
return matchGet("^http://(www\.)?filefactory.com/file/.+", url) != False
@staticmethod
def handle(url):
data = GET(url)
sndUrl = matchGet('action="(/dlf/[^"]+)', data)
if not sndUrl:
ui.error("Failed to get url")
return False
sndUrl = urlparse.urljoin(url, sndUrl)
data = GET(sndUrl)
downloadLink = matchGet('href="([^"]+)">Click here to begin', data)
wait = matchGet('id="countdown">([0-9]+)', data)
if wait:
ui.wait("Waiting %d seconds for download", int(wait))
file = download(downloadLink)
if file and getFileSize(file) < 1024*30:
content = open(file).read()
fail = matchGet('have exceeded the download limit for free users. Please wait ([0-9]+) minutes to download more files', content)
if fail:
os.unlink(file)
ui.warn("Download failed: Download limit exceeded. Restarting router")
if not restartRouter():
ui.wait("Waiting %d seconds for download", int(fail)*60)
return fileFactory.handle(url)
return file
class fileServe(handler):
NAME = "fileserve.com"
@staticmethod
def canHandle(url):
return matchGet("^http://(www\.)?fileserve.com/file/.+", url) != False
@staticmethod
def handle(url):
data = GET(url, True)
POST(url, "checkDownload=check")
recaptcha_key = matchGet("reCAPTCHA_publickey='([^']+)", data)
captcha_data = captcha_recaptcha_api(recaptcha_key)
captcha_data['recaptcha_shortencode_field'] = matchGet('recaptcha_shortencode_field" value="([^"]+)', data)
POST("http://www.fileserve.com/checkReCaptcha.php", urllib.urlencode(captcha_data))
wait = int(re.sub("[^0-9]+", "", POST(url, "downloadLink=wait")))
ui.wait("Waiting %d seconds for download", wait)
POST(url, "downloadLink=show")
file = download(url, "download=normal")
if file and getFileSize(file) < 1024*30:
content = open(file).read()
fail = matchGet("You need to wait ([0-9]+) seconds to start another download.", content)
if fail:
os.unlink(file)
if not restartRouter():
ui.wait("Waiting %d seconds for download", int(fail))
return fileServe.handle(url)
return file
class fileSonic(handler):
NAME = "filesonic.com"
@staticmethod
def canHandle(url):
return matchGet("^http://(www\.)?filesonic.com/file/.+", url) != False
@staticmethod
def handle(url):
data = GET(url, True)
new_url = "http://www.filesonic.com/file/" + matchGet('[0-9]+\?start=1', data)
data = POST(new_url, "", False)
if not "Download Ready:" in data:
if not "showRecaptcha()" in data:
wait = matchGet('countDownDelay\s*=\s*([0-9]+)', data)
tm = matchGet("name='tm' value='([^']+)", data)
tm_hash = matchGet("name='tm_hash' value='([^']+)", data)
if not tm or not tm_hash:
ui.error("Failed to find values for tm/tm_hash.")
return False
ui.wait("Waiting %d seconds for download", int(wait))
data = POST(new_url, "tm=" + tm + "&tm_hash=" + tm_hash)
captcha_data = captcha_recaptcha("http://api.recaptcha.net/noscript?k=" + matchGet('Recaptcha.create\("([^"]+)', data))
data = POST(new_url, urllib.urlencode(captcha_data), {"X-Requested-With": "XMLHttpRequest"})
download_url = matchGet('http://[^\.]+.filesonic.com/download/[0-9]+/[^"]+', data)
return download(download_url)
class freakShareNet(handler):
NAME = "Freakshare.net"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www.)?freakshare.net/files/", url) != False
@staticmethod
def handle(url):
data = GET(url, True)
wait = matchGet('time = ([0-9]+)', data)
if wait and int(wait) > 300:
ui.warn("You'd have to wait >300 seconds. Restarting router..")
if restartRouter():
return freakShareNet.handle(url)
if wait:
ui.wait("Waiting %d seconds for download", int(wait))
did = matchGet('value="([0-9]+)" name="did"', data)
data = POST(url, "section=benefit&did=" + did)
did = matchGet('value="([0-9]+)" name="did"', data)
if not did:
ui.error("Failed to get download page link.")
return False
return download(url, "submit=Download§ion=waitingtime&did=" + did)
class freeClipsCh(handler):
NAME = "free-clips.ch Redirector"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www.)?free-clips.ch/", url) != False
@staticmethod
def handle(url):
data = GET(url)
links = matchGet('<textarea name="links" id="links" '
'readonly="readonly">([^<]+)<\/textarea>', data)
if not links:
ui.error("Failed to get download links")
for link in links.split("\n"):
addDownload(htmlEntityDecode(link.strip()))
if links:
return True, handler.HANDLER_WAS_REDIRECTED
else:
return False
class hotFileCom(handler):
NAME = "hotfile.com"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www.)?hotfile.com/dl/", url) != False
@staticmethod
def handle(url):
data = GET(url)
if ">starthtimer()" in data:
wait = matchGet("d.getTime()\+([0-9]+)00", data)
if wait:
wait = int(wait)
else:
wait = 3600
if wait > 100:
ui.warn("Download limit reached. You'd have to wait %d seconds. Restarting router." % wait)
if not restartRouter():
ui.wait("Waiting for %d seconds.", wait)
else:
ui.wait("Waiting for %d seconds for download slot", wait)
action = matchGet('action="(/dl/[^"]+)"', data)
if not action:
ui.error("Failed to get download URL")
return False
action = urlparse.urljoin(url, action)
fields = [ "action", "tm", "tmhash", "wait", "waithash", "upidhash" ]
postData = {}
for field in fields:
postData[field] = matchGet('type=hidden name=' + field + ' value=([^> ]+)', data)
ui.wait("Waiting %d seconds for download", int(postData["wait"]))
data = POST(action, urllib.urlencode(postData), True)
action = matchGet('action="(/dl/[^"]+)"', data)
for test in (
'href="([^"]+)"[^>]*>Click here to download',
'href="([^"]+)"[^>]*>Klicken Sie hier zum Download',
'href="([^"]+)"[^>]*>Klicke hier zum Herunterladen'
):
downloadUrl = matchGet(test, data)
if downloadUrl: break
if not downloadUrl:
# Captcha?
captchaAnswer = captcha_recaptcha(data)
if not captchaAnswer:
return False
data = { "action": "checkcaptcha" }
data.update(captchaAnswer)
data = POST("http://hotfile.com" + action, urllib.urlencode(data))
if not downloadUrl:
for test in (
'href="([^"]+)"[^>]*>Click here to download',
'href="([^"]+)"[^>]*>Klicken Sie hier zum Download',
'href="([^"]+)"[^>]*>Klicke hier zum Herunterladen'
):
downloadUrl = matchGet(test, data)
if downloadUrl: break
if not downloadUrl:
ui.error("Failed to get download URL")
return False
return download(downloadUrl)
class letItBitNet(handler):
NAME = "letitbit.net"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www\.)?letitbit.net/download/", url) != False
@staticmethod
def handle(url):
data = GET(url)
form = {}
form["uid"] = matchGet('name="uid" value="([^"]{13}[^"]+)"', data)
form["frameset"] = "Download file"
form["fix"] = 1
data = POST("http://letitbit.net/download3.php", urllib.urlencode(form))
url2 = matchGet('src="(/tmpl/tmpl_frame_top[^"]+)', data)
if not url2:
ui.error("Page syntax error")
return False
data = GET('http://letitbit.net%s' % url2)
downloadUrl = matchGet(
'<a href="(http://[a-z0-9]+\.letitbit.net/download[0-9]+/[^<"]+)"', data)
if not downloadUrl:
ui.error("Failed to get download url")
return False
wait = matchGet('name="errt">([0-9]+)<', data)
if wait:
ui.wait("Waiting %d seconds for download", int(wait))
return download(downloadUrl)
class linkBucksCom(handler):
NAME = "linkbucks.com Redirector"
@staticmethod
def canHandle(url):
return matchGet("^http://[0-9a-z]+.linkbucks.com/url/", url)
@staticmethod
def handle(url):
addDownload(matchGet("^http://[a-z0-9]+.linkbucks.com/url/(.+)", url))
return True, handler.HANDLER_WAS_REDIRECTED
class linkSaveIn(handler):
NAME = "linksave.in Redirector"
@staticmethod
def canHandle(url):
if matchGet("^http://(?:www\.)?linksave.in", url) == False: return False
if not js.isAvailable():
ui.info("I could handle this URL if I had a javascript interpreter available (p.e. Rhino)")
return False
return True
@staticmethod
def handle(url):
data = GET(url, True)
post = {"login":"submit"}
if "besucherpasswort" in data:
post["besucherpasswort"] = ui.input("This page requires a password")
captcha = matchGet('/captcha/cap.php[^"]+', data)
if captcha:
# Has a captcha
for field in ("hash", "id"):
post[field] = matchGet('name="%s" value="([^"]+)"' % field, data)
captchaURL = urlparse.urljoin(url, captcha)
post["code"] = ui.getCaptcha(captchaURL)
if not post["code"]:
ui.error("Failed to get captcha")
return False
data = POST(url, urllib.urlencode(post))
if "Wrong code. Please retry" in data:
ui.warn("Misspelled captcha")
return linkSaveIn.handle(url)
further = re.findall('"(http://linksave.in/[^"/\\.]+)"', data)
urls = False
for page in further:
page1 = GET(page)
page2url = matchGet('scrolling="auto".+src="([^"]+)"', page1)
if not page2url:
# This is no error
continue
page2 = GET(page2url)
script = matchGet('<script (?:language="javascript"|type="text/javascript")>(?!if.parent)(.+?)<\/script>', page2)
if not script:
ui.error("Failed to find script in doorpage. Maybe the site changed it's design?")
return False
content = js.run(script)
if not "rapidshare.com/files" in content:
page3url = matchGet('http://linksave.in/[^\'"]+', content)
if not page3url:
ui.error("Failed to get second doorpage. Maybe the site changed it's design?")
print content
return False
page3 = GET(page3url)
script = matchGet('<script (?:language="javascript"|type="text/javascript")>(?!if.parent)(.+?)<\/script>', page3)
if not script:
ui.error("Failed to find script in second doorpage. Maybe the site changed it's design?")
return False
print "snd--script--"
content = js.run(script)
rsUrl = matchGet('"(http://[^\.]+\.rapidshare\.com/files/[^"]+)"', content)
if rsUrl:
addDownload(htmlEntityDecode(rsUrl))
urls = True
continue
print "Args. What's that?"
print content
sys.exit(1)
if urls:
return True, handler.HANDLER_WAS_REDIRECTED
return False
class linkSafeToClan(handler):
NAME = "link-safe.to-clan.de Redirector"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www.)?link-safe.to-clan.de/", url) != False
@staticmethod
def handle(url):
data = GET(url)
iframe = matchGet('<iframe[^>]+src="([^"]+)"', data)
if not iframe:
ui.error("Failed to get linked page")
return False
addDownload(iframe, True)
return True, handler.HANDLER_WAS_REDIRECTED
class linkedMoney(handler):
NAME = "linkedmoney.com Redirector"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www\.)?linkedmoney.com", url) != False
@staticmethod
def handle(url):
data = GET(url)
url = matchGet('<iframe id="linkedmoney_frame"[^>]+src="([^"]+)"', data)
if url:
addDownload(htmlEntityDecode(url), True)
return True, handler.HANDLER_WAS_REDIRECTED
return False
class megauploadCom(handler):
NAME = "megaupload.com"
@staticmethod
def canHandle(url):
return matchGet("^http://(www\.)?megaupload.com", url) != False
@staticmethod
def handle(url):
data = GET(url)
if not data:
ui.error("Failed to load %s" % url)
return False
formFields = {}
for field in [ "captchacode", "megavar" ]:
formFields[field] = matchGet('name="%s"\s+value="([^"]+)"' % field, data)
if not formFields[field]:
ui.error("Failed to get field %s from page" % field)
return False
fileName = matchGet("<b>(?:Dateiname|Filename):</font>\s*<[^>]+>(.*?)<", data)
captchaURL = matchGet('src="(http://[^"]+/gencap.php[^"]+)"', data)
formFields["captcha"] = ui.getCaptcha(urlparse.urljoin(url, captchaURL),
"megaupload")
if not formFields["captcha"]:
ui.error("Failed to get captcha")
return False
data = POST(url, urllib.urlencode(formFields))
durl = matchGet('"downloadlink"><a href="([^"]+)"', data)
#ui.wait("Waiting %2d seconds for download ticket", 45)
return download(durl, targetFileName=fileName) != False
class megaroticCom(megauploadCom):
NAME = "megarotic.com"
@staticmethod
def canHandle(url):
return matchGet("^http://(www\.)?megarotic.com", url) != False
class netfolderIn(handler):
NAME = "netfolder.in Redirector"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www.)?netfolder.in/", url) != False
@staticmethod
def handle(url):
data = GET(url)
if not data:
ui.error("Failed to load %s" % url)
return False
urls = map(htmlEntityDecode, re.findall('href="(http://netload.in/[^"]+)', data))
if urls:
for url in urls:
addDownload(url)
return True, handler.HANDLER_WAS_REDIRECTED
else:
return False
class netloadIn(handler):
NAME = "netload.in"
USED_BEFORE = False
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www.)?netload.in/", url) != False
@staticmethod
def handle(url):
if netloadIn.USED_BEFORE == True:
restartRouter()
netloadIn.USED_BEFORE = False
data = GET(url)
if not data:
ui.error("Failed to load %s" % url)
return False
if "Sorry, we don't host the requested file" in data:
ui.error("File %s not found" % url)
return False
# Step 1: Get captcha page
nurl = matchGet('href="(index.php\?[^"]+captcha=1)"', data)
if nurl:
if not nurl:
ui.error("Failed to get captcha page url")
return False
nurl = "http://netload.in/" + htmlEntityDecode(nurl)
data = GET(nurl)
if not data:
ui.error("Failed to load %s" % nurl)
return False
# Step 2: Get captcha
curl = matchGet('src="(share/includes/captcha.php[^"]+)"', data)
if not curl:
ui.error("Failed to get captcha url")
return False
captcha = ui.getCaptcha("http://netload.in/" + htmlEntityDecode(curl), "netload")
if not captcha:
ui.error("Failed to get captcha")
return False
# Step 3: Countdown
wait = matchGet("countdown\(([0-9]+)", data)
# Step 4: Door page
faction = matchGet('action="(index.php\?[^"]+)"', data)
fileId = matchGet('name="file_id"[^>]+value="([^"]+)"', data)
if not faction or not fileId:
ui.error("Failed to get download form data")
return False
faction = "http://netload.in/" + urllib2.unquote(faction)
postData = "file_id=%s&captcha_check=%s&start=" % (fileId, urllib.quote(captcha))
data = POST(faction, postData)
# Step 5: Countdown
wait = matchGet("countdown\(([0-9]+)", data)
if wait:
ui.wait("Waiting %2d seconds for download ticket.", int(wait) / 100)
# Step 6: Download
downloadUrl = matchGet('class="Orange_Link"\s+href="([^"]+)"', data)
if not downloadUrl:
ui.error("Failed to get download url.")
ui.warn("Did you enter the correct captcha code?!")
ui.warn("Consider restarting your router")
return False
netloadIn.USED_BEFORE = True
return download(htmlEntityDecode(downloadUrl)) != False
class rapidShare(handler):
NAME = "rapidshare.com"
@staticmethod
def canHandle(url):
#http://rs711l36.rapidshare.com/#!download|711|418026388|amateur435.rar|111339
#http://rapidshare.com/files/418026388/amateur435.rar
return matchGet("^http://(?:www\.|rs[0-9]+\.)?rapidshare.com", url) != False
@staticmethod
def handle(url):
host = matchGet("^http://([^/]+)", url)
fileID = matchGet('#!download|[^|]+|([0-9]+)', url)
if not fileID:
fileID = matchGet('files\/([0-9]+)', url)
fileName = matchGet('#!download|[^|]+|[^|]+|([^|]+)', url)
if not fileName:
fileName = matchGet('files\/[0-9]+\/(.+)', url)
if not fileID or not fileName:
ui.error("URL format not recognized")
return False
apiResponse = GET("http://api.rapidshare.com/cgi-bin/rsapi.cgi?sub=download_v1&fileid=%s&filename=%s&try=1&cbf=RSAPIDispatcher&cbid=1" % (fileID, fileName))
downloadToken = matchGet('RSAPIDispatcher\s*\(([0-9]),"DL:([^,]+),([^,]+),([0-9]+)', apiResponse)
if not downloadToken:
ui.error("Invalid API Response")
return False
if downloadToken[0] == "1":
ui.wait("Waiting %d seconds for download..", int(downloadToken[3]))
return download("http://%s/cgi-bin/rsapi.cgi?sub=download_v1&editparentlocation=0&bin=1&fileid=%s&filename=%s&dlauth=%s"
% (downloadToken[1], fileID, fileName, downloadToken[2]))
if downloadToken[0] == "2":
return download(url + "?directstart=1")
ui.info("Unknown rapidshare response: " + str(downloadToken))
return False
class rapidLayerIn(handler):
NAME = "rapidlayer.in Redirector"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www\.)?rapidlayer.in", url) != False
@staticmethod
def handle(url):
data = GET(url)
if not data:
ui.error("Failed to download %s" % url)
return False
urlEncoded = matchGet('}[a-z]="([^"]+)"', data)
sub = matchGet("([a-z])=\\1-([0-9]+)", data)
if not sub:
return False
sub = int(sub[1])
def decode(schar):
char = ord(schar)
if char < 128:
char -= sub
if char < 32:
char = 127 + (char - 32)
return chr(char)
url = "".join(map(decode, urlEncoded))
if url:
url = htmlEntityDecode(url)
addDownload(urllib.unquote(url), True)
return True, handler.HANDLER_WAS_REDIRECTED
return False
class relinkUs(handler):
NAME = "relink.us Redirector"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www\.)?relink.us", url) != False
@staticmethod
def handle(url):
data = GET(url, True)
downloads = re.findall("getFile\('([^']+)'\)", data)
urls = False
for file in downloads:
data = GET("http://www.relink.us/frame.php?" + file)
nurl = matchGet('<iframe[^>]+src="([^"]+)"', data)
if nurl:
addDownload(nurl)
urls = True
else:
ui.error("Failed to get URL for at least one file: %s" % nurl)
ui.wait("Waiting %d seconds for next request", 2)
if urls:
return True, handler.HANDLER_WAS_REDIRECTED
else:
return False
class s2l(handler):
NAME = "s2l.biz / share-links.biz redirector"
@staticmethod
def canHandle(url):
return False
if not matchGet("^http://(?:www\.)?(?:s2l|share-links).biz/.+", url): return False
if not js.isAvailable(): return False
try:
import gtk
assert(gtk.gdk.display_get_default() != None)
except:
ui.warn("I could handle this link if you install pyGTK")
return False
return True
@staticmethod
def handle(url):
import gtk
assert(gtk.gdk.display_get_default() != None)
data = POST(url, "dl.start=Normaler+Download")
captchaUrl = matchGet('(/captcha.gif[^"]+)"[^>]+usemap', data)
if not captchaUrl:
ui.error("Failed to get captcha image")
return False
captchaUrl = "http://share-links.biz" + matchGet('(/captcha.gif[^"]+)"[^>]+usemap', data)
coords = re.findall('coords="([^"]+)" href="([^"]+)"', data)
image = ui.basicCaptchaImage(captchaUrl)
window = gtk.Dialog()
window.set_title("Select the text from the background")
ebox = gtk.EventBox()
image.clicked = False
def click(widget, event):
image.clicked = image.get_pointer()
window.hide()
window.get_content_area().add(ebox)
ebox.add(image)
ebox.connect("button-press-event", click)
window.show_all()
window.run()
window.hide()
while gtk.events_pending():
gtk.main_iteration_do()
if not image.clicked: return False
x, y = image.clicked
nextstep = False
for coord, url in coords:
tx, ty, bx, by = (int(x.strip()) for x in coord.split(","))
if tx <= x and ty <= y and bx >= x and by >= y:
nextstep = "http://share-links.biz" + url
break
if not nextstep:
ui.error("Failed to find URL for these coordinates")
data = GET(nextstep)
for url in re.findall("_get\('([^']+)', [0-9]+, ''", data):
data = GET("http://share-links.biz/get/lnk/" + url)
# TODO JS Seite auch laden. Ansonsten ist der Parameter der JS Funktion leer
subpage = matchGet('src="([^"]+)" name="Main"', data)
data = GET(subpage)
script = matchGet('javascript">(.+)', data)
# TODO JS ausführen
class saveQube(handler):
NAME = "saveqube.com"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www\.)?saveqube.com", url) != False
@staticmethod
def handle(url):
data = POST(url, "free=")
dllink = matchGet('href="([^"]+)" id="freedownload"', data)
if not dllink:
ui.error("Failed to get download link")
return False
wait = matchGet('id="timer">([0-9]+)', data)
if wait:
ui.wait("Waiting %d seconds for download", int(wait))
return download(dllink)
class securedIn(handler):
NAME = "secured.in Redirector"
@staticmethod
def canHandle(url):
if not matchGet("^http://(?:www\.)?secured.in", url):
return False
return js.isAvailable()
@staticmethod
def handle(url):
data = GET(url)
if not data:
ui.error("Failed to download %s" % url)
return False
dlIds = re.findall("accessDownload\([0-9]+, [0-9]+, '([^']+)'", data)
if not dlIds:
ui.error("Captcha code wrong")
return False
for dlId in dlIds:
data = POST("http://secured.in/ajax-handler.php",
"cmd=download&download_id=%s" % dlId)
if data == "error":
ui.error("Failed to download dl id %s" % dlId)
continue
dlUrl = js.run("http://secured.in/scripts/cypher.js", "cypher(%r)" % data, "2e00570531f53a03bf3ee33196522057de5fe922")
if not dlUrl:
ui.warn("Failed to decypher url")
continue
addDownload(dlUrl)
return True, handler.HANDLER_WAS_REDIRECTED
class shrangleCom(handler):
NAME = "Shragle.com"
@staticmethod
def canHandle(url):
if not matchGet("^http://(?:www\.)?shragle.com/files", url):
return False
return True
@staticmethod
def handle(url):
data = GET(url)
url = matchGet('action="(http://[^"]+download.php)"', data)
if not url:
ui.error("Failed to get download form. Maybe the site changed it's design?")
return False
fields = { "fileID": "", "dlSession": "", "userID": "", "password": "", "lang": "", "submit": "" }
for field in fields:
value = matchGet('name="%s"\s+value="([^"]*)"' % field, data)
if value == False:
ui.error("Failed to get field '%s'" % field)
return False
fields[field] = value
wait = matchGet('downloadWait = ([0-9]+)', data)
if wait:
wait = int(wait)
ui.wait("Waiting %d seconds for download", wait)
target = download(url, urllib.urlencode(fields))
if target:
if getFileSize(target) > 1024*20 or "<title>Shragle" not in open(target).read():
return True
else:
os.unlink(target)
return False
class turboBitNet(handler):
NAME = "turbobit.net"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www\.)?turbobit.net/.+.html", url) != False
@staticmethod
def handle(url):
data = GET(url)
filename = matchGet("file-icon[^']+'>(.+?)</span>", data)
filepart = matchGet('.net/(.+).html', url)
murl = "http://turbobit.net/download/free/" + filepart
data = GET(murl)
waitTime = matchGet("limit: ([0-9]+),", data)
if waitTime:
waitTime = int(waitTime)
if waitTime > 180:
ui.warn("Having to wait more than 3 minutes. Restarting router.")
if not restartRouter():
ui.wait("Waiting %d seconds for download slot", waitTime)
else:
ui.wait("Waiting %d seconds for download slot", waitTime)
return turboBitNet.handle(url)
captcha = matchGet('src="(http://turbobit.net/captcha/securimg.+?)"', data)
captcha_response = ui.getCaptcha(captcha)
if not captcha_response: return False
data = POST(murl, "captcha_response=" + captcha_response)
if "captcha_response" in data:
ui.warn("Misspelled captcha!")
return turboBitNet.handle(url)
wait = matchGet('limit: ([0-9]+)', data)
if not wait:
ui.warn("Failed to get wait time. Not waiting")
else:
ui.wait("Waiting %d seconds for download", int(wait))
data = GET("http://turbobit.net/download/timeout/" + filepart)
downloadUrl = matchGet("(/download/redirect/[^']+)", data)
return download("http://turbobit.net" + downloadUrl, None, filename)
class uCms(handler):
NAME = "UCMS Redirector"
@staticmethod
def canHandle(url):
return matchGet("location=mirror$", url) != False
@staticmethod
def handle(url):
data = POST(url, "m=download")
urls = re.findall('out.php\?.*?url=([^"& ]+)', data)
if urls:
for url in urls:
addDownload(urllib2.unquote(url))
return True, handler.HANDLER_WAS_REDIRECTED
return False
class uploadBox(handler):
NAME = "uploadbox.com"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www\.)?uploadbox.com/(?:[a-z]+/)?files/[a-zA-Z0-9]+$", url) != False
@staticmethod
def handle(url):
data = POST(url, "free=yes")
if "You allready download some file" in data:
ui.warn("You are already downloading. IP conflict? Trying router reset.")
if restartRouter():
return uploadBox.handle(url)
else:
return False
captchaURL = matchGet('type it in:<img src="([^"]+)"', data)
if not captchaURL:
ui.error("Failed to get captcha from page")
return False
captchaURL = urlparse.urljoin(url, captchaURL)
fileName = matchGet('File name:</strong> ([^<]+)', data)
if not fileName:
ui.error("Failed to get filename from page")
return False
targetUrl = matchGet('action="(/files/[^"]+)"', data)
if not targetUrl:
ui.error("Failed to get download form target from page")
return False
targetUrl = urlparse.urljoin(url, targetUrl)
code = matchGet('name="code" value="([^"]+)"', data)
if not code:
ui.error("Failed to get code from page")
return False
captcha = ui.getCaptcha(captchaURL)
if not captcha:
return False
data = POST(targetUrl, "code=%s&enter=%s&go=Download" % (code, captcha), changeReferer=True)
targetUrl = matchGet('automatically within 2 seconds, please <a href="([^"]+)">click here', data)
if not targetUrl:
ui.error("Failed to download. Captcha wrong?")
return False
ui.wait("Waiting %d seconds for download", 2)
return download(targetUrl)
class uploadedTo(handler):
NAME = "uploaded.to"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www.)?(?:uploaded|ul).to/", url) != False
@staticmethod
def handle(url):
filePart = matchGet("file/([a-z0-9]+)$", url)
data = GET(url)
if not data:
ui.error("Failed to load %s" % url)
return False
if matchGet("Your Free-Traffic is exceeded", data):
wait = matchGet("wait ([0-9]+) minutes", data)
if not wait:
wait = 60
ui.warn("Exceeded download limit, you'd have to wait %s minutes. Restarting router" % wait)
if not restartRouter():
ui.warn("Failed to restart router")
ui.wait("Waiting %d seconds", 60 * int(wait))
return uploadedTo.handle(url)
POST("http://uploaded.to/io/ticket/slot/" + filePart, "")
wait = matchGet("<span>([0-9]+)</span>", data)
wait = int(wait) if wait else 14
ui.wait("Waiting %d seconds for download", wait)
# Captcha
recaptcha_response = captcha_recaptcha_api("6Lcqz78SAAAAAPgsTYF3UlGf2QFQCNuPMenuyHF3")
response = POST("http://uploaded.to/io/ticket/captcha/" + filePart, urllib.urlencode(recaptcha_response))
if response == '{err:"limit-dl"}':
ui.warn("Download limit reached.")
restartRouter()
return uploadedTo.handle(url)
downloadFile = matchGet("url:'([^']+)'", response)
target = download(downloadFile)
if target:
if getFileSize(target) > 1024*20 or "<title>" not in open(target).read():
return True
else:
os.unlink(target)
ui.error("Download failed")
return False
class uploadingCom(handler):
NAME = "uploading.com"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www.)?uploading.com/files/[a-zA-Z0-9]+/", url) != False
@staticmethod
def handle(url, tryno=0):
data = GET(url, changeReferer=True)
file_id = matchGet('name="file_id" value="([^"]+)"', data)
code = matchGet('name="code" value="([^"]+)"', data)
postData = { "action": "second_page", "file_id": file_id, "code": code }
target = matchGet('action="(http://uploading.com/files/get/[^"]+)"', data)
if "Diese IP-Adresse wird momentan schon zum Download benutzt" in data:
ui.error("Download-Limit erreicht. Starte Router neu")
if restartRouter():
return uploadingCom.handle(url)
else:
return False
if not target or not file_id:
ui.error("Failed to get download page")
return False
data = POST(target, urllib.urlencode(postData))
fileName = matchGet('<h2>([^<>]+)</h2>', data)
wait = matchGet('start_timer\(([0-9]+)', data)
if wait:
ui.wait("Waiting %d seconds", int(wait) + 1)
answer = POST("http://uploading.com/files/get/?JsHttpRequest=12603844442720-xml", "action=get_link&file_id=%s&pass=undefined&code=%s" % (file_id,code))
try:
url = json.loads(answer)["js"]["answer"]["link"]
except:
if tryno > 1:
ui.error("Failed to load JSON Data from uploading.com")
return False
return uploadingCom.handle(url, tryno + 1)
file = download(url, None, fileName)
if file and (getFileSize(file) < 1024**3 and "<title>Uploading.com" in open(file).read()):
os.unlink(file)
ui.error("Failed to download the file")
return False
return file
class xirrorCom(handler):
NAME = "xirror.com Redirector"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www\.)?xirror.com", url) != False
@staticmethod
def handle(url):
data = GET(url)
if not data:
ui.error("Failed to load %s" % url)
return False
data = urllib.unquote(data)
urls = re.findall('popup\("([^"]+)', data)
urls = map(lambda x: "".join(reversed(list(x))), urls)
# Only download from one mirror if multiple exist
if any(map(lambda x: "rapidshare" in x, urls)):
urls = filter(lambda x: "rapidshare" in x, urls)
elif any(map(lambda x: "uploaded.to" in x, urls)):
urls = filter(lambda x: "uploaded.to" in x, urls)
elif any(map(lambda x: "netload.in" in x, urls)):
urls = filter(lambda x: "netload.in" in x, urls)
foundOne = False
for url in urls:
foundOne = True
addDownload(url)
return foundOne, handler.HANDLER_WAS_REDIRECTED
class xxxblog(handler):
NAME = "xxxblog.to Redirector"
@staticmethod
def canHandle(url):
return matchGet("^http://xxx-blog.to/s2l/.+", url) != False
@staticmethod
def handle(url):
return s2l.handle("http://s2l.biz/" + matchGet("s2l/(.+)", url))
class yourLayerCom(handler):
NAME = "yourlayer.com Redirector"
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www\.)?yourlayer.com", url) != False
@staticmethod
def handle(url):
data = GET(url)
link = matchGet('<iframe[^>]+src="([^"]+)"', data)
if not link:
return False
addDownload(link)
return True, handler.HANDLER_WAS_REDIRECTED
class youtubeCom(handler):
NAME = "youtube.com"
SIMULTANEOUS_POSSIBLE = True
@staticmethod
def canHandle(url):
return matchGet("^http://(?:www.)?youtube.(?:com|de)/watch", url) != False
@staticmethod
def handle(url):
data = GET(url)
flashvars = re.sub(r"\\u([0-9]+)", lambda i: chr(int(i.group(1), 16)),
re.sub('%([0-9A-F]{2})', lambda i: chr(int(i.group(1), 16)), matchGet(r'flashvars=\\"(.+?)\\"', data))
)
flashvars = re.sub("&([a-z]+);", lambda x: chr(name2codepoint[x.group(1)]) if x.group(1) in name2codepoint
and name2codepoint[x.group(1)] < 255 else x.group(0), flashvars)
# %26itag%3D34%26
fmts = dict(map(reversed, re.findall(r'url=(.*?)&itag=(\d+)', flashvars)))
quality_wishes = [ 38, 37, 22, 45, 43, 35, 34, 18, 17 ]
extensions = { 5: "flv", 34: "flv", 35: "flv", 22: "mp4", 37: "mp4", 38: "mp4", 18: "mp4", 43: "vp8", 45: "vp8", 17: "3gp" }
use = False
for wish in quality_wishes:
if str(wish) in fmts:
quality = wish
use = fmts[str(wish)]
if not use:
return False
url = re.sub('%([0-9A-F]{2})', lambda i: chr(int(i.group(1), 16)), use)
filename = ""
filenameBuilder = matchGet('<meta name="title" content="(.+?)">', data)
filenameBuilder = re.sub("&([a-z]+);", lambda x: chr(name2codepoint[x.group(1)]) if x.group(1) in name2codepoint
and name2codepoint[x.group(1)] < 255 else x.group(0), filenameBuilder)
filenameBuilder = re.sub("&([a-z]+);", lambda x: chr(name2codepoint[x.group(1)]) if x.group(1) in name2codepoint
and name2codepoint[x.group(1)] < 255 else x.group(0), filenameBuilder)
while len(filenameBuilder) > 0:
if filenameBuilder[0] == "\\":
filenameBuilder = filenameBuilder[1:]
if filenameBuilder[0] in ("/",):
filenameBuilder = "-" + filenameBuilder[1:]
filename += filenameBuilder[0]
filenameBuilder = filenameBuilder[1:]
filename += "." + extensions[quality] if quality in extensions else "avi"
if os.system("which ffmpeg > /dev/null") != 0:
ui.info("If you install ffmpeg I'll suggest auto-conversion to mp3 to you")
convertMp3 = False
else:
convertMp3 = ui.ask("Convert to mp3?")
ok = download(url, None, filename)
if not ok: return False
if not convertMp3: return True
filename = ok
if os.system("ffmpeg -i '%s' -acodec libmp3lame -ab 196608 '%s' 2>&1 > /dev/null" % (filename.replace("'", "\\'"),
filename[:-4].replace("'", "\\'") + ".mp3")) != 0:
try: os.unlink(filename[:-4] + ".mp3")
except: pass
ui.warn("Automatic conversion failed, but you still have the mp4 file")
else:
os.unlink(filename)
return True
if __name__ == "__main__":
# Setup a nicer exception handler
def excepthook(type, value, traceback):
if type is KeyboardInterrupt or type is EOFError:
sys.exit(0)
print
print "An error occured:"
sys.__excepthook__(type, value, traceback)
sys.exit(0)
sys.excepthook = excepthook
# Parse options
optionParser = optparse.OptionParser(usage="%prog [options] [urls]",
version = "rsDownload %s\n\n" % RSDOWNLOD_VERSION +
"Supported download pages:\n - " +
"\n - ".join(
sorted(( obj.NAME for obj in globals().values()
if "NAME" in dir(obj) )))
)
optionParser.add_option("-s", "--store-done", dest="storedone",
action="store_true",
help="Store a list of downloaded files in ./.rsDownloadDone")
optionParser.add_option("-i", "--infile", dest="infile", metavar="FILE",
help="Use URLs from FILE as input")
optionParser.add_option("-u", "--update-check", dest="updateCheck", action="store_true",
help="Check if a newer version of this script is available")
optionParser.add_option("-I", "--ignore-active", dest="ignoreActive", action="store_true",
help="Ignore other process instances. Useful for asynchronous downloads through proxies")
optionParser.add_option("", "--debug", dest="debugMode", action="store_true",
help="Debug mode stores information on HTTP requests")
optionParser.add_option("", "--store-clipboard-urls", dest="clipboard", action="store_true",
help="Store URLs from clipboard in file `url'")
optionParser.add_option("-c", "--close-after-download", dest="closeAfterDownload", action="store_true",
help="Closes rsDownload after all downloads have finished.")
(options, args) = optionParser.parse_args()
closeAfterDownload = options.closeAfterDownload
if len(args) != 0:
map(fileList.append, args)
closeAfterDownload = True
handler.options = options
if options.clipboard: # {{{
try:
import gtk
assert(gtk.gdk.display_get_default() != None)
except:
ui.error("The clipboard-scanning feature requires pyGtk")
sys.exit(1)
ui.info("Scanning clipboard for URLs")
urlFile = open("url", "a")
clipboard = gtk.Clipboard()
contents = ""
while True:
while True:
new_contents = clipboard.wait_for_text()
if type(new_contents) is str and contents != new_contents:
contents = new_contents
break
time.sleep(.5)
for file in re.findall("http://[^\"\s<>]+", contents, re.I):
is_possible = False
for obj in globals().values():
if type(obj) is type and issubclass(obj, handler) and obj != handler:
if obj.canHandle(file):
is_possible = True
if is_possible:
urlFile.write(file + "\n")
urlFile.flush()
ui.info("Url detected: %s" % file)
infoSound()
# }}}
if options.updateCheck: # {{{
md5SumNewest = GET("http://www.pberndt.com/Programme/Linux/rsDownload/_version.phpc")
try:
from hashlib import md5
except:
try:
from md5 import md5
except:
ui.error("Feature not available - md5 library missing")
sys.exit(1)
if md5(open(__file__).read()).hexdigest() != md5SumNewest:
ui.info("A new version of rsDownload is available")
myPath = sys.argv[0]
if os.access(myPath, os.R_OK | os.W_OK) and ui.ask("Update to newest version?"):
try:
data = GET("http://www.pberndt.com/raw/Programme/"
"Linux/rsDownload/_download/rsDownload.py",
dispositOk=True)
assert(data != "")
open(myPath, "w").write(data)
ui.info("Update complete")
except:
ui.error("Failed to update script")
sys.exit(1)
else:
ui.info("Your version is up to date")
sys.exit(0)
# }}}
if options.debugMode: #{{{
debugInfo = {}
def wrapper(function):
def inw(*params, **kwparams):
output = function(*params, **kwparams)
debugInfo[time.strftime("[%d.%m.%Y %H:%M:%S] ") + function.func_name + "(" + `params` + `kwparams` + ")"] = output
return output
return inw
GET = wrapper(GET)
POST = wrapper(POST)
download = wrapper(download)
def outputDebug():
if not debugInfo: return
lines = 5 + len(debugInfo)
while True:
print
print "Debug Mode"
print
print "00. Exit"
for key in range(len(debugInfo.keys())):
print "%02d. %s" % (key + 1, debugInfo.keys()[key])
try:
outputWhat = int(input())
if outputWhat == 0: break
os.popen("less", "w").write(debugInfo[debugInfo.keys()[outputWhat - 1]])
print "\033[%dF\033[J" % lines,
except: break
print "\033[%dF\033[J" % lines,
import atexit
atexit.register(outputDebug)
#}}}
if options.storedone and os.access("./.rsDownloadDone", os.F_OK):
doneList = map(lambda x: x.strip(), open("./.rsDownloadDone").readlines())
# Test terminal width code
print "\0337If the program get's stuck here try using a different terminal emulator and file a bug report!",
sys.stdout.flush()
ui._getTerminalWidth()
print "\0338\033[J\r",
sys.stdout.flush()
# Run downloader
ui.setTitle()
if options.infile:
ui.info("Downloader ready. Scanning file '%s' for input" % options.infile, 2)
inFileAlreadyTried = []
elif not closeAfterDownload:
ui.info("Downloader ready. Paste your files here.", 2)
while True:
while len(fileList) == 0:
file = None
if options.infile:
if os.access(options.infile, os.R_OK):
newFiles = filter(lambda x: x.strip() not in inFileAlreadyTried and x.strip(), open(options.infile).readlines())
if newFiles:
file = newFiles[0].strip()
inFileAlreadyTried.append(file)
elif closeAfterDownload:
sys.exit(0)
elif closeAfterDownload:
sys.exit(0)
if not file:
time.sleep(1)
else:
file = raw_input().strip()
if file:
fileList.append(file)
file = fileList.pop(0).strip()
if "://" not in file:
continue
if file in doneList:
ui.info("Already downloaded %s" % file)
if closeAfterDownload and len(fileList) == 0 and not options.infile: break
continue
success = False
skipThis = False
for obj in globals().values():
if type(obj) is type and issubclass(obj, handler) and obj != handler:
if obj.canHandle(file):
dlIsOk, problem = testAndSetIfInProgress(file, obj.NAME)
if not dlIsOk:
if problem == 0:
ui.info("Another instance is already downloading %s (using %s)" % (file, obj.NAME))
skipThis = True
break
elif obj.SIMULTANEOUS_POSSIBLE:
pass
else:
ui.info("Already downloading from %s, skipping file" % obj.NAME)
skipThis = True
break
ui.info("Using %s for %s" % (obj.NAME, file), 2)
response = obj.handle(file)
testAndSetIfInProgress(file, None, True)
flags = 0
if type(response) is tuple:
success = response[0]
if len(response) > 1:
flags = response[1]
else:
success = response
if success:
break
if skipThis:
if closeAfterDownload and len(fileList) == 0 and not options.infile: break
continue
if not success:
ui.error("Failed to download %s" % file)
errorHook()
else:
if not flags & handler.HANDLER_WAS_REDIRECTED:
ui.info("Download of %s complete." % file, 2)
doneList.append(file)
doneHook(flags)
# Store done list
if options.storedone:
if os.access("./.rsDownloadDone", os.R_OK):
file = open("./.rsDownloadDone", "r+")
else:
file = open("./.rsDownloadDone", "w+")
fcntl.lockf(file, fcntl.LOCK_SH)
old_doneList = map(lambda x: x.strip(), file.readlines())
map(doneList.append, filter(lambda x: x not in doneList, old_doneList))
file.seek(0)
file.truncate()
file.write("\n".join(doneList))
file.close()
# Clear cookie jar after download
cookieJar.clear()
useReferer = ""
# Quit if closeAfterDownload is enabled
if len(fileList) == 0 and not options.infile and closeAfterDownload:
sys.exit(0 if success else 1)
Download
- Dateiname
- rsDownload.py
- Größe
- 82.23kb
- Copyright (c) Phillip Berndt, 2006-2013
- Letztes Update 30.10.2013
