#!/usr/bin/python
# vim:fileencoding=utf-8:ft=python
#
# Download files from filehosters
#
# For end-users:
#  PyGTK is needed for captchas. If PyGTK is not present, this script will use
#  ImageMagick's "display" command to show you the captcha image.
#
# Configuration:
#  Edit the section "configuration"
#  If you've got no clue about python: Simply insert your own commands
#  between the parentheses of the system("") command. They'll be executed when
#  the specific event (def XXX(..):) occurs.
#
# For developers:
#  See the classes (ie. rapidShare). Each class has a canHandle function to
#  check if the class can handle an URL and a handle function to actually
#  download the file. Those functions should return True when the download was
#  successfull. Note that all functions are static!
#
# 
import urllib2
import urllib
import urlparse
import re
import os
import threading
import sys
import random
import time
import getopt
import cookielib
import optparse
import termios
import tty
import fcntl
import select
from htmlentitydefs import name2codepoint

## CONFIGURATION {{{ ###########################################################
def restartRouter():
	"""
		Restart router to get a new IP
	"""
	os.system("routerReset")
	return True
def doneHook(flags):
	"""
		Called when a download is finished
		See handler.HANDLER_* for values of flags
	"""
	return
	if not flags & handler.HANDLER_WAS_REDIRECTED:
		os.system("aplay -q /usr/share/sounds/k3b_success1.wav 2>&1 >/dev/null &")
def errorHook():
	"""
		Called when an error occurred
	"""
	return
	os.system("aplay -q /usr/share/sounds/k3b_error1.wav 2>&1 >/dev/null &")
def captchaHook(data, caller):
	"""
		Called when a captcha is to be deciphered
		data holds the raw data of the image!
		Use caller to check which handler is calling this
		Return False or the code
	"""
	# Anticaptcha does not work ATM
	if False and caller == "rapidshare" and os.system("which anticaptcha 2>&1 >/dev/null") == 0:
		# Call anticaptcha
		fileName = os.path.abspath(impFilename("captcha.jpg"))
		file = open(fileName, "w")
		file.write(data)
		file.close()

		try:
			captcha = os.popen("anticaptcha %r --method 22a" % fileName).read().strip()
		except:
			os.unlink(fileName)
			return False
		os.unlink(fileName)
		if len(captcha) != 4:
			return False
		ui.info("Anticaptcha detected captcha successfully: %s" % captcha)
		return captcha
	else:
		# Play sound to inform the user that interaction is needed
		os.system("aplay -q /usr/share/sounds/k3b_wait_media1.wav 2>&1 >/dev/null &")
	return False

# Do not edit from here on
RSDOWNLOD_VERSION = "1.0"

# Install cookie handler for urllib2 {{{
useReferer = ""
class HTTPCookieAndRefererProcessor(urllib2.BaseHandler):
	def __init__(self, cookiejar=None):
		import cookielib
		if cookiejar is None:
			cookiejar = cookielib.CookieJar()
		self.cookiejar = cookiejar
	def http_request(self, request):
		global useReferer
		if useReferer and not request.has_header("Referer"):
			request.add_header("Referer", useReferer)
		self.cookiejar.add_cookie_header(request)
		return request
	def http_response(self, request, response):
		self.cookiejar.extract_cookies(response, request)
		return response
	https_request = http_request
	https_response = http_response

cookieJar = cookielib.CookieJar()
opener = urllib2.build_opener(HTTPCookieAndRefererProcessor(cookieJar))
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
urllib2.install_opener(opener)
# }}}

# Initialize file list
fileList = []
doneList = []
## }}} #########################################################################
## USER INTERFACE {{{ ##########################################################
class ui(object):
	hasTermWidth = True

	_COLOR = {
		"error": "\033[00;31m",
		"info": "\033[00;32m",
		"warn": "\033[00;33m",
		"info2": "\033[00;34m",
		"def": "\033[0m"
	}

	@staticmethod
	def _getTerminalWidth():
		tsize = 80
		if ui.hasTermWidth and sys.stdin.isatty():
			oldSettings = termios.tcgetattr(sys.stdin.fileno())
			tty.setraw(sys.stdin.fileno())
			mfcntl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
			fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, os.O_NONBLOCK | mfcntl)
			print "\033[18t",
			sys.stdout.flush()
			size = ""
			iter = 0
			while "t" not in size:
				fp, foo, foo = select.select([sys.stdin], [], [], 0.2)
				if sys.stdin in fp:
					size += sys.stdin.read(10)
				else:
					iter += 1
					if iter > 10:
						break
			if "t" in size:
				tsize = int(size[size.find(";") + 1:-1])
			else:
				ui.hasTermWidth = False
			fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, mfcntl)
			termios.tcsetattr(sys.stdin.fileno(), termios.TCSADRAIN, oldSettings)
		return tsize

	@staticmethod
	def endProgress():
		"""
			Erase the current console line
			(End progress())
		"""
		print "\033[2K\r", # Erase line
		#print "\033[1A", # Move line up
		#print "\033[2K\r", # Erase line
		sys.stdout.flush()

	_progressData = {}
	@staticmethod
	def startProgress():
		"""
			Start progress()
		"""
		ui._progressData = { 
			'start': time.time(),
			'last':  time.time(),
			'data':  0,
			'lastDsp': time.time() - 10
			}

	@staticmethod
	def _formatSize(size):
		units = [ "b", "Kb", "Mb", "Gb" ]
		ret = units.pop(0)
		while size > 1024 and len(units) > 0:
			size /= 1024.0
			ret = units.pop(0)
		return "%03.2f%s" % (size, ret)

	@staticmethod
	def _formatTime(seconds):
		hours = int(seconds / 3600)
		seconds %= 3600
		minutes = int(seconds / 60)
		seconds %= 60
		return "%02d:%02d:%02d" % (hours, minutes, seconds)

	@staticmethod
	def progress(done, length):
		"""
			Diplay a progress message
		"""
		timeNow     = time.time()
		timeElapsed = timeNow - ui._progressData["start"]
		if timeElapsed == 0:
			avgSpeed = 0
		else:
			avgSpeed = done * 1.0 / timeElapsed

		timeLast = ui._progressData["last"]
		if timeNow - timeLast == 0:
			speed = 0
		else:
			speed       = (done - ui._progressData["data"]) * 1.0 / (timeNow - timeLast)
		ui._progressData["last"] = time.time()
		ui._progressData["data"] = done
		if length > 0:
			pdone  = done * 1.0 / length
			timeToGo = 0
			if pdone > 0:
				timeToGo = (timeElapsed / pdone) - timeElapsed
			info = "  %02d%% %s/%s %8s/s %s" % (pdone * 100, ui._formatSize(done),
				ui._formatSize(length), ui._formatSize(speed),
				ui._formatTime(timeToGo))
		if sys.stdout.isatty():
			if ui._progressData["lastDsp"] + 0.5 > timeNow:
				return
			ui._progressData["lastDsp"] = timeNow

			print "\033[2K\r", # Erase line
			if length > 0:
				barWidth = ui._getTerminalWidth() - len(info) - 5 - 2
				barFilled = int(barWidth * pdone)
				bar = ""
				if barFilled > 0:
					bar = "-" * (barFilled - 1) + ">"
				bar += " " * (barWidth - len(bar))
				bar = "[%s]" % bar
				print " %s%s" % (bar, info),
			else:
				print " %s downloaded, %8s/s" % (ui._formatSize(done),
					ui._formatSize(speed)),
		else:
			if ui._progressData["lastDsp"] + 5 < timeNow:
				ui._progressData["lastDsp"] = timeNow
				if length > 0:
					print info
				else:
					print " %s downloaded, %8s/s" % (ui._formatSize(done),
						ui._formatSize(speed))
		sys.stdout.flush()

	@staticmethod
	def _gt(color):
		"""
			Return a colored ">>>" string
		"""
		if sys.stdout.isatty():
			return "%s>>>%s" % (ui._COLOR[color], ui._COLOR["def"])
		else:
			return ">>>"

	@staticmethod
	def error(message):
		"""
			Output error message
		"""
		print >> sys.stderr, ui._gt("error"), message

	@staticmethod
	def info(message, type=1):
		"""
			Output info message
			Don't use the type parameter, it's meant for use in
			this script's main function only!
		"""
		if type == 1:
			print ui._gt("info"), message
		else:
			print ui._gt("info2"), message

	@staticmethod
	def warn(message):
		"""
			Output warning message
		"""
		print ui._gt("warn"), message

	@staticmethod
	def wait(message, seconds):
		"""
			Wait for something. Message should contain a %d (which
			will be replaced by the remaining seconds)
		"""
		if sys.stdout.isatty():
			while seconds > 0:
				print "\033[2K\r", # Erase line
				print ui._gt("warn"), message % seconds,
				sys.stdout.flush()
				seconds -= 1
				time.sleep(1)
			print "\033[2K\r", # Erase line
		else:
			print ui._gt("warn"), message % seconds
			sys.stdout.flush()
			while seconds > 0:
				seconds -= 1
				time.sleep(1)
		sys.stdout.flush()

	@staticmethod
	def getCaptcha(url, caller=None):
		"""
			Decipher the captcha from the given URL.
			Will execute the function captchaHook or, if it fails,
			display a GTK+ dialog, asking for the code (with
			console / imagemagick as a fallback)
		"""
		try:
			dataObject = urllib2.urlopen(url)
			data = dataObject.read()
		except:
			ui.error("Failed to load captcha ")
			return False
	
		# Try hook first
		response = captchaHook(data, caller)
		if response:
			return response

		# Then try GTK+
		try:
			import gtk
			loader = gtk.gdk.PixbufLoader()
			try:
				loader.write(data)
				loader.close()
			except:
				ui.error("Failed to load captcha ")
				return False
			dialog = gtk.Dialog(u"Get captcha", buttons=(gtk.STOCK_OK, gtk.RESPONSE_ACCEPT))
			image = gtk.Image()
			image.set_from_pixbuf(loader.get_pixbuf())
			dialog.vbox.add(image)
			entry = gtk.Entry()
			def returnOnReturn(widget, event):
				if event.keyval == gtk.keysyms.Return:
					dialog.response(gtk.RESPONSE_ACCEPT)
			entry.connect("key-press-event", returnOnReturn)
			dialog.vbox.add(entry)
			dialog.show_all()
			dialog.run()
			dialog.hide()
			gtk.main_iteration()
			gtk.main_iteration()
			if entry.get_text() == "":
				return False
			return entry.get_text()
		except:
			return False

		# Try console finally
		ui.info("Please enter the captcha from the following image")
		display = os.popen("display -", "w")
		display.write(data)
		display.close()
		return raw_input(" Captcha: ").strip()
## }}} #########################################################################
## BASE CLASSES {{{ ############################################################
class handler(object):
	HANDLER_WAS_REDIRECTED = 2 ** 0
	options = {} # Will be set by __main__

	NAME = "Base handler"
	@staticmethod
	def canHandle(url):
		"""
			Return if this handler is able to handle
			this url
		"""
		pass

	@staticmethod
	def handle(url):
		"""
			Handles an URL
			May append files to the global download list
			Returns either
			 - True:  Successful download
			 - False: Download failed
			 - bool, bitmask: 
			          Additional information, see constants above
		"""
		pass
## }}} #########################################################################
## HELPER FUNCTIONS {{{ ########################################################
def matchGet(rex, string):
	"""
		Match regular expresion rex on string
		Returns
		 - False if rex does not match anywhere in string
		 - The matched part of string if rex does not contain
		   parentheses
		 - The first group if only one group exists
		 - A touple with all group values elsewise
	"""
	match = re.search(rex, string)
	if match:
		if len(match.groups()) == 0:
			return string[match.span()[0]:match.span()[1]]
		if len(match.groups()) == 1:
			return match.groups()[0]
		else:
			return match.groups()
	return False

class WouldDispositException(Exception):
	pass
def GET(url, changeReferer=False):
	"""
		Place a GET request on URL, return data

		changeReferer will store the URL and use it as a referer header
		for all further requests (during this download)
	"""
	try:
		dataObject = urllib2.urlopen(url)
		if "Content-Disposition" in dataObject.info():
			raise WouldDispositException()
		if changeReferer:
			global useReferer
			useReferer = url
		return dataObject.read()
	except WouldDispositException:
		raise WouldDispositException()
	except:
		ui.error("Failed to download %s" % url)
		return False
	
def POST(url, data, changeReferer=False):
	"""
		Place a POST request with data on URL,
		return data

		changeReferer will store the URL and use it as a referer header
		for all further requests (during this download)
	"""
	try:
		if changeReferer:
			global useReferer
			useReferer = url
		return urllib2.urlopen(url, data).read()
	except:
		ui.error("Failed to download %s" % url)
		return False

def addDownload(url, redirected=False):
	"""
		Add a download to the download list
	"""
	global fileList

	fileList.insert(0, url)
	if not redirected:
		ui.info("Added new download %s" % url)

def download(url, postData = None, targetFileName = None):
	"""
		Download url to targetFileName (will be extracted from url /
		the content-disposition header if omitted)
		Returns False on failure and the file name elsewise.
	"""
	filename = False
	targetFile = None
	targetFileName = None
	if 1: #try:
		request = urllib2.Request(url)
		if postData:
			request.add_data(postData)
		urlobject = urllib2.urlopen(request)

		headers = urlobject.info()
		if "Content-Disposition" in headers:
			dispositionHeader = headers["Content-Disposition"]
			targetFileName = matchGet("filename=(.)(.+)\\1$", dispositionHeader)
			if targetFileName:
				targetFileName = targetFileName[1]
		if not targetFileName:
			targetFileName = os.path.basename(urlobject.geturl())
			if "?" in targetFileName:
				targetFileName = targetFileName[:targetFileName.index("?")]
		targetFileName = impFilename(targetFileName)
		targetFile = open(targetFileName, "w")
		if "Content-Length" in headers:
			fileLength = int(headers["Content-Length"])
		else:
			fileLength = -1

		ui.startProgress()
		size = 0
		while True:
			data = urlobject.read(1024 * 8)
			if not data:
				break
			size += len(data)
			targetFile.write(data)
			ui.progress(size, fileLength)
		ui.endProgress()
		del targetFile
		return targetFileName
	#except:
		try:
			if targetFile:
				del targetFile
			if targetFileName and os.access(targetFileName, os.F_OK):
				os.unlink(targetFileName)
		except:
			pass
		print
		ui.error("Download failed: %s" % sys.exc_info()[1])
		return False

def impFilename(fileName):
	"""
		Return fileName or, if this file does already exists,
		fileName~nnn.
	"""
	i = 0
	if os.access(fileName, os.F_OK):
		fileName = "%s~%%d" % fileName
		while os.access(fileName % i, os.F_OK):
			i += 1
		fileName = fileName % i
	return fileName

def getFileSize(name):
	return os.stat(name)[6]

def htmlEntityDecode(text):
	"""
		Decode HTML entities
	"""
	def entSubst(arg):
		if arg.group(1) == "#":
			return unichr(int(arg.group(2)))
		else:
			cp = name2codepoint.get(arg.group(2))
			if cp:
				cp = unichr(cp)
			else:
				cp = arg.group(2)
			return cp
	return re.sub("&(#?)(\d{1,5}|\w{1,8});", entSubst, text)

def getScriptPath():
	return os.path.abspath(os.path.dirname(sys.argv[0]))
## }}} #########################################################################

## BLOG REDIRECTOR {{{ #########################################################
class blogRedirector(handler):
	NAME = "Blog Redirector Service"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://[^/]+/(?:nl|ff|rc|ul).+", url) != False

	@staticmethod
	def handle(url):
		page = GET(url)
		frame = matchGet('src="(.+?)"', page)

		try:
			redirector = urllib2.urlopen(urlparse.urljoin(url, frame))
			redirectUrl = redirector.geturl()
			redirector.close()

			if "error_traffic_exceeded_free" in redirectUrl:
				ui.warn("Download limit exceeded. Restarting router...")
				routerReset()
				return blogRedirector.handle(url)
		except:
			return False

		if url:
			addDownload(redirectUrl, True)
			return True, handler.HANDLER_WAS_REDIRECTED
		return False
## }}} #########################################################################
## EASY-SHARE.COM {{{ ##########################################################
class easyShareCom(handler):
	NAME = "easy-share.com"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(?:www\.|w[0-9]+\.)?easy-share.com/[0-9]+", url) != False

	@staticmethod
	def handle(url):
		data = GET(url, changeReferer=True)
		if not 'src="/kaptcha' in data:
			wait = matchGet("w='([0-9]+)'", data)
			captchaAjax = matchGet("u='([^']+)'", data)
			if not wait or not captchaAjax:
				print data
				ui.error("Failed to get download information")
				return False
			ui.wait("Waiting %d seconds for download", int(wait))
			data = GET(urlparse.urljoin(url, captchaAjax))

		downloadUrl = matchGet('action="([^"]+)"', data)
		captcha = matchGet('img src="(/kaptcha[^"]+)', data)
		if not downloadUrl or not captcha:
			ui.error("Failed to get download information from AJAX form")
			return False
		captcha = urlparse.urljoin(url, captcha)
		print captcha
		form = { 'id': matchGet('name="id" value="([^"]+)"', data) }
		form["captcha"] = ui.getCaptcha(captcha)

		fileName = download(downloadUrl, urllib.urlencode(form))
		if fileName and getFileSize(fileName) < 1024*25:
			content = open(fileName).read()
			if "and earn money.</title>" in content:
				ui.error("Download failed. Captcha wrong?")
				os.unlink(fileName)
				return False
		return fileName

## }}} #########################################################################
## LETITBIT.NET {{{ ############################################################
class letItBitNet(handler):
	NAME = "letitbit.net"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(?:www\.)?letitbit.net/download/", url) != False

	@staticmethod
	def handle(url):
		data = GET(url)
		form = {}
		form["uid"] = matchGet('name="uid" value="([^"]{13}[^"]+)"', data)
		form["frameset"] = "Download file"
		form["fix"] = 1
		data = POST("http://letitbit.net/download3.php", urllib.urlencode(form))
		url2 = matchGet('src="(/tmpl/tmpl_frame_top[^"]+)', data)
		if not url2:
			ui.error("Page syntax error")
			return False
		data = GET('http://letitbit.net%s' % url2)

		downloadUrl = matchGet(
			'<a href="(http://[a-z0-9]+\.letitbit.net/download[0-9]+/[^<"]+)"', data)
		if not downloadUrl:
			ui.error("Failed to get download url")
			return False

		wait = matchGet('name="errt">([0-9]+)<', data)
		if wait:
			ui.wait("Waiting %d seconds for download", int(wait))


		return download(downloadUrl)

## }}} #########################################################################
## FILES.AG {{{ ################################################################
class filesAg(handler):
	NAME = "files.ag"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(?:www\.)?files.ag/files/[0-9]+/.+", url) != False

	@staticmethod
	def handle(url):
		data = POST(url, "start=Free")
		downloadUrl = matchGet('action="([^"]+)"', data)
		if not downloadUrl:
			ui.error("Failed to get download url")
			return False
		wait = matchGet('download-timeout">([^<]+)<', data)
		if wait:
			ui.wait("Waiting %d seconds for download slot", int(wait))
		return download("http://files.ag" + downloadUrl)

## }}} #########################################################################
## ANONYM.TO {{{ ###############################################################
class anonymTo(handler):
	NAME = "Anonym.To Redirector"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(?:www\.)?anonym.to", url) != False

	@staticmethod
	def handle(url):
		match = matchGet("^http://(?:www\.)?anonym.to/\?(.+)$", url)
		if match:
			addDownload(urllib.unquote(match), True)
			return True, handler.HANDLER_WAS_REDIRECTED
		return False
## }}} #########################################################################
## DEPOSITFILES.COM {{{ ########################################################
class depositfilesCom(handler):
	NAME = "depositfiles.com"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(?:www\.)?depositfiles.com", url) != False

	@staticmethod
	def handle(url):
		data = GET(url)
		target = matchGet('<form[^>]+action="(.+)" [^>]+ id="gateway_form"', data)
		if not target:
			ui.error('Failed to get download form')
			return False
		data = POST("http://depositfiles.com" + target, "gateway_result=1")
		wait = int(matchGet('id="download_waiter_remain">([^<]+)<\/', data))

		formData = {}
		for field in ( "file_password", "gateway_result", "icid", "go" ):
			formData[field] = matchGet('name="%s" value="([^"]+)"' % field, data)
			if formData[field] == None:
				ui.error("Failed to get download field '%s'" % field)
				return False
		if matchGet("var img_code_url = '([^']+)'", data):
			captcha = "http://depositfiles.com" + matchGet("var img_code_url = '([^']+)'", data) + "&rnd=1211011667"
			formData["img_code"] = ui.getCaptcha(captcha, "depositfilesCom")

		ui.wait("Waiting %d seconds for download", wait + 2)

		alt = matchGet('dwnsrc = "([^"]+)"', data)
		if alt:
			return download(alt)
		else:
			return download("http://depositfiles.com" + target, urllib.urlencode(formData))

## }}} #########################################################################
## BINLOAD.TO {{{ ##############################################################
class binloadTo(handler):
	NAME = "binload.to"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(www\.)?binload.to/file/[a-z0-9]+", url) != False

	@staticmethod
	def handle(url):
		data = GET(url)
		if not data:
			ui.error("Failed to load %s" % url)
			return False
		urlData = matchGet('action="(http://[^.]+.binload.to/download.php[^"]+)"', data)
		dlSession = matchGet('dlSession"\s+value="([^"]+)"', data)
		fileName = matchGet('Download: (.+?)<\/h3>', data)
		if not urlData or not fileName or not dlSession:
			ui.error("Failed to get download link from binload.to page")
			return False
		wait = matchGet('cdLength = ([0-9]+)', data)
		if wait:
			# Ugly but needed - netload adds some time to the time given in the JS oO
			ui.wait("Waiting %d seconds for download ticket.", int(wait) * 2)
		postData = "dlSession=%s&down=Download&login_nickname=&login_passwort=" % dlSession
		downloadedFile = download(urlData, postData, fileName)
		if not downloadedFile:
			return False
		if getFileSize(downloadedFile) < 1024 ** 2:
			data = open(downloadedFile).read()
			msg  = matchGet('^<script[^(]+\("([^"]+)"\);', data)
			if msg:
				ui.error("Error while downloading: %s" % msg)
				os.unlink(downloadedFile)
				return False
		return True
## }}} #########################################################################
## FASTLOAD.NET {{{ ############################################################
class fastloadNet(handler):
	NAME = "fast-load.net"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(www\.)?fast-load.net/", url) != False

	@staticmethod
	def handle(url):
		data = GET(url)
		if not data:
			ui.error("Failed to load %s" % url)
			return False
		fid = matchGet('name="fid"\s+value="([^"]+)"', data)
		file = matchGet('<font style="font-color:grey; font-size:8pt;">([^<]+)<\/font>', data)

		if not fid:
			ui.error("Failed to get download link")
			return False

		captcha = ui.getCaptcha("http://www.fast-load.net//includes/captcha.php", "fastloadNet")
		if not captcha:
			return False

		downloadedFile = download("http://www.fast-load.net//download.php", "fid=%s&captcha_code=%s" % (fid, captcha), file)
		if not downloadedFile:
			return False
		if getFileSize(downloadedFile) < 1024 ** 2:
			data = open(downloadedFile).read()
			msg  = matchGet('^wrong captcha', data)
			ui.error("Wrong captcha")
			return False
		return True
## }}} #########################################################################
## DATENSCHLEUDER.CC {{{ #######################################################
class datenschleuderCC(handler):
	NAME = "datenschleuder.cc Redirector"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(?:www\.)?datenschleuder.cc", url) != False

	@staticmethod
	def handle(url):
		data = GET(url)
		if not data:
			ui.error("Failed to load %s" % url)
			return False
		if handler.options.prefer == "rapidshare":
			mirror = "rapidshare"
			dlUrl = 'http://rapidshare.com/[^"]+'
		elif handler.options.prefer == "uploaded.to":
			mirror = "uploaded"
			dlUrl = 'http://(?:www\.)?uploaded.to/[^"]+'
		elif handler.options.prefer == "netload.in":
			mirror = "netload"
			dlUrl = 'http://netload.in/[^"]+'
		redir = matchGet('http://www.datenschleuder.cc/redir.php\?\S+mirror=%s[^"]+' % mirror,
			data)
		if not redir:
			ui.error("Failed to get download frame link")
			return False
		data = GET(redir)
		if not data:
			ui.error("Failed to load %s" % redir)
			return False
		redir = matchGet(dlUrl, data)
		if not redir:
			ui.error("Failed to get download link")
			return False
		addDownload(redir)
		return True, handler.HANDLER_WAS_REDIRECTED
## }}} #########################################################################
## FREE-CLIPS.CH {{{ ###########################################################
class freeClipsCh(handler):
	NAME = "free-clips.ch Redirector"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(?:www.)?free-clips.ch/", url) != False

	@staticmethod
	def handle(url):
		data = GET(url)
		links = matchGet('<textarea name="links" id="links" '
			'readonly="readonly">([^<]+)<\/textarea>', data)
		if not links:
			ui.error("Failed to get download links")
		for link in links.split("\n"):
			addDownload(htmlEntityDecode(link.strip()))
		if links:
			return True, handler.HANDLER_WAS_REDIRECTED
		else:
			return False
## }}} #########################################################################
## LINK-SAFE.TO-CLAN.DE {{{ ####################################################
class linkSafeToClan(handler):
	NAME = "link-safe.to-clan.de Redirector"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(?:www.)?link-safe.to-clan.de/", url) != False

	@staticmethod
	def handle(url):
		data = GET(url)
		iframe = matchGet('<iframe[^>]+src="([^"]+)"', data)
		if not iframe:
			ui.error("Failed to get linked page")
			return False
		addDownload(iframe, True)
		return True, handler.HANDLER_WAS_REDIRECTED
## }}} #########################################################################
## LINKEDMONEY.COM {{{ #########################################################
class linkedMoney(handler):
	NAME = "linkedmoney.com Redirector"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(?:www\.)?linkedmoney.com", url) != False

	@staticmethod
	def handle(url):
		data = GET(url)
		url = matchGet('<iframe id="linkedmoney_frame"[^>]+src="([^"]+)"', data)
		if url:
			addDownload(htmlEntityDecode(url), True)
			return True, handler.HANDLER_WAS_REDIRECTED
		return False
## }}} #########################################################################
## MEGAUPLOAD.COM / MEGA*.COM ############################################### {{{
class megauploadCom(handler):
	NAME = "megaupload.com"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(www\.)?megaupload.com", url) != False

	@staticmethod
	def handle(url):
		data = GET(url)
		if not data:
			ui.error("Failed to load %s" % url)
			return False
		formFields = {}

		for field in [ "d", "imagecode", "megavar" ]:
			formFields[field] = matchGet('name="%s"\s+value="([^"]+)"' % field, data)
			if not formFields[field]:
				ui.error("Failed to get field %s from page" % field)
				return False
		fileName = matchGet("<b>(?:Dateiname|Filename):</b>\s*(.*?)<", data)
		captchaURL = matchGet('src="(/capgen.php[^"]+)"', data)
		formFields["imagestring"] = ui.getCaptcha(urlparse.urljoin(url, captchaURL),
			"megaupload")
		if not formFields["imagestring"]:
			ui.error("Failed to get captcha")
			return False
		data = POST(urlparse.urljoin(url, "/"), urllib.urlencode(formFields))

		field1 = matchGet("var . = String.fromCharCode\(Math.abs\(([-0-9]+)", data)
		field2 = matchGet("var . = '(.)' \+ String.fromCharCode\(Math.sqrt\(([0-9]+)", data)
		durl = matchGet('href="(http://www[0-9]+\.mega\w+.com/[^\'"]+)\'[^\'"]+\'([^"]+)"',
			data)
		try:
			durl = durl[0] + field2[0] + chr(int(int(field2[1]) ** 0.5)) + \
				chr(abs(int(field1))) + durl[1]
		except:
			ui.error("Failed to reconstruct download URL")
			ui.warn("Either the page changed it's design or you entered the captcha wrong")
			return False
		ui.wait("Waiting %2d seconds for download ticket", 45)
		return download(durl, targetFileName=fileName) != False
class megaroticCom(megauploadCom):
	NAME = "megarotic.com"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(www\.)?megarotic.com", url) != False
## }}} #########################################################################
## NETFOLDER.IN {{{ ############################################################
class netfolderIn(handler):
	NAME = "netfolder.in Redirector"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(?:www.)?netfolder.in/", url) != False

	@staticmethod
	def handle(url):
		data = GET(url)
		if not data:
			ui.error("Failed to load %s" % url)
			return False
		urls = map(htmlEntityDecode, re.findall('href="(http://netload.in/[^"]+)', data))
		if urls:
			for url in urls:
				addDownload(url)
			return True, handler.HANDLER_WAS_REDIRECTED
		else:
			return False
## }}} #########################################################################
## NETLOAD.IN {{{ ##############################################################
class netloadIn(handler):
	NAME = "netload.in"
	USED_BEFORE = False

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(?:www.)?netload.in/", url) != False

	@staticmethod
	def handle(url):
		if netloadIn.USED_BEFORE == True:
			restartRouter()
			netloadIn.USED_BEFORE = False

		data = GET(url)
		if not data:
			ui.error("Failed to load %s" % url)
			return False
		# Step 1: Get captcha page
		nurl = matchGet('href="(index.php\?[^"]+captcha=1)"', data)
		if nurl:
			if not nurl:
				ui.error("Failed to get captcha page url")
				return False
			nurl = "http://netload.in/" + htmlEntityDecode(nurl)
			data = GET(nurl)
			if not data:
				ui.error("Failed to load %s" % nurl)
				return False
		# Step 2: Get captcha
		curl = matchGet('src="(share/includes/captcha.php[^"]+)"', data)
		if not curl:
			ui.error("Failed to get captcha url")
			return False
		captcha = ui.getCaptcha("http://netload.in/" + htmlEntityDecode(curl), "netload")
		if not captcha:
			ui.error("Failed to get captcha")
			return False
		# Step 3: Countdown
		wait = matchGet("countdown\(([0-9]+)", data)
		# Step 4: Door page
		faction = matchGet('action="(index.php\?[^"]+)"', data)
		fileId = matchGet('name="file_id"[^>]+value="([^"]+)"', data)
		if not faction or not fileId:
			ui.error("Failed to get download form data")
			return False
		faction = "http://netload.in/" + urllib2.unquote(faction)
		postData = "file_id=%s&captcha_check=%s&start=" % (fileId, urllib.quote(captcha))
		data = POST(faction, postData)
		# Step 5: Countdown
		wait = matchGet("countdown\(([0-9]+)", data)
		if wait:
			ui.wait("Waiting %2d seconds for download ticket.", int(wait) / 100)
		# Step 6: Download
		downloadUrl = matchGet('class="Orange_Link"\s+href="([^"]+)"', data)
		if not downloadUrl:
			ui.error("Failed to get download url.")
			ui.warn("Did you enter the correct captcha code?!")
			ui.warn("Consider restarting your router")
			return False
		netloadIn.USED_BEFORE = True
		return download(htmlEntityDecode(downloadUrl)) != False
## }}} #########################################################################
## RAPID SHARE {{{ #############################################################
class rapidShare(handler):
	NAME = "Rapidshare"
	USED_BEFORE = False

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(?:www\.|rs[0-9]+\.)?rapidshare.com", url) != False

	@staticmethod
	def handle(url):
		data = GET(url)
		ERRORS = ( "The file could not be found",
			"The uploader has removed this file from the server" )
		for err in ERRORS:
			if err in data:
				ui.error(err)
				return False
		url = matchGet('<form[^>]+action="([^"]+)', data)
		if not url:
			ui.error("Failed to get download page url")
			return False
		data = POST(url, "dl.start=Free")

		if "You have reached the download limit for free-users" in data:
			stime = matchGet('Or try again in about ([0-9]+) minutes', data)
			ui.warn("You'd have to wait %s minutes" % stime)
			ui.warn("Download limit reached. Restarting router.")
			if not restartRouter():
				ui.warn("Router restart failed.")
				ui.wait("Waiting %d seconds", int(stime) * 60)
			return rapidShare.handle(url)

		seconds = matchGet('var c=([0-9]+)', data)
		if not seconds:
			ui.error("Failed to get security data from page")
			return False
		seconds = time.time() + int(seconds)
		if seconds - time.time() > 0:
			ui.wait("Waiting %d seconds for download", seconds - time.time())

		dlTarget = matchGet('<form[^>]+name="dlf?" action="([^"]+)', data)
		file = download(dlTarget)

		if file and getFileSize(file) < 10*1024:
			content = open(file).read()
			if "Wrong access code" in content:
				os.unlink(file)
				ui.warn("Captcha code wrong")
				return rapidShare.handle(url)
		return file
		
## }}} #########################################################################
## RAPIDLAYER.IN {{{ ###########################################################
class rapidLayerIn(handler):
	NAME = "Rapidlayer.In Redirector"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(?:www\.)?rapidlayer.in", url) != False

	@staticmethod
	def handle(url):
		data = GET(url)
		if not data:
			ui.error("Failed to download %s" % url)
			return False
		urlEncoded = matchGet('}[a-z]="([^"]+)"', data)
		sub = matchGet("([a-z])=\\1-([0-9]+)", data)
		if not sub:
			return False
		sub = int(sub[1])
		def decode(schar):
			char = ord(schar)
			if char < 128:
				char -= sub
				if char < 32:
					char = 127 + (char - 32)
			return chr(char)
		url = "".join(map(decode, urlEncoded))
		if url:
			url = htmlEntityDecode(url)
			addDownload(urllib.unquote(url), True)
			return True, handler.HANDLER_WAS_REDIRECTED
		return False
## }}} #########################################################################
## RELINK.US {{{ ###############################################################
class relinkUs(handler):
	NAME = "Relink.us Redirector"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(?:www\.)?relink.us", url) != False

	@staticmethod
	def handle(url):
		data = GET(url)
		links = re.findall("action='(go.php[^']+)", data)
		if not links:
			return False
		for link in links:
			data = POST("http://relink.us/" + link, "")
			nurl = matchGet('<iframe[^>]+src="([^"]+)"', data)
			if nurl:
				addDownload(nurl, True)
			else:
				ui.error("Failed to get URL for at least one file: %s" % nurl)
		return True, handler.HANDLER_WAS_REDIRECTED
## }}} #########################################################################
## SECURED.IN {{{ ##############################################################
class securedIn(handler):
	NAME = "Secured.In Redirector"

	@staticmethod
	def canHandle(url):
		try:
			import gtkmozembed
			hasEmbed = True
		except:
			hasEmbed = False
			
		ok = matchGet("^http://(?:www\.)?secured.in", url) != False
		if ok and hasEmbed:
			return True
		elif ok:
			ui.error("You need gtkmozembed (gnome-python extras) for this to work")
		return False

	@staticmethod
	def handle(url):
		data = GET(url)
		if not data:
			ui.error("Failed to download %s" % url)
			return False
		captchaUrl = matchGet('src="(captcha-[^"]+.jpg)"', data)
		captchaHash = matchGet('id="captcha_hash"[^>]+value="([^"]+)"', data)
		if not captchaUrl or not captchaHash:
			ui.error("Failed to get captcha url")
			return False
		captcha = ui.getCaptcha("http://secured.in/" + captchaUrl, "secured.in")
		if not captcha:
			ui.error("Failed to get captcha")
			return False
		postData = "captcha_key=%s&captcha_hash=%s" % (urllib.quote(captcha), captchaHash)
		data = POST(url, postData)
		if not data:
			ui.error("Failed to download %s" % url)
			return False
		dlIds = re.findall("accessDownload\([0-9]+, [0-9]+, '([^']+)'", data)
		if not dlIds:
			ui.error("Captcha code wrong")
			return False
		for dlId in dlIds:
			data = POST("http://secured.in/ajax-handler.php",
				"cmd=download&download_id=%s" % dlId)
			if data == "error":
				ui.error("Failed to download dl id %s" % dlId)
				continue
			# Call decipher
			# Debian has problems here, execute
			#  export LD_LIBRARY_PATH=/usr/lib/firefox
			#  export MOZILLA_FIVE_HOME=/usr/lib/firefox
			# to fix them
			import gtkmozembed
			import gtk
			moz = gtkmozembed.MozEmbed()
			win = gtk.Window()
			win.add(moz)
			win.show_all()
			data = ('<script src="http://secured.in/scripts/cypher.js"></script>' +
				'<script>b=%r;' % data +
				'document.title=cypher(b);</script>')
			moz.render_data(data, long(len(data)), 'file:///', 'text/html')
			moz.connect("title", lambda x: gtk.main_quit())
			gtk.main()
			dlUrl = moz.get_title()
			win.hide()
			del moz
			del win
			
			addDownload(dlUrl)
		return True, handler.HANDLER_WAS_REDIRECTED
## }}} #########################################################################
## UCMS {{{ ####################################################################
class uCms(handler):
	NAME = "UCMS Redirector"

	@staticmethod
	def canHandle(url):
		return matchGet("location=mirror$", url) != False

	@staticmethod
	def handle(url):
		data = POST(url, "m=download")
		urls = re.findall('out.php\?.*?url=([^"& ]+)', data)
		if urls:
			for url in urls:
				addDownload(urllib2.unquote(url))
			return True, handler.HANDLER_WAS_REDIRECTED
		return False
## }}} #########################################################################
## UPLOADED.TO {{{ #############################################################
class uploadedTo(handler):
	NAME = "uploaded.to"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(?:www.)?uploaded.to/", url) != False

	@staticmethod
	def handle(url):
		data = GET(url)
		if not data:
			ui.error("Failed to load %s" % url)
			return False
		if matchGet("Your Free-Traffic is exceeded", data):
			ui.warn("Exceeded download limit. Restarting router")
			if not restartRouter():
				ui.warn("Failed to restart router")
				ui.wait("Waiting %d seconds", 60 ** 2)
			return uploadedTo.handle(url)
		urlData = matchGet('action="(http://[^.]+.uploaded.to/dl[^"]+)"', data)
		fileName = matchGet('<title>(.+?) ... at', data)
		if not urlData or not fileName:
			ui.error("Failed to get download link from uploaded.to page")
			return False
		wait = matchGet('srcs = ([0-9]+)', data)
		if wait:
			ui.wait("Waiting %d seconds", int(wait))
		target = download(urlData, "download_submit=Download", fileName)
		if target:
			if getFileSize(target) > 1024*15 or "<title>uploaded.to" not in open(target).read():
				return True
		ui.error("Download failed")
		return False
## }}} #########################################################################
## XIRROR.COM {{{ ##############################################################
class xirrorCom(handler):
	NAME = "xirror.com Redirector"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(?:www\.)?xirror.com", url) != False

	@staticmethod
	def handle(url):
		data = GET(url)
		if not data:
			ui.error("Failed to load %s" % url)
			return False
		data = urllib.unquote(data)
		urls = re.findall('popup\("([^"]+)', data)
		urls = map(lambda x: "".join(reversed(list(x))), urls)
		if handler.options.prefer == "rapidshare":
			urls = filter(lambda x: "rapidshare" in x, urls)
		elif handler.options.prefer == "uploaded.to":
			urls = filter(lambda x: "uploaded.to" in x, urls)
		elif handler.options.prefer == "netload.in":
			urls = filter(lambda x: "netload.in" in x, urls)
		foundOne = False
		for url in urls:
			foundOne = True
			addDownload(url)
		return foundOne, handler.HANDLER_WAS_REDIRECTED
## }}} #########################################################################
## YOURLAYER.COM {{{ ###########################################################
class yourLayerCom(handler):
	NAME = "yourlayer.com Redirector"

	@staticmethod
	def canHandle(url):
		return matchGet("^http://(?:www\.)?yourlayer.com", url) != False

	@staticmethod
	def handle(url):
		data = GET(url)
		link = matchGet('<iframe[^>]+src="([^"]+)"', data)
		if not link:
			return False
		addDownload(link)
		return True, handler.HANDLER_WAS_REDIRECTED
## }}} #########################################################################

if __name__ == "__main__":
	# Parse options
	optionParser = optparse.OptionParser(usage="%prog [options] [urls]",
		version = "rsDownload %s\n\n" % RSDOWNLOD_VERSION +
			"Supported download pages:\n - " +
			"\n - ".join(
				sorted(( obj.NAME for obj in globals().values()
				if "NAME" in dir(obj) )))
		 )
	optionParser.add_option("-s", "--store-done", dest="storedone",
		action="store_true",
		help="Store a list of downloaded files in ./.rsDownloadDone")
	optionParser.add_option("-p", "--prefer", dest="prefer", metavar="PAGE",
		default="rapidshare",
		help="Tell redirector pages which service to prefer")
	optionParser.add_option("-i", "--infile", dest="infile", metavar="FILE",
		help="Use URLs from FILE as input")
	optionParser.add_option("-u", "--update-check", dest="updateCheck", action="store_true",
		help="Check if a newer version of this script is available")
	(options, args) = optionParser.parse_args()
	map(fileList.append, args)
	handler.options = options

	if options.updateCheck: # {{{
		md5SumNewest = GET("http://www.pberndt.com/Programme/Linux/rsDownload/_version.phpc")
		try:
			from hashlib import md5
		except:
			try:
				from md5 import md5
			except:
				ui.error("Feature not available - md5 library missing")
				sys.exit(1)
		if md5(open(__file__).read()).hexdigest() != md5SumNewest:
			ui.info("A new version of rsDownload is available")
			sys.exit(1)
		else:
			ui.info("Your version is up to date")
			sys.exit(0)
		# }}}

	validPrefers = ("rapidshare", "uploaded.to", "netload.in")
	if options.prefer not in validPrefers:
		optionParser.error("-p accepts %s" % ", ".join(validPrefers))
	if options.storedone and os.access("./.rsDownloadDone", os.F_OK):
		doneList = map(lambda x: x.strip(), open("./.rsDownloadDone").readlines())


	# Run downloader
	if options.infile:
		ui.info("Downloader ready. Scanning file '%s' for input" % options.infile, 2)
		inFileAlreadyTried = []
	else:
		ui.info("Downloader ready. Paste your files here.", 2)
	while True:
		while len(fileList) == 0:
			file = None
			if options.infile:
				if os.access(options.infile, os.R_OK):
					newFiles = filter(lambda x: x.strip() not in inFileAlreadyTried and x.strip(), open(options.infile).readlines())
					if newFiles:
						file = newFiles[0].strip()
						inFileAlreadyTried.append(file)
				if not file:
					time.sleep(1)
			else:
				try:
					file = raw_input().strip()
				except KeyboardInterrupt:
					sys.exit(0)
				except EOFError:
					sys.exit(0)
			if file:
				fileList.append(file)
		file = fileList.pop(0)
		if "://" not in file:
			continue
		if file in doneList:
			ui.info("Already downloaded %s" % file)
			continue
		success = False
		for obj in globals().values():
			if type(obj) is type and issubclass(obj, handler) and obj != handler:
				if obj.canHandle(file):
					ui.info("Using %s for %s" % (obj.NAME, file), 2)
					response = obj.handle(file)
					flags = 0
					if type(response) is tuple:
						success = response[0]
						if len(response) > 1:
							flags = response[1]
					else:
						success = response
					if success:
						break
		if not success:
			ui.error("Failed to download %s" % file)
			errorHook()
		else:
			if not flags & handler.HANDLER_WAS_REDIRECTED:
				ui.info("Download of %s complete." % file, 2)
				doneList.append(file)
			doneHook(flags)

		# Store done list
		if options.storedone:
			file = open("./.rsDownloadDone", "w")
			file.write("\n".join(doneList))
			file.close()

		# Clear cookie jar after download
		cookieJar.clear()
		useReferer = ""