diff --git a/.gitignore b/.gitignore
index 55f2777..f7e9d81 100755
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,15 @@
 *.ini
 *.pyc
 botmily.db
+botmily.7z
+Thumbs.db
+tells.csv
+quotedb.7z
+quote.csv
+botmily.sql
+quote table.csv
+Thumbs.db
+quotej.csv
+Thumbs.db
+temp.jpg
+Thumbs.db
diff --git a/Thumbs.db b/Thumbs.db
new file mode 100644
index 0000000..3431ff7
Binary files /dev/null and b/Thumbs.db differ
diff --git a/botmily/bot.py b/botmily/bot.py
index a242a6c..70f0b51 100755
--- a/botmily/bot.py
+++ b/botmily/bot.py
@@ -8,22 +8,24 @@
 import socket
 import sys
 import traceback
-
+import math
 from botmily import config
 from botmily import irc
 import plugins
 
+import select
+import time
+
 class bot():
 	def __init__(self):
 		self.server = config.server
 		self.nickname = config.name
-		self.realname = b"Botmily https://github.com/kgc/botmily"
+		self.realname = b"Botdrew https://github.com/kgc/botmily"
 		self.channels = config.channels
 		self.password = config.password
-
-		self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-		self.socket.connect((self.server, 6667))
-		self.irc = irc.irc_handler(self.socket, self , self.irc_error)
+		self.sa_user = config.sa_user
+		self.sa_password = config.sa_password
+		self.timeout = config.timeout
 
 		print("Initializing plugins...")
 		self.commands = {}
@@ -34,7 +36,56 @@ def __init__(self):
 			self.commands.update(plugin.commands)
 			self.triggers.extend(plugin.triggers)
 
-		asyncore.loop()
+
+		Continue_State = True
+		print("Attempting Connection! Mash Ctrl+C or whatever to exit.")
+		while (Continue_State):
+			Continue_State = self.connect()
+			if Continue_State:
+				try:
+					print("Sleeping for 30 seconds before retrying...")
+					time.sleep(30)
+				except KeyboardInterrupt, err:
+					Continue_State = False
+
+		print("Exiting!")
+
+	def drop(self, calling_irc_handler):
+		# Prevent accidental rogue irc timers?
+		print("DC message from " + calling_irc_handler.__str__())
+		if self.irc is calling_irc_handler:
+			self.socket.close()
+			asyncore.close_all()
+		else:
+			print("Bot is ignoring out of date handler. Masti is bad at threading!")
+
+	def connect(self):
+		# we COULD close our old socket, if we have one, but... it should get closed in garbage collection so who cares?
+		
+		print("Connecting to:",self.server)
+
+		self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+
+		try:
+			self.socket.connect((self.server, 6667))
+		except socket.gaierror, err:
+			print("Could not initiate connection!")
+			return True
+
+		self.irc = irc.irc_handler(self.socket, self , self.irc_error)
+
+		print("Connection loop starting...")
+
+		try:
+			asyncore.loop()
+		except select.error, err:
+			pass
+		except KeyboardInterrupt, err:
+			print(" <- Keyboard Interupt detected, cancelling thread timers where possible...")
+			self.irc.stop()
+			return False
+
+		return True
 
 	def join(self, nick, user, host, channel):
 		if nick == self.nickname:
@@ -60,8 +111,20 @@ def privmsg(self, nick, user, host, channel, message):
 			if len(possible_commands) == 1:
 				message_data["command"] = possible_commands[0][0]
 				try:
+					# Contact the bound function, supplying message data and self as a paramter
 					output = possible_commands[0][1](message_data, self)
-					self.say(nick, channel, output)
+					if isinstance(output, str) or isinstance(output, unicode):
+						self.say(nick, channel, output)
+					elif isinstance(output, dict):
+						# Merge the two dictionaries:
+						o_dict = message_data.copy()
+						o_dict.update(output)
+						# Check we have the new output key
+						if "output" in o_dict:
+							self.say(o_dict["nick"], o_dict["channel"], o_dict["output"])
+					else:
+						# Fuck it, print it anyway:
+						self.say(nick, channel, output)
 				except Exception, E:
 					print('Encountered error while processing commmand %s with input %s' %(str(possible_commands[0][1]),str(message_data)))
 					traceback.print_exc()
@@ -85,9 +148,26 @@ def privmsg(self, nick, user, host, channel, message):
 					print('Encountered error while processing trigger %s with input %s' %(str(function),str(message_data)))
 					traceback.print_exc()
 					self.say(nick,channel,'I crashed while trying to deal with something you said @_@')
+
+		#for omni in self.allmsg:
 				
 
 	def say(self, nick, channel, output):
+		maxsay = 400
+		if output is None:
+			return
+		if len(output) > maxsay:
+			maxval = int(math.ceil(len(output)/float(maxsay)))
+			for x in range(0,maxval):
+				if x < 2:
+					self.do_msg(nick,channel,output[maxsay*x:maxsay*(x+1)])
+				else:
+					self.do_msg(nick,channel,output[maxsay*x:maxsay*(x+1) - 100] + "... %d lines additional ommitted!" % (maxval - 3))
+					break
+		else:
+			self.do_msg(nick,channel,output)
+
+	def do_msg(self,nick,channel,output):
 		if output is None:
 			return
 		if self.nickname == channel:
@@ -97,4 +177,4 @@ def say(self, nick, channel, output):
 
 	def irc_error(self):
 		print('Nasty error caught, trying to continue anyway , details below :  ')
-		traceback.print_exc()
\ No newline at end of file
+		traceback.print_exc()
diff --git a/botmily/config.py b/botmily/config.py
index 06029b1..89b050c 100755
--- a/botmily/config.py
+++ b/botmily/config.py
@@ -8,6 +8,9 @@
 server = ''
 channels = []
 password = ''
+sa_user = ''
+sa_password = ''
+timeout = 300
 tumblr_blog = ''
 tumblr_user = ''
 tumblr_password = ''
@@ -17,38 +20,40 @@
 lastfm_api_key = ''
 
 def getConfig():
-    global name
-    global server
-    global channels
-    global password
-    global tumblr_blog
-    global tumblr_user
-    global tumblr_password
-    global tumblr_title
-    global tumblr_tumbling
-    global wolframalpha_api_key
-    global lastfm_api_key
-    global oauth_token
-    global oauth_secret
-    global consumer_key
-    global consumer_secret
-    config = ConfigParser()
-    config.read('config.ini')
-    name = config.get('main', 'name')
-    server = config.get('main', 'server')
-    channels = config.get('main', 'channels').split(" ")
-    password = config.get('main', 'password')
-    tumblr_tumbling = config.getboolean('tumblr','tumbling')
-    tumblr_blog = config.get('tumblr', 'blog')
-    tumblr_user = config.get('tumblr' , 'user')
-    tumblr_password = config.get('tumblr', 'password')
-    tumblr_title = config.get('tumblr','post_titles')
-    wolframalpha_api_key = config.get('wolframalpha', 'api_key')
-    lastfm_api_key = config.get('lastfm', 'api_key')
-    oauth_token = config.get('twitter', 'oauth_token')
-    oauth_secret = config.get('twitter', 'oauth_secret')
-    consumer_key = config.get('twitter', 'consumer_key')
-    consumer_secret = config.get('twitter', 'consumer_secret')
-    print("I will use the name: " + name)
-    print("I will connect to the server: " + server)
-    print("I will connect to the channels: " + ", ".join(channels))
+	global name
+	global server
+	global channels
+	global password
+	global sa_user
+	global sa_password
+	global timeout
+	global wolframalpha_api_key
+	global lastfm_api_key
+	global oauth_token
+	global oauth_secret
+	global consumer_key
+	global consumer_secret
+	global YOUTUBE_API_KEY
+	config = ConfigParser()
+	config.read('config.ini')
+	name = config.get('main', 'name')
+	server = config.get('main', 'server')
+	channels = config.get('main', 'channels').split(" ")
+	password = config.get('main', 'password')
+	sa_user = config.get('main', 'sa_user')
+	sa_password = config.get('main', 'sa_password')
+	try:
+		timeout = int(config.get('main', 'timeout'))
+	except:
+		print("Could not convert timeout value to integer! Defaulting to 300!")
+		timeout = 300
+	wolframalpha_api_key = config.get('wolframalpha', 'api_key')
+	lastfm_api_key = config.get('lastfm', 'api_key')
+	oauth_token = config.get('twitter', 'oauth_token')
+	oauth_secret = config.get('twitter', 'oauth_secret')
+	consumer_key = config.get('twitter', 'consumer_key')
+	consumer_secret = config.get('twitter', 'consumer_secret')
+	YOUTUBE_API_KEY = config.get('youtube', 'api_key')
+	print("I will use the name: " + name)
+	print("I will connect to the server: " + server)
+	print("I will connect to the channels: " + ", ".join(channels))
\ No newline at end of file
diff --git a/botmily/irc.py b/botmily/irc.py
index a6eeef7..ebfe2fa 100644
--- a/botmily/irc.py
+++ b/botmily/irc.py
@@ -1,11 +1,13 @@
 from __future__ import division
-from __future__ import print_function
+#from __future__ import print_function
 from __future__ import unicode_literals
 
 import asynchat
 import time
 import unicodedata
 
+from threading import Timer
+
 controls = {'bold':  '\u0002',
             'color': '\u0003',
             'clear': '\u000f'}
@@ -52,10 +54,15 @@ def __init__(self, sock, bot , error_callback = None):
 		self.set_terminator(b"\r\n")
 		self.bot = bot
 		self.push(b"NICK %s\r\n" %bot.nickname)
-		self.push(b"USER botmily 0 0 :Botmily\r\n")
+		self.push(b"USER botdrew 0 0 :Botdrew\r\n")
 		if error_callback:
 			self.handle_error = error_callback
 
+		# Set up a ping counter:
+		self.pingcount = 0
+		self.mytimer = Timer(self.bot.timeout, self.ping_check, ())
+		self.mytimer.start()
+
 	def collect_incoming_data(self, data):
 		self.ibuffer += data
 
@@ -105,15 +112,38 @@ def pong(self, message):
 		self.push(b"PONG " + message + b"\r\n")
 
 	def raw_PRIVMSG(self, prefix, params):
+		# Increment the number of messages seen here as well because maybe that will be more robust:
+		self.pingcount += 1
+		
 		nick, user, host = split_prefix(prefix)
 		channel = params[0]
 		message = params[-1].decode("utf-8", "replace")
 		self.bot.privmsg(nick, user, host, channel, message)
 
 	def raw_PING(self, prefix, params):
+		#print("Ping")
+		self.pingcount += 1
 		self.pong(params[0])
 
 	def raw_JOIN(self, prefix, params):
 		nick, user, host = split_prefix(prefix)
 		channel = params[0]
 		self.bot.join(nick, user, host, channel)
+
+	def ping_check(self):
+		print self.__str__() + " ~ Seen Pings/Messages:",self.pingcount
+		self.mytimer.cancel()
+		if self.pingcount > 0:
+			self.mytimer = Timer(self.bot.timeout, self.ping_check, ())
+			self.mytimer.start()
+			self.pingcount = 0
+		else:
+			print self.__str__() + " ~ Suspected Disconnection? Closing socket."
+			asynchat.async_chat.handle_close(self)
+			self.bot.drop(self)
+			return False
+	
+	def stop(self):
+		if self.mytimer:
+			self.mytimer.cancel()
+		return 0
\ No newline at end of file
diff --git a/faceSquares.py b/faceSquares.py
deleted file mode 100644
index 6746142..0000000
--- a/faceSquares.py
+++ /dev/null
@@ -1,69 +0,0 @@
-from PIL import Image , ImageDraw , ImageFilter
-import faceapi, urllib2
-from StringIO import StringIO
-import pprint
-#had to make this method because PIL wont let me draw a rectangle properly
-def drawRect(img,top_left,bottom_right, width , color):
-	draw = ImageDraw.Draw(img)
-	top_right = bottom_right[0],top_left[1]
-	bottom_left = top_left[0], bottom_right[1]
-
-	if width > 1:
-		top_left_l1 = top_left[0]  , top_left[1] - width/2
-		top_right_l1 = top_right[0] , top_right[1] - width/2
-		l1 = [top_left_l1,top_right_l1]
-
-		top_right_l2 = top_right[0] - width/2, top_right[1]
-		bottom_right_l2 = bottom_right[0] - width/2 , bottom_right[1]
-		l2 = [top_right_l2 , bottom_right_l2]
-
-		bottom_right_l3 = bottom_right[0]  , bottom_right[1] + width/2
-		bottom_left_l3 = bottom_left[0]  , bottom_left[1] + width/2
-		l3 = [bottom_right_l3,bottom_left_l3]
-
-		bottom_left_l4 = bottom_left[0] + width/2 , bottom_left[1]
-		top_left_l4 = top_left[0] + width/2 , top_left[1]
-		l4 = [bottom_left_l4, top_left_l4]
-	else:
-		l1 = [top_left,top_right]
-		l2 = [top_right,bottom_right]
-		l3 = [bottom_left,bottom_right]
- 		l4 = [bottom_left,top_left]
-
-	draw.line(l1, fill=color, width=width)
-	draw.line(l2, fill=color, width=width)
-	draw.line(l3 ,fill=color, width=width)
-	draw.line(l4, fill=color, width=width)
-
-def getImageFromUrl(url):
-	opener1 = urllib2.build_opener()  
-	page1 = opener1.open(url)  
-	data = StringIO(page1.read())
-	img = Image.open(data)
-	return img
-
-def drawTags(tags,imgurl):
-	image = getImageFromUrl(imgurl).convert('RGB').filter(ImageFilter.FIND_EDGES)
-	for tag in tags:
-		pprint.pprint(tag)
-		if tag.has_key('attributes'):
-			if tag['attributes'][0].has_key('gender'):
-				gender = tag['attributes'][0]['gender']
-				center = int(tag['center']['x']) , int(tag['center']['y'])
-				height = int(tag['height'])
-				width = int(tag['width'])
-				print gender,center,width,height,image
-				drawTag(gender,center,width,height,image)
-	return image
-
-def drawTag(gender,center,width,height,im):
-	c1 = (center[0] - (width/2)) , (center[1] - (height/2))
-	c3 = (center[0] + (width/2)) , (center[1] + (height/2))
-	c2 = (center[0] + (width/2)) , (center[1] - (height/2))
-	c4 = (center[0] - (width/2)) , (center[1] + (height/2))
-	if gender == 'female':
-		drawRect(im,c1,c3,width=4,color='blue')
-	elif gender == 'male':
-		drawRect(im,c1,c3,width=4,color='pink')
-	else:
-		raise Exception('Gender_Binary_Error , please check your priveldge')
diff --git a/faceapi.py b/faceapi.py
deleted file mode 100644
index 638e591..0000000
--- a/faceapi.py
+++ /dev/null
@@ -1,89 +0,0 @@
-
-from __future__ import unicode_literals
-import httplib, urllib , base64 , json 
-from botmily.db import db
-import pprint
-
-MASHAPE_AUTH = {'X-Mashape-Authorization': 'dWNwZDBxdnF1bjhjdnlveWdtZnNtdTBpdXhodWFrOmNlOGZiMGVlM2JkZTAzMTk0YmI2ZWNhZDBjNzMwOTFhYzQ2NzUyMTI='}
-
-def Detect(imageUrl):
-    httpcnx = httplib.HTTPSConnection('lambda-face-detection-and-recognition.p.mashape.com' , strict = True)    
-    params = {'images':imageUrl}
-    url = "/detect" 
-    url = url + "?" + urllib.urlencode( params )
-    httpcnx.request('GET' , url , None , MASHAPE_AUTH)
-    response = httpcnx.getresponse()           
-    return json.loads(response.read())
-
-def makeBlurb(face):
-    blurb = ''
-    try:
-        if len(face['photos']) == 1:
-            photo = face['photos'][0]
-            if photo.has_key('tags'):
-                facecount = 0
-                for tag in photo['tags']:
-                    if len(tag['attributes'][0].keys()) > 1:
-                        facecount += 1
-                if facecount > 1:
-                    blurb = '%s faces in this photo, ' %facecount
-                    faceNum = 1
-                    for tag in photo['tags']:
-                        if len(tag['attributes'][0].keys()) > 1:
-                            blurb = blurb +' Face %i: ' %faceNum +  getTagBlurb(tag) 
-                            faceNum += 1
-                elif facecount == 1:
-                    blurb = 'One face in this photo, ' + getTagBlurb(tag)
-                else:
-                    return None
-                
-                return blurb
-    except Exception , e:
-        print e
-        print '\nFace Json :'
-        print face
-    return None
-    
-def cleanTags(tags):
-    tids = {}
-    toPurge = []
-    try:
-        if len(tags['photos']) == 1:
-            photo = tags['photos'][0]
-            if photo.has_key('tags'):
-                for tag in photo['tags']:
-                    if tids.has_key(tag['tid']):
-                        toPurge.append(tag)
-                    elif len(tag['attributes'][0].keys()) < 2:
-                        toPurge.append(tag)
-                    else:
-                        tids[tag['tid']] = True
-
-    except Exception , e:
-        print e
-        print '\nFace Json :'
-        print tags
-
-    for tag in toPurge:
-        tags['photos'][0]['tags'].remove(tag)
-    return tags
-
-def getTagBlurb(tag):
-    stringDict = {'gender' : '' , 'age' : '' , 'face' : '','glasses':'' , 'smiling':'' , 'lips':'' , 'mood': ''}
-    attributes = tag['attributes']   
-    for attribute in attributes:
-        if attribute.has_key('gender'):
-            stringDict['gender'] = "\u0002%s\u000f(%s%%) " %(attribute['gender'],attribute['confidence'])
-
-    blurb = stringDict['gender'] + stringDict['age'] + stringDict['mood'] 
-    return blurb
-
-
-
-def getTag(face):
-    if len(face['photos']) == 1:
-            photo = face['photos'][0]
-            if photo.has_key('tags'):
-                if len(photo['tags']) == 1:
-                    return photo['tags'][0]
-    return None
diff --git a/makeMacro.py b/makeMacro.py
index 728cbcf..00979ff 100644
--- a/makeMacro.py
+++ b/makeMacro.py
@@ -49,7 +49,7 @@ def makeMacro(imgUrl , text ,fileName):
     imgWidth = image.size[0]
     imgHeight = image.size[0]
     draw = ImageDraw.Draw(image)
-    font = ImageFont.truetype("impact.ttf", 42)
+    font = ImageFont.truetype("impact.ttf", 64)
     bbox = (50, 0, imgWidth, imgHeight)
     drawtext(draw, text, font, "white", bbox)
     image.save(fileName , 'JPEG')
diff --git a/plugins/4chan.py b/plugins/4chan.py
index 7541929..f8e9c59 100755
--- a/plugins/4chan.py
+++ b/plugins/4chan.py
@@ -8,39 +8,54 @@
 import re
 from urllib2 import urlopen
 
-from BeautifulSoup import BeautifulStoneSoup
+from bs4 import BeautifulSoup
 
 import imgur
 import makeMacro
 
 def fourchan(message_data, bot):
     board = ''
+
+    # Commands that should be treated as gross:
+    pm_commands = ["slurm","j","dudes","ecchi","hentai","dick"]
+
+    # This could have been a dict btw lol
     if message_data['command'] == "anime":
         board = "/a/"
-    if message_data['command'] == "dick":
+    elif message_data['command'] == "dick":
         board = "/d/"
-    if message_data['command'] == "technology":
+    elif message_data['command'] == "dudes":
+        board = "/hm/"
+    elif message_data['command'] == "j":
+        board = "/s/"
+    elif message_data['command'] == "slurm":
+        board = "/lgbt/"
+    elif message_data['command'] == "technology":
         board = "/g/"
-    if message_data['command'] == "videogame":
+    elif message_data['command'] == "videogame":
         board = "/v/"
-    if message_data['command'] == "animals":
+    elif message_data['command'] == "animals":
         board = "/an/"
-    if message_data['command'] == "hentai":
+    elif message_data['command'] == "hentai":
         board = "/h/"
-    if message_data['command'] == "ecchi":
+    elif message_data['command'] == "ecchi":
         board = "/e/"
-    if message_data['command'] == "pokemon":
+    elif message_data['command'] == "pokemon":
         board = "/vp/"
 
     result = urlopen('http://boards.4chan.org' + board)
-    soup = BeautifulStoneSoup(result, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)
-    images = soup.findAll('a', attrs={'class': 'fileThumb'})
+    soup = BeautifulSoup(result.read()) #BeautifulStoneSoup(result, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)
+    images = soup.find_all('a',class_='fileThumb') #soup.findAll('a', attrs={'class': 'fileThumb'})
     url = 'http:' + random.choice(images)['href']
     if message_data['parsed'] != "":
         makeMacro.makeMacro(url, message_data['parsed'], "temp.jpg")
         url = imgur.postToImgur(str("temp.jpg"))
-    return url
 
-commands = {"technology": fourchan, "animals": fourchan, "pokemon": fourchan, "ecchi": fourchan, "videogame": fourchan, "hentai": fourchan, "anime": fourchan, "dick": fourchan}
+    if message_data['command'] in pm_commands:
+        return {"output":url, "channel": bot.nickname}
+    else:
+        return url
+
+commands = {"technology": fourchan, "slurm":  fourchan, "j": fourchan, "dudes": fourchan, "animals": fourchan, "pokemon": fourchan, "ecchi": fourchan, "videogame": fourchan, "hentai": fourchan, "anime": fourchan, "dick": fourchan}
 triggers = []
 
diff --git a/plugins/altmedtxt.py b/plugins/altmedtxt.py
new file mode 100644
index 0000000..3a78d4b
--- /dev/null
+++ b/plugins/altmedtxt.py
@@ -0,0 +1,21 @@
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from twitter import Twitter
+from twitter import OAuth
+from botmily import config
+import random
+#randomly return one of the last 200 tweets from https://twitter.com/altmed_txt
+#rate limit is 300 requests per 15 minutes
+def altmed(message_data, bot):
+    t = Twitter(api_version=1.1, auth=OAuth(config.oauth_token,
+    config.oauth_secret, config.consumer_key, config.consumer_secret))
+    tweetno = random.randint(0,199)
+    tweet = t.statuses.user_timeline.altmed_txt(count=200)[tweetno]
+    return tweet['text']
+
+
+commands = {"altmed": altmed}
+triggers = []
+
diff --git a/plugins/beer.py b/plugins/beer.py
new file mode 100644
index 0000000..170d0ab
--- /dev/null
+++ b/plugins/beer.py
@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+#copied from fixed version of .urban
+
+
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from urllib2 import urlopen
+
+from ratebeer import RateBeer
+
+def beer(message_data, bot):
+	rb = RateBeer()
+	results = rb.search(message_data["parsed"])
+	highest_ratings = -1 #get at least 1
+	if results['beers']:
+		for beer in results['beers']:
+			if beer['num_ratings'] > highest_ratings: #pull highest rated beer from result set
+				topbeer = beer
+				topdetails = rb.beer(beer['url'])
+				highest_ratings = beer['num_ratings']
+		if 'overall_rating' in topbeer: #overall_rating doesn't always exist http://www.ratebeer.com/ratingsqa.asp
+			reply =  topbeer['name'].encode('utf-8') + ': Rating ' + str(topbeer['overall_rating']) + ', http://www.ratebeer.com'+ topbeer['url'].encode('utf-8') + ' '
+		else:
+			reply = topbeer['name'].encode('utf-8') + ': http://www.ratebeer.com'+ topbeer['url'].encode('utf-8') + ' (' + str(topbeer['num_ratings']) +' ratings) '
+		
+		reply += topdetails['style'].encode('utf-8') + ', ' + str(topdetails['abv']) + "% ABV, " + str(topdetails['calories']) + ' calories from alcohol, brewed by ' + topdetails['brewery'].encode('utf-8')
+	else:
+		reply = 'Not Found'
+	return reply.encode('utf-8')
+	
+commands = {"beer": beer}
+triggers = []
+
diff --git a/plugins/bitcoin.py b/plugins/bitcoin.py
deleted file mode 100644
index 14df0d2..0000000
--- a/plugins/bitcoin.py
+++ /dev/null
@@ -1,42 +0,0 @@
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-import random
-import re
-import json
-import urllib2
-
-from botmily import irc
-
-regex1 = r'\$([0-9]+\.?[0-9][0-9]?)'
-regex2 = r".*?(\d+\.?\d\d)[     ]*dolla"
-
-def get_btc_price():
-    response = urllib2.urlopen('https://mtgox.com/api/1/BTCUSD/ticker')
-    tick = json.loads(response.read())['return']
-    data = {}
-    data['average'] = float(tick['avg']['value'])
-    data['low'] = float(tick['low']['value'])
-    data['high'] = float(tick['high']['value'])
-    data['last'] = float(tick['last']['value'])
-    data['volume'] = float(tick['vol']['value'])
-    return data
-
-def btc_price(message_data, bot):
-    try:
-        tick = get_btc_price()
-    except urllib2.URLError:
-        return "MtGox is down :("
-    return "Current price: $" + irc.color(str(tick['last']), 'orange') + " - High: $" + irc.color(str(tick['high']), 'orange') + " - Low: $" + irc.color(str(tick['low']), 'orange') + " - Volume: " + str(tick['volume']) + "BTC"
-
-def btc_convert(message_data, bot):
-    if random.randint(0, 9) != 0:
-        return
-    amount = float(message_data["re"].group(1))
-    avg = get_btc_price()['average']
-    return 'If you converted that to bitcoins you could have %f BTC!' %((amount / avg))
-
-commands = {"bitcoin": btc_price, "btc": btc_price}
-triggers = [(regex1, btc_convert), (regex2, btc_convert)]
-
diff --git a/plugins/bitcointxt.py b/plugins/bitcointxt.py
new file mode 100644
index 0000000..6ff2966
--- /dev/null
+++ b/plugins/bitcointxt.py
@@ -0,0 +1,21 @@
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from twitter import Twitter
+from twitter import OAuth
+from botmily import config
+import random
+#randomly return one of the last 200 tweets from https://twitter.com/bitcoin_txt
+#rate limit is 300 requests per 15 minutes
+def bitcoin(message_data, bot):
+    t = Twitter(api_version=1.1, auth=OAuth(config.oauth_token,
+    config.oauth_secret, config.consumer_key, config.consumer_secret))
+    tweetno = random.randint(0,199)
+    tweet = t.statuses.user_timeline.bitcoin_txt(count=200)[tweetno]
+    return tweet['text']
+
+
+commands = {"bitcoin": bitcoin}
+triggers = []
+
diff --git a/plugins/catte.py b/plugins/catte.py
deleted file mode 100755
index 186d9e1..0000000
--- a/plugins/catte.py
+++ /dev/null
@@ -1,19 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-import re
-import urllib2
-
-def catte(message_data, bot):
-    try:
-        result = urllib2.urlopen('http://cattes.me:3333/random')
-    except urllib2.URLError:
-        return 'No cattes :('
-    return 'http://cattes.me:3333/images/' + result.read()
-
-commands = {"catte": catte}
-triggers = []
-
diff --git a/plugins/down.py b/plugins/down.py
new file mode 100644
index 0000000..c91ee69
--- /dev/null
+++ b/plugins/down.py
@@ -0,0 +1,21 @@
+from __future__ import division
+#from __future__ import print_function
+from __future__ import unicode_literals
+
+import urllib2
+
+# Checks if a website is down
+def down(message_data, bot):
+    '''.down <url> -- checks to see if the site is down'''
+    webaddress = message_data['parsed'].strip()
+    if 'http://' not in webaddress:
+        webaddress = 'http://' + webaddress
+
+    try:
+        result = urllib2.urlopen(webaddress)
+        return webaddress + " seems to be up."
+    except urllib2.URLError:
+        return webaddress + " seems to be down."
+
+commands = {"down": down}
+triggers = []
\ No newline at end of file
diff --git a/plugins/etymology.py b/plugins/etymology.py
index 8c85bdd..4d602aa 100755
--- a/plugins/etymology.py
+++ b/plugins/etymology.py
@@ -7,14 +7,20 @@
 import re
 from urllib2 import urlopen
 
-from BeautifulSoup import BeautifulStoneSoup
+from bs4 import BeautifulSoup
 
 def etymology(message_data, bot):
-    result = urlopen('http://www.etymonline.com/index.php?term=' + message_data["parsed"])
-    soup = BeautifulStoneSoup(result, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)
-    if soup.dl is None:
-        return "Not found"
-    return "".join(soup.dl.findAll(text=True)).replace("\n", " ")
+	result = urlopen('http://www.etymonline.com/index.php?term=' + message_data["parsed"])
+	reply = ''
+	soup = BeautifulSoup(result.read()) #BeautifulStoneSoup(result, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)
+	if soup.dl is None:
+		reply = 'Not found'
+	else:
+		reply = "".join(soup.dl.findAll(text=True)).replace("\n", " ")
+	if reply == 'Not found':
+		return reply
+	else:
+		return reply[:300]+'... http://www.etymonline.com/index.php?term=' + message_data["parsed"]
 
 commands = {"etymology": etymology}
 triggers = []
diff --git a/plugins/face.py b/plugins/face.py
deleted file mode 100644
index cda9db8..0000000
--- a/plugins/face.py
+++ /dev/null
@@ -1,44 +0,0 @@
-from __future__ import division
-from __future__ import unicode_literals
-from urlparse import urlparse
-import random
-import re
-import httplib, urllib , base64 , json 
-import faceapi , imgur
-from botmily.db import db
-from botmily import config
-from pprint import pprint
-import faceSquares, imgur
-
-regex = r'\(?\bhttp://[-A-Za-z0-9+&@#/%?=~_()|!:,.;]*[-A-Za-z0-9+&@#/%=~_()|]'
-
-def checkValidImage(url):
-    lowerurl = url.lower()
-    if lowerurl.endswith('.jpg') or lowerurl.endswith('.png') or lowerurl.endswith('.jpeg') or lowerurl.endswith('.gif'):
-        try:
-            urllib.urlopen(url)
-        except IOError:
-            return "URL Unreachable or something idk fix ur shit"
-    else:
-        return 'Not recognized image type , type .jpg , .png , .jpeg or .gif'    
-    return None
-
-def face(message_data, bot):
-    imgurl = message_data["parsed"]
-    invalid = checkValidImage(imgurl)
-    if invalid:
-        return invalid   
-    result = faceapi.Detect(imgurl)
-    result = faceapi.cleanTags(result)
-    characteristics = faceapi.makeBlurb(result)
-    if characteristics:
-        img = faceSquares.drawTags(result['photos'][0]['tags'],imgurl)
-        img.save('temp.png', 'PNG')
-        postedUrl = imgur.postToImgur(str('temp.png'))
-        return characteristics + ', what I saw %s' %postedUrl
-    else:
-        return "Couldn't find a face, you are too ugly maybe :("
-
-commands = {"face": face, "passmeter": face}
-triggers = []
-
diff --git a/plugins/google.py b/plugins/google.py
index 45842c1..ece791b 100755
--- a/plugins/google.py
+++ b/plugins/google.py
@@ -10,7 +10,7 @@
 from urllib import quote_plus
 from urllib2 import urlopen
 
-from BeautifulSoup import BeautifulStoneSoup
+from bs4 import BeautifulSoup
 
 from botmily import irc
 
@@ -21,8 +21,8 @@ def google(message_data, bot):
         return "No results found"
     first_result = json_data['responseData']['results'][0]
     output = first_result['unescapedUrl'] + ' | '
-    output += irc.bold(BeautifulStoneSoup(first_result['titleNoFormatting'], convertEntities=BeautifulStoneSoup.HTML_ENTITIES).text) + ' | '
-    output += BeautifulStoneSoup(first_result['content'], convertEntities=BeautifulStoneSoup.HTML_ENTITIES).text
+    output += irc.bold(BeautifulSoup(first_result['titleNoFormatting'].read()).text) + ' | ' #irc.bold(BeautifulStoneSoup(first_result['titleNoFormatting'], convertEntities=BeautifulStoneSoup.HTML_ENTITIES).text) + ' | '
+    output += BeautifulSoup(first_result['content'].read()).text #BeautifulStoneSoup(first_result['content'], convertEntities=BeautifulStoneSoup.HTML_ENTITIES).text
     return output
 
 def gis(message_data, bot):
diff --git a/plugins/imgay.py b/plugins/imgay.py
deleted file mode 100755
index 2e3b61c..0000000
--- a/plugins/imgay.py
+++ /dev/null
@@ -1,14 +0,0 @@
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-import re
-
-from botmily import irc
-
-def imgay(message_data, bot):
-    return 'same'
-
-commands = {}
-triggers = [("im gay", imgay)]
-
diff --git a/plugins/mtg.py b/plugins/mtg.py
index 87061e1..ccbf504 100755
--- a/plugins/mtg.py
+++ b/plugins/mtg.py
@@ -8,11 +8,11 @@
 from urllib import quote_plus
 from urllib2 import urlopen
 
-from BeautifulSoup import BeautifulStoneSoup
+from bs4 import BeautifulSoup
 
 def mtg(message_data, bot):
     result = urlopen('http://magiccards.info/query?v=card&s=cname&q=' + quote_plus(message_data["parsed"]))
-    soup = BeautifulStoneSoup(result, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)
+    soup = BeautifulStoneSoup(result.read()) #BeautifulStoneSoup(result, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)
     card = soup.findAll('table', align='center')[1]
     output = card.find('a').text + ' | '
     output += card.find('p').text.strip().replace('\n', ' ') + ' | '
diff --git a/plugins/nanowrimo.py b/plugins/nanowrimo.py
new file mode 100644
index 0000000..47067e1
--- /dev/null
+++ b/plugins/nanowrimo.py
@@ -0,0 +1,21 @@
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from twitter import Twitter
+from twitter import OAuth
+from botmily import config
+import random
+#randomly return one of the last 200 tweets from https://twitter.com/nanowrimo_txt
+#rate limit is 300 requests per 15 minutes
+def nanowrimo(message_data, bot):
+    t = Twitter(api_version=1.1, auth=OAuth(config.oauth_token,
+    config.oauth_secret, config.consumer_key, config.consumer_secret))
+    tweetno = random.randint(0,199)
+    tweet = t.statuses.user_timeline.nanowrimo_txt(count=200)[tweetno]
+    return tweet['text']
+
+
+commands = {"nanowrimo": nanowrimo}
+triggers = []
+
diff --git a/plugins/newtube.py b/plugins/newtube.py
new file mode 100644
index 0000000..d12bfd7
--- /dev/null
+++ b/plugins/newtube.py
@@ -0,0 +1,136 @@
+# Rewritten 30th May 2015 to support Youtube v3 API
+
+from __future__ import division
+# no one wants print(), go away:
+#from __future__ import print_function
+from __future__ import unicode_literals
+
+import locale
+import re
+import time
+from datetime import datetime, timedelta
+
+# 3.0 API:
+from apiclient.discovery import build
+
+# Import ISODATE for ISO duration functions
+import isodate
+
+# Youtube API things:
+YOUTUBE_API_SERVICE_NAME = "youtube"
+YOUTUBE_API_VERSION = "v3"
+
+# URL bases for different search result types:
+YOUTUBE_SHORT_URL = "http://youtu.be/"
+YOUTUBE_PLAYLIST_URL = "http://www.youtube.com/playlist?list="
+YOUTUBE_CHANNEL_URL = "http://www.youtube.com/channel/"
+
+# Trigger regex. Detects youtube urls/IDs
+# Note: Does not need capital A-Z due to re.I used on regex search in initialising code
+regex = r'(?:youtube.*?(?:v=|/v/)|youtu\.be/|yooouuutuuube.*?id=)([-_a-z0-9]+)'
+
+# Import the IRC thing, for irc.bold
+from botmily import irc
+from botmily import config
+
+# Converts ISO time into a big long string with days allegedly, hours, minutes and seconds
+def convertISOTime(duration):
+	# Well, duration is now an ISO string in the format PT##H##M##S
+	# This is a non trivial format so we use the ISODATE library for this
+	# You can get it with pip install isodate
+	converted_duration = isodate.parse_duration(duration)
+
+	# Now do this same kinda weird code as before but whatever I imagine it works:
+	d = datetime(1,1,1) + converted_duration
+	if d.day-1 > 0:
+		return '%d days %d hours %d minutes %d seconds' %(d.day-1,d.hour,d.minute,d.second)
+	elif d.hour > 0:
+		return '%d hours %d minutes %d seconds' %(d.hour,d.minute,d.second)
+	elif d.minute > 0:
+		return '%d minutes %d seconds' %(d.minute,d.second)
+	else:	
+		return '%d seconds' %d.second 
+
+# Youtube API v3 Search functionality
+def search(message_data, bot):
+	# Create API service:
+	youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, developerKey=config.YOUTUBE_API_KEY)
+
+	search_term = message_data['parsed']
+
+	# Call the search.list method to retrieve results matching the specified
+	# query term.
+	search_response = youtube.search().list(
+	q=search_term,
+	part="id,snippet",
+	maxResults=1
+	).execute()
+
+	# Technically we can differentiate between videos, channels, and playlists, but we're just going to hardcodedely report one result:
+	my_result = "No results"
+
+	# Generate the prototype for the string we will respond with:
+	prototype_format = "\u0002%s\u000f - %s%s"
+
+	# Format the result in the appropriate manner (nb we get a list because we just get a list, but it's only 1 entry long (for now...))
+	for search_result in search_response.get("items", []):
+		if search_result['id']['kind'] == "youtube#video":
+		  my_result = (prototype_format % (search_result['snippet']['title'],YOUTUBE_SHORT_URL,search_result['id']['videoId']))
+		elif search_result['id']['kind'] == "youtube#channel":
+		  my_result = (prototype_format % (search_result['snippet']['title'],YOUTUBE_CHANNEL_URL,search_result['id']['channelId']))
+		elif search_result['id']['kind'] == "youtube#playlist":
+		  my_result = (prototype_format % (search_result['snippet']['title'],YOUTUBE_PLAYLIST_URL,search_result['id']['playlistId']))
+	
+	return my_result
+
+# Youtube API v3 Video info functionality
+def parse(message_data, bot):
+	# Takes a message_data dictionary as a param
+	# Get the regex match entry from the dictionry, and select result 1 from it. This is all done in the bot script somewhere.
+ 	video_id = message_data['re'].group(1)
+
+	# Create API service:
+	youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, developerKey=config.YOUTUBE_API_KEY)
+
+	# Call the videos.list method to retrieve results matching the specified
+	# query term.
+	video_responces = youtube.videos().list(
+	id=video_id,
+	part="id,snippet,contentDetails,statistics" # We call all these things to uh, well I don't know why we do ID really, but we do it to get all the info we need
+	).execute()
+	# snippet has most of the title/uploader/date info
+	# contentDetails has the duration
+	# statistics has the likes/views
+
+	my_result = None
+
+	for video_responce in video_responces.get("items", []):
+		if video_responce['kind'] == "youtube#video":
+			my_result = irc.bold(unicode(video_responce['snippet']['title'])) + " - length "
+			my_result += irc.bold(convertISOTime(video_responce['contentDetails']['duration']))
+
+			# Generate score out of 5 I guess?
+			# Get how many likes/dislikes the video has
+			likes = int(video_responce['statistics']['likeCount'])
+			dislikes = int(video_responce['statistics']['dislikeCount'])
+			# Work out the total
+			totals = likes + dislikes
+			# If non zero, add the score
+			if totals > 0:
+				score = 5 * (float(likes) / float(totals))
+				my_result += " - rated " + irc.bold(locale.format("%.2f", score)) + "/5.0 (" + locale.format("%d",totals) + ")"
+
+			# Do something entirely dissimilar but the same kinda for the views. Which is to say: add the views
+			views = video_responce['statistics']['viewCount']
+			if views:
+				my_result += " - " + irc.bold(views) + " views"
+
+			# Add the remaining user and upload time fields:
+			my_result += " - " + irc.bold(unicode(video_responce['snippet']['channelTitle'])) + " on "
+			my_result += irc.bold(time.strftime("%Y.%m.%d", time.strptime(video_responce['snippet']['publishedAt'], "%Y-%m-%dT%H:%M:%S.000Z")))
+
+ 	return my_result
+
+# Register with the bot:
+commands = {"youtube": search, "y": search, "yt": search}
+triggers = [(regex, parse)]
diff --git a/plugins/privilege.py b/plugins/privilege.py
deleted file mode 100644
index 3810675..0000000
--- a/plugins/privilege.py
+++ /dev/null
@@ -1,15 +0,0 @@
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-import re
-
-from botmily import irc
-
-def privilege(message_data, bot):
-    bot.irc.kick(message_data["channel"], message_data["nick"]);
-    return 'out'
-
-commands = {}
-triggers = [("check your .*privilege", privilege)]
-
diff --git a/plugins/pua.py b/plugins/pua.py
new file mode 100644
index 0000000..5187147
--- /dev/null
+++ b/plugins/pua.py
@@ -0,0 +1,21 @@
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from twitter import Twitter
+from twitter import OAuth
+from botmily import config
+import random
+#randomly return one of the last 200 tweets from https://twitter.com/PUA_txt
+#rate limit is 300 requests per 15 minutes
+def puatxt(message_data, bot):
+    t = Twitter(api_version=1.1, auth=OAuth(config.oauth_token,
+    config.oauth_secret, config.consumer_key, config.consumer_secret))
+    tweetno = random.randint(0,199)
+    tweet = t.statuses.user_timeline.PUA_txt(count=200)[tweetno]
+    return tweet['text']
+
+
+commands = {"puatxt": puatxt}
+triggers = []
+
diff --git a/plugins/quote.py b/plugins/quote.py
index b703165..785bfe3 100755
--- a/plugins/quote.py
+++ b/plugins/quote.py
@@ -5,23 +5,123 @@
 from __future__ import unicode_literals
 
 import time
+import random
+import re
 
 from botmily.db import db
 
+LastSearch = {}
+LastResults = {}
+LastNum = {}
+
+def retformat(num,total,row):
+    return "Quote %d/%d" % (num+1,total) + ": (" + row[0] + ") " + row[1]
+
 def quote(message_data, bot):
+    global LastSearch, LastResults, LastNum
+
     db.execute("create table if not exists quote(sender text, quote text, time integer)")
     if message_data["parsed"][:4] == "add ":
         db.execute("insert into quote(sender, quote, time) values (:sender, :quote, :time)", {"sender": message_data["nick"], "quote": message_data["parsed"][4:], "time": int(time.time())})
         db.commit()
+        # Clear out all caches;
+        LastSearch = {}
+        LastResults = {}
+        LastNum = {}
         return "Quote added."
     elif message_data["parsed"][:7] == "search ":
-        quotes = db.execute("select sender, quote, time from quote where quote like :quote order by random()", {"quote": "%" + message_data["parsed"][7:] + "%"}).fetchall()
-        if len(quotes) == 0:
+
+        # Seperate this data by channel;
+        chan = message_data['channel']
+
+        searchstring = message_data["parsed"]
+        splitstring = searchstring.split()
+        # If there are more than 3 parts, we are supplying a number we want to use perhaps, try to convert it to a number
+        # If it succeeds, pop it from the search query
+        num = 0
+        if len(splitstring) >= 3:
+            try:
+                num = int(splitstring[-1])
+                if num != 0:
+                    # Valid number detected, remove this number:
+                    splitstring.pop()
+                    searchstring = " ".join(splitstring)
+            except ValueError:
+                num = 0
+                pass
+
+        # Check to see if this is the same as the last thing we searched so we can skip the DB pull:
+        quotes = None
+        IsReallyNext = False
+        if chan in LastSearch:
+            if searchstring == LastSearch[chan]:
+                quotes = LastResults[chan]
+                # Also sometimes this is really just someone doing the same search over and over to hear the next quote, so pretend to be next:
+                IsReallyNext = True
+
+        # If the list was not cached, do a DB pull:
+        if quotes == None:
+            quotes = db.execute("select sender, quote, time from quote where quote like :quote order by time ASC", {"quote": "%" + searchstring[7:] + "%"}).fetchall()
+
+        # If results is now empty, don't update global dicts:
+        total = len(quotes)
+        if total == 0:
             return "Nothing found."
-        row = quotes[0]
-        return "Found " + str(len(quotes)) + " quotes: (" + row[0] + ") " + row[1]
+
+        # update cache dicts:
+        LastSearch[chan] = searchstring
+        LastResults[chan] = quotes
+        
+        # If num is out of range, select from the last value if we're really a 'next' command, or select randomly from range 0 - total
+        if num <= 0 or num > total:
+            if (IsReallyNext):
+                num = ((LastNum[chan] + 1) % total)
+            else: num = random.randrange(0,total) 
+        else:
+            # Otherwise substract one from supplied digit due to 0 indexing etc
+            num -= 1
+
+        # Update the cached value for this:
+        LastNum[chan] = num
+        
+        # Select that quote row, and provied to formatting function
+        row = quotes[num]
+        return retformat(num,total,row)
+
+    elif message_data["parsed"][:4] == "next":
+        chan = message_data['channel']
+
+        splitstring = message_data["parsed"].split()
+        # If there are more than 2 parts, we are supplying a number we want to use probably
+        num = 0
+        if len(splitstring) >= 2:
+            try:
+                num = int(splitstring[-1])
+            except ValueError:
+                num = 0
+                pass
+
+        if not(chan in LastResults):
+            return "No stored query. Try search instead!"
+        quotes = LastResults[chan]
+
+        total = len(quotes)
+        if total == 0:
+            return "No stored query. Try search instead!"
+
+        # If num is out of range, recover last request and add 1 mod total:
+        if num <= 0 or num > total:
+            num = (LastNum[chan] + 1) % total
+        else:
+            # Otherwise substract one from supplied digit due to 0 indexing etc
+            num -= 1
+
+        LastNum[chan] = num
+
+        row = quotes[num]
+        return retformat(num,total,row)
     else:
-        return "Unknown command."
+        return "Use quote followed by 'add', 'search ...', 'search  ... #', 'next' or 'next #'!"
 
 commands = {"quote": quote}
 triggers = []
diff --git a/plugins/seen.py b/plugins/seen.py
new file mode 100644
index 0000000..4fd2db9
--- /dev/null
+++ b/plugins/seen.py
@@ -0,0 +1,56 @@
+from __future__ import division
+#from __future__ import print_function
+from __future__ import unicode_literals
+
+import time
+import datetime
+from util import timesince
+
+
+from botmily.db import db
+
+# Literally match any message from anyone:
+regex = r'(.+)'
+
+def getdelta(t):
+    delta = timesince.timesince(t) + " ago"
+    # Check if it's been over a month since we saw them:
+    days = datetime.timedelta(seconds=(time.time() - t)).days
+    # Consider hiding this is it's been fewer than some number of days?
+    if days >= 0:
+        t = int(t)
+        dt = datetime.datetime.fromtimestamp(t)
+        delta = delta + " (" + dt.__str__() + ")"
+    
+    return delta
+
+def seeninput(message_data, bot):
+    # This could be improved but it doesn't matter:
+    db.execute("create table if not exists seen(name text, time integer, quote text, chan text, primary key(name, chan))")
+    db.execute("replace into seen(name, time, quote, chan) values (:name, :time, :quote, :chan)", {"name": message_data["nick"], "time": int(time.time()), "quote": message_data["message"], "chan": message_data['channel']})
+    db.commit()
+    pass
+
+def seen(message_data, bot):
+    searchname = message_data['parsed'].strip()
+    chan = message_data['channel']
+
+    if searchname.lower() == bot.nickname.lower(): # user is looking for us, being a smartass
+        return "You need to get your eyes checked."
+
+    if searchname.lower() == message_data['nick'].lower():
+        return "Have you looked in a mirror lately?"
+
+    last_seen = db.execute("select name, time, quote from seen where name like ? and chan = ?", (searchname, chan)).fetchone()
+    
+    if last_seen:
+        reltime = getdelta(last_seen[1])
+        if last_seen[0] != searchname.lower():  # for glob matching
+            searchname = last_seen[0]
+        return '%s was last seen %s saying: %s' % \
+                    (searchname, reltime, last_seen[2])
+    else:
+        return "I've never seen %s" % searchname
+
+commands = {"seen": seen}
+triggers = [(regex, seeninput)]
\ No newline at end of file
diff --git a/plugins/somethingawful.py b/plugins/somethingawful.py
new file mode 100644
index 0000000..5e93e70
--- /dev/null
+++ b/plugins/somethingawful.py
@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+#copied from fixed version of .urban
+
+
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import re
+
+from urllib2 import urlopen
+
+from bs4 import BeautifulSoup
+from util import urlnorm, http
+from botmily import config
+
+
+# regex for detecting websites
+regex = r"(?i)forums\.somethingawful\.com/\S+threadid=(\d+)"
+showthread = "http://forums.somethingawful.com/showthread.php?noseen=1"
+
+def somethingawful(message_data, bot):
+	if message_data["parsed"][:5] == "post ":
+		try:
+			n = int(message_data["parsed"][5:])
+			output = 'http://forums.somethingawful.com/showthread.php?action=showpost&postid=' + message_data["parsed"][5:]
+		except ValueError:
+			output = 'Invalid input'
+	return output
+
+
+def login(user, password):
+	http.jar.clear_expired_cookies()
+	if any(cookie.domain == 'forums.somethingawful.com' and	cookie.name == 'bbuserid' for cookie in http.jar):
+		if any(cookie.domain == 'forums.somethingawful.com' and	cookie.name == 'bbpassword' for cookie in http.jar):
+			return
+		assert("malformed cookie jar")
+	http.get("http://forums.somethingawful.com/account.php", cookies=True, post_data="action=login&username=%s&password=%s" % (user, password))
+
+
+def urltranslatesa(message_data, bot):
+	#get url and normalize
+	url = urlnorm.normalize(message_data['re'].group().encode('utf-8'))
+	if config.sa_user is None or config.sa_password is None:
+		return
+	login(config.sa_user, config.sa_password)
+	thread = http.get_html(showthread, threadid=message_data['re'].group(1), perpage='1', cookies=True)
+	breadcrumbs = thread.xpath('//div[@class="breadcrumbs"]//a/text()')
+	if not breadcrumbs:
+		return
+	thread_title = breadcrumbs[-1]
+	forum_title = breadcrumbs[-2]
+	poster = thread.xpath('//dt[contains(@class, author)]//text()')[0]
+	# 1 post per page => n_pages = n_posts
+	num_posts = thread.xpath('//a[@title="Last page"]/@href')
+	print(num_posts)
+	if not num_posts:
+		num_posts = 1
+	else:
+		num_posts = int(num_posts[0].rsplit('=', 1)[1])
+	return '\x02%s\x02 > \x02%s\x02 by \x02%s\x02, %s post%s' % (forum_title, thread_title, poster, num_posts,'s' if num_posts > 1 else '')
+
+forum_abbrevs = {
+	'Serious Hardware / Software Crap': 'SHSC',
+	'The Cavern of COBOL': 'CoC',
+	'General Bullshit': 'GBS',
+	'Haus of Tech Support': 'HoTS'
+}
+commands = {"sa": somethingawful, "somethingawful": somethingawful}
+triggers = [(regex, urltranslatesa)]
\ No newline at end of file
diff --git a/plugins/steam.py b/plugins/steam.py
index c649728..fb2bc14 100755
--- a/plugins/steam.py
+++ b/plugins/steam.py
@@ -5,24 +5,23 @@
 from __future__ import unicode_literals
 
 import re
-import urllib2
+from urllib2 import urlopen
 
-from BeautifulSoup import BeautifulStoneSoup
+from bs4 import BeautifulSoup
 
 from botmily import irc
 
 def steam(message_data, bot):
-    try:
-        result = urllib2.urlopen('http://www.steamcalculator.com/id/' + message_data["parsed"])
-    except urllib2.URLError:
-        return "Nothing found sorry"
-    soup = BeautifulStoneSoup(result, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)
-    data = soup.find('div', id='rightdetail')
-    game_count = re.search('Found ([0-9]+)', data.text).group(1)
-    output = irc.bold(message_data["parsed"]) + ' owns ' + irc.bold(game_count) + ' Games with a value of ' + irc.bold(re.search('\$.*', data.text).group(0)) + '.'
-    if int(game_count) >= 125:
-        output += ' <--- jesus fuck quit buying games you neckbeard.'
-    return output
+	result = urlopen('http://alabasterslim.com/worth.php?account=' + message_data["parsed"])
+	soup = BeautifulSoup(result.read())
+	error = soup.find('div',id='centredetail')
+	if error is not None:
+		output = 'Nothing found'
+	else:
+		data = soup.find_all('fieldset')
+		resultfield = data[1].text.encode('ascii','ignore')
+		output = message_data["parsed"].encode('utf-8') + ' owns' + resultfield[resultfield.find('You own')+7:resultfield.find(' - What does this')].encode('utf-8') + ' - http://alabasterslim.com/worth.php?account=' + message_data["parsed"].encode('utf-8')
+	return output
 
 commands = {"sc": steam, "steamcalc": steam}
 triggers = []
diff --git a/plugins/test.py b/plugins/test.py
new file mode 100644
index 0000000..1ffb748
--- /dev/null
+++ b/plugins/test.py
@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import time
+
+from botmily.db import db
+
+def quote(message_data, bot):
+    return message_data.__str__()
+
+commands = {"test": quote}
+triggers = []
+
diff --git a/plugins/tumblrdottext.py b/plugins/tumblrdottext.py
index 35e10ad..a222f1e 100644
--- a/plugins/tumblrdottext.py
+++ b/plugins/tumblrdottext.py
@@ -11,7 +11,7 @@
 def tumblr(message_data, bot):
     t = Twitter(api_version=1.1, auth=OAuth(config.oauth_token,
     config.oauth_secret, config.consumer_key, config.consumer_secret))
-    tweetno = random.randint(0,199)
+    tweetno = random.randint(8000,8099)
     tweet = t.statuses.user_timeline.tumblrtxt(count=200)[tweetno]
     return tweet['text']
 
diff --git a/plugins/urban.py b/plugins/urban.py
index b720e5f..a87d03c 100755
--- a/plugins/urban.py
+++ b/plugins/urban.py
@@ -7,14 +7,15 @@
 import re
 from urllib2 import urlopen
 
-from BeautifulSoup import BeautifulStoneSoup
+from bs4 import BeautifulSoup
 
 def urban(message_data, bot):
   result = urlopen('http://www.urbandictionary.com/define.php?term=' + message_data["parsed"])
-  soup = BeautifulStoneSoup(result, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)
-  definition = soup.find('div', attrs={'class': 'definition'})
+  soup = BeautifulSoup(result.read())
+  wordresult = soup.find('a', class_='word')
+  definition = soup.find('div', class_='meaning')
   if definition:
-    return definition.text
+    return wordresult.text.replace('\n','') + ': ' + definition.text.replace('\n','')
   else:
     return "Nothing found"
 
diff --git a/plugins/urlhistory.py b/plugins/urlhistory.py
new file mode 100644
index 0000000..e4750bf
--- /dev/null
+++ b/plugins/urlhistory.py
@@ -0,0 +1,96 @@
+from __future__ import division
+#from __future__ import print_function
+#from __future__ import unicode_literals
+
+import time
+import datetime
+import math
+from util import timesince, urlnorm
+
+from botmily.db import db
+
+# regex for detecting websites
+regex = r'([a-zA-Z]+://|www\.)[^ ]+'
+
+# Some kind of ignored url stuff that idk what is but whatever:
+ignored_urls = [urlnorm.normalize("http://google.com")]
+
+expiration_period = 60 * 60 * 24  # 1 day
+
+def db_init(db):
+    db.execute("create table if not exists urlhistory"
+                 "(chan, url, nick, time)")
+    db.commit()
+
+def insert_history(db, chan, url, nick):
+    now = time.time()
+    db.execute("insert into urlhistory(chan, url, nick, time) "
+                 "values(?,?,?,?)", (chan, url, nick, time.time()))
+    db.commit()
+
+def get_history(db, chan, url):
+    db.execute("delete from urlhistory where time < ?",
+                 (time.time() - expiration_period,))
+    return db.execute("select nick, time from urlhistory where "
+            "chan=? and url=? order by time desc", (chan, url)).fetchall()
+
+
+def getdelta(t):
+    # Make it easy to change this if we have to;
+    delta = timesince.timesince(t)
+    return delta
+
+def nicklist(nicks):
+    nicks = sorted(dict(nicks), key=unicode.lower)
+    if len(nicks) <= 2:
+        return ' and '.join(nicks)
+    else:
+        return ', and '.join((', '.join(nicks[:-1]), nicks[-1]))
+
+
+def format_reply(history):
+    if not history:
+        return
+
+    last_nick, recent_time = history[0]
+    last_time = getdelta(recent_time)
+
+    if len(history) == 1:
+        return "%s linked that %s ago." % (last_nick, last_time)
+
+    hour_span = math.ceil((time.time() - history[-1][1]) / 3600)
+    hour_span = '%.0f hours' % hour_span if hour_span > 1 else 'hour'
+
+    hlen = len(history)
+    ordinal = ["once", "twice", "%d times" % hlen][min(hlen, 3) - 1]
+
+    if len(dict(history)) == 1:
+        last = "last linked %s ago" % last_time
+    else:
+        last = "last linked by %s %s ago" % (last_nick, last_time)
+
+    return "that url has been posted %s in the past %s by %s (%s)." % (ordinal,
+            hour_span, nicklist(history), last)
+
+def urlinput(message_data, bot):
+    # Verify the database exists;
+    db_init(db)
+
+    # normalise the url
+    url = urlnorm.normalize(message_data['re'].group().encode('utf-8'))
+
+    if url not in ignored_urls:
+        url = url.decode('utf-8')
+
+        # Load our primitives from message_data
+        chan = message_data['channel']
+        nick = message_data['nick']
+
+        history = get_history(db, chan, url)
+        insert_history(db, chan, url, nick)
+        if nick not in dict(history):
+            return format_reply(history)
+
+
+commands = {}
+triggers = [(regex, urlinput)]
\ No newline at end of file
diff --git a/plugins/weather.py b/plugins/weather.py
index 046f897..52234fb 100755
--- a/plugins/weather.py
+++ b/plugins/weather.py
@@ -31,16 +31,32 @@ def weather(message_data, bot):
     except ElementTree.ParseError:
         return "Error getting weather data"
     current_observation = weather.find('current_observation')
-    if current_observation is None:
-        return "Error getting weather data"
-    display_location = current_observation.find('display_location')
-    string = display_location.find('full').text + ': '
-    string = string + current_observation.find('weather').text + ', '
-    string = string + current_observation.find('temperature_string').text + ', '
-    string = string + current_observation.find('relative_humidity').text + ', Wind is blowing '
-    string = string + current_observation.find('wind_string').text.replace('F','f', 1) + '.'
+    results = weather.find('results')
+    if current_observation is not None:
+        display_location = current_observation.find('display_location')
+        string = display_location.find('full').text + ': '
+        string += current_observation.find('weather').text + ', '
+        string += current_observation.find('temperature_string').text + ', '
+        string += current_observation.find('relative_humidity').text + ', Wind is blowing '
+        string += current_observation.find('wind_string').text.replace('F','f', 1) + '.'
+    elif results is not None:
+        string = "Found the following cities/locations, please specify: "
+        for r in results:
+            string += "[ " + r.find('name').text + ', ' + r.find('city').text + ', '
+            state = r.find('state')
+            if state.text is not None:
+                string += state.text + ', '
+            string += r.find('country_name').text + "(" + r.find('country').text + ") ] "
+    else:
+        string = "City/location not found"
     return string
 
 commands = {"weather": weather}
 triggers = []
 
+if __name__ == '__main__':
+    import sys
+    if (len(sys.argv) < 2):
+        print("Please provide search-argument")
+        sys.exit(1)
+    print(weather({"parsed" : sys.argv[1]},None))
diff --git a/plugins/youtube.py b/plugins/youtube.py
deleted file mode 100755
index bc3b299..0000000
--- a/plugins/youtube.py
+++ /dev/null
@@ -1,61 +0,0 @@
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-import locale
-import re
-import time
-from datetime import datetime, timedelta
-from gdata.youtube import service
-
-from botmily import irc
-
-regex = r'(?:youtube.*?(?:v=|/v/)|youtu\.be/|yooouuutuuube.*?id=)([-_a-z0-9]+)'
-
-def convertHMS(secs):
-	sec = timedelta(seconds=int(secs))
-	d = datetime(1,1,1) + sec
-	if d.day-1 > 0:
-		return '%d days %d hours %d minutes %d seconds' %(d.day-1,d.hour,d.minute,d.second)
-	elif d.hour > 0:
-		return '%d hours %d minutes %d seconds' %(d.hour,d.minute,d.second)
-	elif d.minute > 0:
-		return '%d minutes %d seconds' %(d.minute,d.second)
-	else:	
-		return '%d seconds' %d.second 
-  
-def search(message_data, bot):
-	yt_service = service.YouTubeService()
-	query = service.YouTubeVideoQuery()
-	query.vq = message_data["parsed"]
-	query.orderby = 'relevance'
-	query.racy = 'include'
-	feed = yt_service.YouTubeQuery(query)
-	if len(feed.entry) == 0:
-		return "No results"
-	title = feed.entry[0].title.text
-	link = feed.entry[0].link[0].href
-	return "\u0002%s\u000f - %s" %(unicode(title, encoding='utf-8') , link)
-
-def parse(message_data, bot):
-	id = message_data["re"].group(1)
-	youtube = service.YouTubeService()
-	youtube.ssl = True
-	try:
-		entry = youtube.GetYouTubeVideoEntry(video_id=id)
-	except service.RequestError:
-		return None
-	string = irc.bold(unicode(entry.media.title.text, encoding='utf-8')) + " - length "
-	string += irc.bold(convertHMS(entry.media.duration.seconds)) + " - rated "
-	if entry.rating is not None:
-		string += irc.bold(locale.format("%.2f", float(entry.rating.average))) + "/5.0 ("
-		string += entry.rating.num_raters + ") - "
-	if entry.statistics is not None:
-		string += irc.bold(locale.format("%d", float(entry.statistics.view_count), True)) + " views - "
-	string += irc.bold(unicode(entry.author[0].name.text, encoding='utf-8')) + " on "
-	string += irc.bold(time.strftime("%Y.%m.%d", time.strptime(entry.published.text, "%Y-%m-%dT%H:%M:%S.000Z")))
-	return string
-
-commands = {"yt": search, "youtube": search}
-triggers = [(regex, parse)]
-
diff --git a/tumble.py b/tumble.py
deleted file mode 100644
index 7bc98e2..0000000
--- a/tumble.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import tumblr
-import urllib2 , urllib
-from PIL import Image
-from cStringIO import StringIO
-from makeMacro import overlayAchieve
-import pycurl
-def postToImgur(filename):    
-    store = StringIO()
-    c = pycurl.Curl()
-    values = [
-              ("key", "a95e5a7d90d1f821714e449bb47e1051"),
-              ("image", (c.FORM_FILE, filename))]
-    c.setopt(c.URL, "http://api.imgur.com/2/upload.xml")
-    c.setopt(c.HTTPPOST, values)
-    c.setopt(c.WRITEFUNCTION,store.write)
-    c.perform()
-    c.close()
-
-    retval = store.getvalue()
-    originalindex = retval.rfind('<original>')
-    endindex = retval.find('</original>')
-    urlneeded = retval[originalindex+10:endindex]
-    return urlneeded
-
-def makePost(user,password,blog,title,imgUrl,caption,tumblrCaption):
-    filename = overlayAchieve(caption,imgUrl)
-    url = 'http://www.tumblr.com/api/write'
-    imageurl = postToImgur('./%s'%filename)
-    vals = {
-                'email': user,
-                'password': password,
-                'type': 'photo',
-                'source': imageurl,
-                'caption':tumblrCaption,
-                'group':blog
-            }
-    data = urllib.urlencode(vals)
-    req = urllib2.Request(url,data)
-    try:
-            response = urllib2.urlopen(req)
-            postId = response.read()
-            postUrl = blog + '/post/%s' %postId
-            return postUrl
-    except urllib2.URLError, e:
-            print e.code
-            print e.read()
-            return None
-        
-def getLastPost(blogName):        
-    blogApi= tumblr.Api(blogName)
-    blog = blogApi.read()
-    lastpost = blog.next()
-    return unicode(lastpost['url-with-slug']) , unicode(lastpost['photo-url-1280']) , unicode(lastpost['url-with-slug'])
\ No newline at end of file
diff --git a/util/__init__.py b/util/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/util/http.py b/util/http.py
new file mode 100644
index 0000000..8ca0c63
--- /dev/null
+++ b/util/http.py
@@ -0,0 +1,75 @@
+import cookielib
+import json
+import urllib
+import urllib2
+import urlparse
+
+from urllib import quote, quote_plus as _quote_plus
+from urllib2 import HTTPError, URLError
+
+from lxml import etree, html
+
+user_agent = 'Skybot/1.0 http://github.com/rmmh/skybot'
+
+ua_firefox = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6'
+ua_internetexplorer = 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
+
+jar = cookielib.CookieJar()
+
+
+def get(*args, **kwargs):
+	return open(*args, **kwargs).read()
+
+
+def get_html(*args, **kwargs):
+	return html.fromstring(get(*args, **kwargs))
+
+
+def get_xml(*args, **kwargs):
+	return etree.fromstring(get(*args, **kwargs))
+
+
+def get_json(*args, **kwargs):
+	return json.loads(get(*args, **kwargs))
+
+
+def open(url, query_params=None, user_agent=user_agent, post_data=None,
+		get_method=None, cookies=False, **kwargs):
+
+	if query_params is None:
+		query_params = {}
+	query_params.update(kwargs)
+	url = prepare_url(url, query_params)
+	request = urllib2.Request(url, post_data)
+
+	if get_method is not None:
+		request.get_method = lambda: get_method
+	request.add_header('User-Agent', user_agent)
+	if cookies:
+		opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(jar))
+	else:
+		opener = urllib2.build_opener()
+	return opener.open(request)
+
+
+def prepare_url(url, queries):
+	if queries:
+		scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
+		query = dict(urlparse.parse_qsl(query))
+		query.update(queries)
+		query = urllib.urlencode(dict((to_utf8(key), to_utf8(value))
+									for key, value in query.iteritems()))
+		url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))
+
+	return url
+
+
+def to_utf8(s):
+	if isinstance(s, str):
+		return s
+	else:
+		return s.encode('utf8', 'ignore')
+
+
+def quote_plus(s):
+	return _quote_plus(to_utf8(s))
\ No newline at end of file
diff --git a/util/timesince.py b/util/timesince.py
new file mode 100644
index 0000000..a3cea46
--- /dev/null
+++ b/util/timesince.py
@@ -0,0 +1,102 @@
+# Copyright (c) Django Software Foundation and individual contributors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#  1. Redistributions of source code must retain the above copyright notice,
+#     this list of conditions and the following disclaimer.
+#
+#  2. Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#
+#  3. Neither the name of Django nor the names of its contributors may be used
+#     to endorse or promote products derived from this software without
+#     specific prior written permission.
+#
+#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"AND
+#ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+#WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+#DISCLAIMED.IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+#ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+#(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+#LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+#ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+#SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import datetime
+
+
+def timesince(d, now=None):
+    """
+    Takes two datetime objects and returns the time between d and now
+    as a nicely formatted string, e.g. "10 minutes".  If d occurs after now,
+    then "0 minutes" is returned.
+
+    Units used are years, months, weeks, days, hours, and minutes.
+    Seconds and microseconds are ignored.  Up to two adjacent units will be
+    displayed.  For example, "2 weeks, 3 days" and "1 year, 3 months" are
+    possible outputs, but "2 weeks, 3 hours" and "1 year, 5 days" are not.
+
+    Adapted from http://blog.natbat.co.uk/archive/2003/Jun/14/time_since
+    """
+    chunks = (
+      (60 * 60 * 24 * 365, ('year', 'years')),
+      (60 * 60 * 24 * 30, ('month', 'months')),
+      (60 * 60 * 24 * 7, ('week', 'weeks')),
+      (60 * 60 * 24, ('day', 'days')),
+      (60 * 60, ('hour', 'hours')),
+      (60, ('minute', 'minutes'))
+    )
+
+    # Convert int or float (unix epoch) to datetime.datetime for comparison
+    if isinstance(d, int) or isinstance(d, float):
+        d = datetime.datetime.fromtimestamp(d)
+
+    # Convert datetime.date to datetime.datetime for comparison.
+    if not isinstance(d, datetime.datetime):
+        d = datetime.datetime(d.year, d.month, d.day)
+    if now and not isinstance(now, datetime.datetime):
+        now = datetime.datetime(now.year, now.month, now.day)
+
+    if not now:
+        now = datetime.datetime.now()
+
+    # ignore microsecond part of 'd' since we removed it from 'now'
+    delta = now - (d - datetime.timedelta(0, 0, d.microsecond))
+    since = delta.days * 24 * 60 * 60 + delta.seconds
+    if since <= 0:
+        # d is in the future compared to now, stop processing.
+        return u'0 ' + 'minutes'
+    for i, (seconds, name) in enumerate(chunks):
+        count = since // seconds
+        if count != 0:
+            break
+
+    if count == 1:
+        s = '%(number)d %(type)s' % {'number': count, 'type': name[0]}
+    else:
+        s = '%(number)d %(type)s' % {'number': count, 'type': name[1]}
+
+    if i + 1 < len(chunks):
+        # Now get the second item
+        seconds2, name2 = chunks[i + 1]
+        count2 = (since - (seconds * count)) // seconds2
+        if count2 != 0:
+            if count2 == 1:
+                s += ', %d %s' % (count2, name2[0])
+            else:
+                s += ', %d %s' % (count2, name2[1])
+    return s
+
+
+def timeuntil(d, now=None):
+    """
+    Like timesince, but returns a string measuring the time until
+    the given time.
+    """
+    if not now:
+        now = datetime.datetime.now()
+    return timesince(now, d)
diff --git a/util/urlnorm.py b/util/urlnorm.py
new file mode 100644
index 0000000..4089710
--- /dev/null
+++ b/util/urlnorm.py
@@ -0,0 +1,133 @@
+"""
+URI Normalization function:
+ * Always provide the URI scheme in lowercase characters.
+ * Always provide the host, if any, in lowercase characters.
+ * Only perform percent-encoding where it is essential.
+ * Always use uppercase A-through-F characters when percent-encoding.
+ * Prevent dot-segments appearing in non-relative URI paths.
+ * For schemes that define a default authority, use an empty authority if the
+   default is desired.
+ * For schemes that define an empty path to be equivalent to a path of "/",
+   use "/".
+ * For schemes that define a port, use an empty port if the default is desired
+ * All portions of the URI must be utf-8 encoded NFC from Unicode strings
+
+implements:
+  http://gbiv.com/protocols/uri/rev-2002/rfc2396bis.html#canonical-form
+  http://www.intertwingly.net/wiki/pie/PaceCanonicalIds
+
+inspired by:
+  Tony J. Ibbs,    http://starship.python.net/crew/tibs/python/tji_url.py
+  Mark Nottingham, http://www.mnot.net/python/urlnorm.py
+"""
+
+__license__ = "Python"
+
+import re
+import unicodedata
+import urlparse
+from urllib import quote, unquote
+
+default_port = {
+    'http': 80,
+}
+
+
+class Normalizer(object):
+    def __init__(self, regex, normalize_func):
+        self.regex = regex
+        self.normalize = normalize_func
+
+normalizers = ( Normalizer( re.compile(r'(?:https?://)?(?:[a-zA-Z0-9\-]+\.)?(?:amazon|amzn){1}\.(?P<tld>[a-zA-Z\.]{2,})\/(gp/(?:product|offer-listing|customer-media/product-gallery)/|exec/obidos/tg/detail/-/|o/ASIN/|dp/|(?:[A-Za-z0-9\-]+)/dp/)?(?P<ASIN>[0-9A-Za-z]{10})'),
+                            lambda m: r'http://amazon.%s/dp/%s' % (m.group('tld'), m.group('ASIN'))),
+                Normalizer( re.compile(r'.*waffleimages\.com.*/([0-9a-fA-F]{40})'),
+                            lambda m: r'http://img.waffleimages.com/%s' % m.group(1) ),
+                Normalizer( re.compile(r'(?:youtube.*?(?:v=|/v/)|youtu\.be/|yooouuutuuube.*?id=)([-_a-z0-9]+)'),
+                            lambda m: r'http://youtube.com/watch?v=%s' % m.group(1) ),
+    )
+
+
+def normalize(url):
+    """Normalize a URL."""
+
+    scheme, auth, path, query, fragment = urlparse.urlsplit(url.strip())
+    userinfo, host, port = re.search('([^@]*@)?([^:]*):?(.*)', auth).groups()
+
+    # Always provide the URI scheme in lowercase characters.
+    scheme = scheme.lower()
+
+    # Always provide the host, if any, in lowercase characters.
+    host = host.lower()
+    if host and host[-1] == '.':
+        host = host[:-1]
+    if host and host.startswith("www."):
+        if not scheme:
+            scheme = "http"
+        host = host[4:]
+    elif path and path.startswith("www."):
+        if not scheme:
+            scheme = "http"
+        path = path[4:]
+
+    # Only perform percent-encoding where it is essential.
+    # Always use uppercase A-through-F characters when percent-encoding.
+    # All portions of the URI must be utf-8 encoded NFC from Unicode strings
+    def clean(string):
+        string = unicode(unquote(string), 'utf-8', 'replace')
+        return unicodedata.normalize('NFC', string).encode('utf-8')
+    path = quote(clean(path), "~:/?#[]@!$&'()*+,;=")
+    fragment = quote(clean(fragment), "~")
+
+    # note care must be taken to only encode & and = characters as values
+    query = "&".join(["=".join([quote(clean(t), "~:/?#[]@!$'()*+,;=")
+        for t in q.split("=", 1)]) for q in query.split("&")])
+
+    # Prevent dot-segments appearing in non-relative URI paths.
+    if scheme in ["", "http", "https", "ftp", "file"]:
+        output = []
+        for input in path.split('/'):
+            if input == "":
+                if not output:
+                    output.append(input)
+            elif input == ".":
+                pass
+            elif input == "..":
+                if len(output) > 1:
+                    output.pop()
+            else:
+                output.append(input)
+        if input in ["", ".", ".."]:
+            output.append("")
+        path = '/'.join(output)
+
+    # For schemes that define a default authority, use an empty authority if
+    # the default is desired.
+    if userinfo in ["@", ":@"]:
+        userinfo = ""
+
+    # For schemes that define an empty path to be equivalent to a path of "/",
+    # use "/".
+    if path == "" and scheme in ["http", "https", "ftp", "file"]:
+        path = "/"
+
+    # For schemes that define a port, use an empty port if the default is
+    # desired
+    if port and scheme in default_port.keys():
+        if port.isdigit():
+            port = str(int(port))
+            if int(port) == default_port[scheme]:
+                port = ''
+
+    # Put it all back together again
+    auth = (userinfo or "") + host
+    if port:
+        auth += ":" + port
+    if url.endswith("#") and query == "" and fragment == "":
+        path += "#"
+    normal_url = urlparse.urlunsplit((scheme, auth, path, query,
+        fragment)).replace("http:///", "http://")
+    for norm in normalizers:
+        m = norm.regex.match(normal_url)
+        if m:
+            return norm.normalize(m)
+    return normal_url