From 16625108476eca6b623b197a0622e7e66b4fabde Mon Sep 17 00:00:00 2001 From: seba Date: Thu, 22 Dec 2011 05:32:32 +0100 Subject: [PATCH] Massive bugfixes for twitter-rfc1149 --- tunnel/rfc1149/rfc1149.py | 52 ++++++++++++++++++++++------- tunnel/rfc1149/uphelper.py | 67 ++++++++++++++++++++++++++++---------- 2 files changed, 91 insertions(+), 28 deletions(-) diff --git a/tunnel/rfc1149/rfc1149.py b/tunnel/rfc1149/rfc1149.py index 3af1957..2dea1e5 100755 --- a/tunnel/rfc1149/rfc1149.py +++ b/tunnel/rfc1149/rfc1149.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import base64 +import collections import logging import math import re @@ -21,13 +22,34 @@ class TwittStreamHandler(tweepy.StreamListener): def __init__(self, dev): super(TwittStreamHandler, self).__init__() self.dev = dev - self.fragments = {} + self.fragments = collections.defaultdict(str) def on_status( self, status ): - print '-' * 20 - print "incoming:", unicode(status.text) - print "hex:", binToHexStr(status.text), len(status.text) - # reassemble messages, put them together if nessecary. + #print '-' * 20 + #print "incoming:", unicode(status.text), "from", dir(status) + #print "hex:", binToHexStr(status.text), len(status.text) + # reassemble messages, write them to dev when complete + (isFragment, packetLen, packetId, packet) = None, None, None, None + try: + (isFragment, packetLen, packetId, packet) = UPHelper.decode(status.text) + except ValueError, e: + print "Could not decode tweet, omitting (Error was: %s).\n\tText was: %s" % (e, status.text) + raise + return + #print "Parsed packet:", (isFragment, packetLen, packetId) + #print "\t contents:", packet + if isFragment: + self.fragments[packetId] += packet + print " >+ Added fragment with id", packetId + else: + toSend = None + if self.fragments.has_key(packetId): + toSend = self.fragments[packetId] + packet + else: + toSend = packet + print " >> Received packet with id", packetId + print repr(toSend) + self.dev.write(toSend) def on_limit(self, track): print "We got limited ", track @@ -41,7 +63,7 @@ class TwittStreamHandler(tweepy.StreamListener): sys.exit(2) else: print "At the moment there is no error-handling for this, so we just kill everything. Remember: This software doesn't even deserve the label 'alpha' ;)" - sys.exit(1) + #sys.exit(1) def on_timeout(self, status_code): print "Got an timeout: ", status_code @@ -51,15 +73,18 @@ class DownstreamThread(threading.Thread): def __init__(self, dev, auth, endpoint): threading.Thread.__init__(self) self.auth = auth + self.api = tweepy.API(self.auth) self.endpoint = endpoint self.dev = dev self.daemon = True def run(self): stream = tweepy.Stream(auth=self.auth, listener=TwittStreamHandler(self.dev)) - stream.userstream() - stream.filter(self.endpoint, None) - stream.sample() + user = self.api.get_user(self.endpoint) + print "Endpoint is", self.endpoint, "with id", user.id + stream.filter([user.id]) + #stream.userstream() + #stream.sample() class RFC1149(Ether2Any): def __init__(self): @@ -113,14 +138,19 @@ if not Conf['twitter']['ACCESS_KEY']: authConf.close() self.auth.set_access_token(self.twitterConf['ACCESS_KEY'], self.twitterConf['ACCESS_SECRET']) self.api = tweepy.API(self.auth) + self.me = self.api.me() + print "Logged in as %s" % (self.me.screen_name,) def _dec(self, s): """ ... """ return "".join(map(lambda x: chr(x+48), reversed(s))) def sendToNet(self, packet): - for subPacket in UPHelper.encode(packet): - print subPacket + fragments = UPHelper.encode(packet) + print " >> Sending out %d bytes in %d tweet%s" % (len(packet), len(fragments), len(fragments)!=1 and "s" or "") + for fragment in fragments: + # FIXME: catch tweepy.error.TweepError + self.api.update_status(fragment) if __name__ == '__main__': rfc = RFC1149() diff --git a/tunnel/rfc1149/uphelper.py b/tunnel/rfc1149/uphelper.py index c46ac6d..a6e9b11 100644 --- a/tunnel/rfc1149/uphelper.py +++ b/tunnel/rfc1149/uphelper.py @@ -1,8 +1,8 @@ #!/usr/bin/python # -*- coding: utf-8 -*- -import random import bitarray +import random class UPHelper(): """ The Unicode Packet Helper @@ -15,7 +15,7 @@ class UPHelper(): The header in the metadata is as following: <9 bits length of payload> - <32 bit random paket id>""" + <32 bit random paket id greater than 0>""" @staticmethod def intToBits(n, length): @@ -36,10 +36,16 @@ class UPHelper(): ret = (ret << 1) | (i and 1 or 0) return ret + @staticmethod + def toBin(val, leading): + """ Binary formatter for python versions without str.format() """ + x = bin(val).replace("0b", "") + return "0" * (leading-len(x)) + x + @staticmethod def encode(data): """ Generate list of packets with a header from data. """ - packetId = random.randint(0, 2**32) + packetId = random.randint(1, 2**32) fragments = [] while len(data) > 280: fragments.append(data[0:280]) @@ -60,7 +66,7 @@ class UPHelper(): # write header (bits: 1 fragment, 9 length, 32 id) header = bitarray.bitarray(1) # write fragment-bit - header[0] = (y+1 == len(fragments)) + header[0] = not (y+1 == len(fragments)) # append packet length header.extend(UPHelper.intToBits(lenX, 9)) # add packet id @@ -90,27 +96,54 @@ class UPHelper(): raise ValueError("This is not a valid packet, header is too short (should be at least 11, is %d)" % len(packet)) header = bitarray.bitarray() for i in range(11): - header.extend("{:04b}".format(ord(packet[i]) >> 16)) + # format with binary is not understood by older python versions + #header.extend("{:04b}".format(ord(packet[i]) >> 16)) + header.extend(UPHelper.toBin((ord(packet[i]) >> 16), 4)) isFragmented = header[0] - packetLen = UPHelper.bitsToInt(header[1:9]) - packetId = UPHelper.bitsToInt(header[9:]) + packetLen = UPHelper.bitsToInt(header[1:10]) + packetId = UPHelper.bitsToInt(header[10:]) + if packetId == 0: + raise ValueError("Packet id cannot be 0") + rawData = map(lambda x: ord(x) & 0xFFFF, packet) data = [] for p in rawData: data.append(chr(p >> 8)) data.append(chr(p & 255)) data = "".join(data) - - return (isFragmented, packetLen, packetId, data) + return (isFragmented, packetLen, packetId, data[:packetLen]) if __name__ == '__main__': - print UPHelper.encode("foo") - print UPHelper.decode(UPHelper.encode("foo")[0]) - msg = "".join([chr(i) for i in range(256)]) - msg += "".join([chr(i) for i in range(256)]) - p = UPHelper.encode(msg) - print repr(msg) - for x in p: - print UPHelper.decode(x) + msg = '\x00\x00\x08\x00E\x00\x00T\x00\x00@\x00@\x01\x12\x81\n\n\n\x0b\n\n\n\n\x08\x00\xd7Gt\xd2\x00\x01[U\xf1Nl=\x08\x00\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./01234567' + msg = u'\U000c1011\U00061213\U00011415\U000a1617\U00031819\U00041a1b\U000a1c1d\U000a1e1f\U000c2021\u2223\U000c2425\u2627\u2829\u2a2b\u2c2d\u2e2f\u3031\u3233\u3435\u3637\u3839\u3a3b\u3c3d\u3e3f\u4041\u4243\u4445\u4647\u4849\u4a4b\u4c4d\u4e4f\u5051\u5253\u5455\u5657\u5859\u5a5b\u5c5d\u5e5f\u6061\u6263\u6465\u6667\u6869\u6a6b\u6c6d\u6e6f\u7071\u7273\u7475\u7677\u7879\u7a7b\u7c7d\u7e7f\u8081\u8283\u8485\u8687\u8889\u8a8b\u8c8d\u8e8f\u9091\u9293\u9495\u9697\u9899\u9a9b\u9c9d\u9e9f\ua0a1\ua2a3\ua4a5\ua6a7\ua8a9\uaaab\uacad\uaeaf\ub0b1\ub2b3\ub4b5\ub6b7\ub8b9\ubabb\ubcbd\ubebf\uc0c1\uc2c3\uc4c5\uc6c7\uc8c9\ucacb\ucccd\ucecf\ud0d1\ud2d3\ud4d5\ud6d7\ud8d9\udadb\udcdd\udedf\ue0e1\ue2e3\ue4e5\ue6e7\ue8e9\ueaeb\ueced\ueeef\uf0f1\uf2f3\uf4f5\uf6f7\uf8f9\ufafb\ufcfd\ufeff\x01\u0203\u0405\u0607\u0809\u0a0b\u0c0d\u0e0f\u1011\u1213\u1415\u1617\u1819\u1a1b\u1c1d\u1e1f\u2021\u2223\u2425\u2627' + msg = '\x00\x00\x08\x00E\x00\x04\x04\x00\x00@\x00@\x01\x0e\xd1\n\n\n\x0b\n\n\n\n\x08\x005f\x03\xec\x00\x01\xe7\xf7\xf1NR\xf2\r\x00\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7' + enc = UPHelper.encode(msg) + print "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + print "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + print "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + print enc + ret = "" + for e in enc: + e = UPHelper.decode(e) + print e[0:3] + print e[3] + ret += e[3] + m = UPHelper.decode(UPHelper.encode(msg)[0]) + #print m + if ret == msg: + print "success" + else: + print "failure" + print "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + print "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + print "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + print UPHelper.decode(UPHelper.encode("X"*281)[0]) + print UPHelper.decode(UPHelper.encode("X"*281)[1]) + #msg = "".join([chr(i) for i in range(256)]) + #msg += "".join([chr(i) for i in range(256)]) + #p = UPHelper.encode(msg) + #print repr(msg) + #for x in p: + # print UPHelper.decode(x)