From 9ff13a0ee4dfcccce7baba83a86d773ad5576a13 Mon Sep 17 00:00:00 2001 From: Redlink Date: Sun, 18 Feb 2018 14:42:36 +0100 Subject: [PATCH 01/32] add param to hide membership changes in tg --- config.json.example | 2 ++ telematrix/__init__.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/config.json.example b/config.json.example index 67378e0..624d166 100644 --- a/config.json.example +++ b/config.json.example @@ -15,5 +15,7 @@ "user_id_format": "@telegram_{}:DOMAIN.TLD", "db_url": "sqlite:///database.db", + "hide_membership_changes": "True", + "as_port": 5000 } diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 75ef826..f5f68f2 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -44,6 +44,7 @@ USER_ID_FORMAT = CONFIG['user_id_format'] DATABASE_URL = CONFIG['db_url'] + HIDE_MEMBERSHIP_CHANGES = CONFIG['hide_membership_changes'] AS_PORT = CONFIG['as_port'] if 'as_port' in CONFIG else 5000 except (OSError, IOError) as exception: @@ -209,7 +210,6 @@ async def matrix_transaction(request): if matrix_is_telegram(user_id): continue - sender = db.session.query(db.MatrixUser)\ .filter_by(matrix_id=user_id).first() @@ -265,6 +265,8 @@ async def matrix_transaction(request): print(json.dumps(content, indent=4)) elif event['type'] == 'm.room.member': + if HIDE_MEMBERSHIP_CHANGES: + continue if matrix_is_telegram(event['state_key']): continue @@ -538,6 +540,7 @@ async def aiotg_sticker(chat, sticker): db.session.add(message) db.session.commit() + @TG_BOT.handle('photo') async def aiotg_photo(chat, photo): link = db.session.query(db.ChatLink).filter_by(tg_room=chat.id).first() From d52538a86ee8fb866d84483953aead226853fae4 Mon Sep 17 00:00:00 2001 From: Redlink Date: Sun, 18 Feb 2018 19:58:33 +0100 Subject: [PATCH 02/32] add audio, video, file transfer from matrix to tg --- telematrix/__init__.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index f5f68f2..1ab488c 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -23,7 +23,7 @@ # Read the configuration file try: - with open('config.json', 'r') as config_file: + with open('config.json.example', 'r') as config_file: CONFIG = json.load(config_file) HS_TOKEN = CONFIG['tokens']['hs'] @@ -154,7 +154,12 @@ def get_username(user_id): 'image/tiff': 'tif', 'image/x-tiff': 'tif', 'image/bmp': 'bmp', - 'image/x-windows-bmp': 'bmp' + 'image/x-windows-bmp': 'bmp', + 'video/avi': 'avi', + 'msvideo/avi': 'avi', + 'x-msvideo/avi': 'avi', + 'video/mp4': 'mp4', + 'x-video/mp4': 'mp4', } async def matrix_transaction(request): @@ -238,26 +243,39 @@ async def matrix_transaction(request): elif content['msgtype'] == 'm.emote': msg, mode = format_matrix_msg('{}', content) response = await group.send_text("* {} {}".format(displayname, msg), parse_mode=mode) - elif content['msgtype'] == 'm.image': + elif content['msgtype'] in ['m.image', 'm.video', 'm.video', 'm.file']: try: url = urlparse(content['url']) # Append the correct extension if it's missing or wrong - ext = mime_extensions[content['info']['mimetype']] + try: + ext = mime_extensions[content['info']['mimetype']] + except KeyError: + ext = "" if not content['body'].endswith(ext): content['body'] += '.' + ext # Download the file await download_matrix_file(url, content['body']) - with open('/tmp/{}'.format(content['body']), 'rb') as img_file: + with open('/tmp/{}'.format(content['body']), 'rb') as file: # Create the URL and shorten it url_str = MATRIX_HOST_EXT + \ '_matrix/media/r0/download/{}{}' \ .format(url.netloc, quote(url.path)) url_str = await shorten_url(url_str) - caption = '{} sent an image'.format(displayname) - response = await group.send_photo(img_file, caption=caption) + if content['msgtype'] == 'm.image': + caption = '{} sent an image'.format(displayname) + response = await group.send_photo(file, caption=caption) + elif content['msgtype'] == 'm.video': + caption = '{} sent an video'.format(displayname) + response = await group.send_video(file, caption=caption) + elif content['msgtype'] == 'm.audio': + caption = '{} sent an audio file'.format(displayname) + response = await group.send_audio(file, caption=caption) + elif content['msgtype'] == 'm.file': + caption = '{} sent an file'.format(displayname) + response = await group.send_document(file, caption=caption) except: pass else: From 91dc366d76299ab48b7164d56905c5b238603196 Mon Sep 17 00:00:00 2001 From: Redlink Date: Sun, 18 Feb 2018 20:00:28 +0100 Subject: [PATCH 03/32] fix --- telematrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 1ab488c..64d22ee 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -23,7 +23,7 @@ # Read the configuration file try: - with open('config.json.example', 'r') as config_file: + with open('config.json', 'r') as config_file: CONFIG = json.load(config_file) HS_TOKEN = CONFIG['tokens']['hs'] From 48063c2845273af686e92d872b257ad8e3ba6bce Mon Sep 17 00:00:00 2001 From: Redlink Date: Sun, 18 Feb 2018 21:01:31 +0100 Subject: [PATCH 04/32] typo --- telematrix/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 64d22ee..ed1f915 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -268,13 +268,13 @@ async def matrix_transaction(request): caption = '{} sent an image'.format(displayname) response = await group.send_photo(file, caption=caption) elif content['msgtype'] == 'm.video': - caption = '{} sent an video'.format(displayname) + caption = '{} sent a video'.format(displayname) response = await group.send_video(file, caption=caption) elif content['msgtype'] == 'm.audio': caption = '{} sent an audio file'.format(displayname) response = await group.send_audio(file, caption=caption) elif content['msgtype'] == 'm.file': - caption = '{} sent an file'.format(displayname) + caption = '{} sent a file'.format(displayname) response = await group.send_document(file, caption=caption) except: pass @@ -471,6 +471,7 @@ async def register_join_matrix(chat, room_id, user_id): user_id, {'displayname': name}) await matrix_post('client', 'join/{}'.format(room_id), user_id, {}) + async def update_matrix_displayname_avatar(tg_user): name = tg_user['first_name'] if 'last_name' in tg_user: From 88a4b0ebe43150755442079af8541b5f1cbcf1e0 Mon Sep 17 00:00:00 2001 From: Redlink Date: Wed, 21 Feb 2018 10:49:02 +0100 Subject: [PATCH 05/32] new handler --- telematrix/__init__.py | 49 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index ed1f915..aa9b70a 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -607,6 +607,55 @@ async def aiotg_photo(chat, photo): db.session.add(message) db.session.commit() + +@TG_BOT.handle('video') +async def aiotg_photo(chat, video): + link = db.session.query(db.ChatLink).filter_by(tg_room=chat.id).first() + if not link: + print('Unknown telegram chat {}: {}'.format(chat, chat.id)) + return + + await update_matrix_displayname_avatar(chat.sender); + room_id = link.matrix_room + user_id = USER_ID_FORMAT.format(chat.sender['id']) + txn_id = quote('{}{}'.format(chat.message['message_id'], chat.id)) + + file_id = video[-1]['file_id'] + uri, length = await upload_tgfile_to_matrix(file_id, user_id) + info = {'mimetype': 'video/mp4', 'size': length, 'h': video[-1]['height'], + 'w': video[-1]['width']} + body = 'Image_{}.jpg'.format(int(time() * 1000)) + + if uri: + j = await send_matrix_message(room_id, user_id, txn_id, body=body, + url=uri, info=info, msgtype='m.video') + + if 'errcode' in j and j['errcode'] == 'M_FORBIDDEN': + await register_join_matrix(chat, room_id, user_id) + await send_matrix_message(room_id, user_id, txn_id + 'join', + body=body, url=uri, info=info, + msgtype='m.video') + + if 'caption' in chat.message: + await send_matrix_message(room_id, user_id, txn_id + 'caption', + body=chat.message['caption'], + msgtype='m.text') + + if 'event_id' in j: + name = chat.sender['first_name'] + if 'last_name' in chat.sender: + name += " " + chat.sender['last_name'] + name += " (Telegram)" + message = db.Message( + chat.message['chat']['id'], + chat.message['message_id'], + room_id, + j['event_id'], + name) + db.session.add(message) + db.session.commit() + + @TG_BOT.command(r'/alias') async def aiotg_alias(chat, match): await chat.reply('The Matrix alias for this chat is #telegram_{}:{}' From 16319d07b3a7403a2102f4179769a3e12c6682b4 Mon Sep 17 00:00:00 2001 From: Redlink Date: Wed, 21 Feb 2018 11:11:14 +0100 Subject: [PATCH 06/32] fix --- telematrix/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index aa9b70a..0bd0105 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -609,7 +609,7 @@ async def aiotg_photo(chat, photo): @TG_BOT.handle('video') -async def aiotg_photo(chat, video): +async def aiotg_video(chat, video): link = db.session.query(db.ChatLink).filter_by(tg_room=chat.id).first() if not link: print('Unknown telegram chat {}: {}'.format(chat, chat.id)) @@ -624,7 +624,7 @@ async def aiotg_photo(chat, video): uri, length = await upload_tgfile_to_matrix(file_id, user_id) info = {'mimetype': 'video/mp4', 'size': length, 'h': video[-1]['height'], 'w': video[-1]['width']} - body = 'Image_{}.jpg'.format(int(time() * 1000)) + body = 'Video_{}.mp4'.format(int(time() * 1000)) if uri: j = await send_matrix_message(room_id, user_id, txn_id, body=body, From 888935cd99c8b844686baaf8309921560a18a1af Mon Sep 17 00:00:00 2001 From: Redlink Date: Thu, 22 Feb 2018 01:33:32 +0100 Subject: [PATCH 07/32] add audio --- telematrix/__init__.py | 51 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 0bd0105..4508a36 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -608,6 +608,53 @@ async def aiotg_photo(chat, photo): db.session.commit() +@TG_BOT.handle('audio') +async def aiotg_audio(chat, audio): + link = db.session.query(db.ChatLink).filter_by(tg_room=chat.id).first() + if not link: + print('Unknown telegram chat {}: {}'.format(chat, chat.id)) + return + + await update_matrix_displayname_avatar(chat.sender); + room_id = link.matrix_room + user_id = USER_ID_FORMAT.format(chat.sender['id']) + txn_id = quote('{}{}'.format(chat.message['message_id'], chat.id)) + + file_id = audio['file_id'] + uri, length = await upload_tgfile_to_matrix(file_id, user_id) + info = {'mimetype': 'audio/mp3', 'size': length} + body = 'Audio_{}.mp3'.format(int(time() * 1000)) + + if uri: + j = await send_matrix_message(room_id, user_id, txn_id, body=body, + url=uri, info=info, msgtype='m.video') + + if 'errcode' in j and j['errcode'] == 'M_FORBIDDEN': + await register_join_matrix(chat, room_id, user_id) + await send_matrix_message(room_id, user_id, txn_id + 'join', + body=body, url=uri, info=info, + msgtype='m.audio') + + if 'caption' in chat.message: + await send_matrix_message(room_id, user_id, txn_id + 'caption', + body=chat.message['caption'], + msgtype='m.text') + + if 'event_id' in j: + name = chat.sender['first_name'] + if 'last_name' in chat.sender: + name += " " + chat.sender['last_name'] + name += " (Telegram)" + message = db.Message( + chat.message['chat']['id'], + chat.message['message_id'], + room_id, + j['event_id'], + name) + db.session.add(message) + db.session.commit() + + @TG_BOT.handle('video') async def aiotg_video(chat, video): link = db.session.query(db.ChatLink).filter_by(tg_room=chat.id).first() @@ -622,8 +669,8 @@ async def aiotg_video(chat, video): file_id = video[-1]['file_id'] uri, length = await upload_tgfile_to_matrix(file_id, user_id) - info = {'mimetype': 'video/mp4', 'size': length, 'h': video[-1]['height'], - 'w': video[-1]['width']} + info = {'mimetype': 'video/mp4', 'size': length, 'h': video['height'], + 'w': video['width']} body = 'Video_{}.mp4'.format(int(time() * 1000)) if uri: From df746aa18873674ba467e12cb5692e1aeec96270 Mon Sep 17 00:00:00 2001 From: Redlink Date: Thu, 22 Feb 2018 01:39:24 +0100 Subject: [PATCH 08/32] fix --- telematrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 4508a36..6605f0e 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -627,7 +627,7 @@ async def aiotg_audio(chat, audio): if uri: j = await send_matrix_message(room_id, user_id, txn_id, body=body, - url=uri, info=info, msgtype='m.video') + url=uri, info=info, msgtype='m.audio') if 'errcode' in j and j['errcode'] == 'M_FORBIDDEN': await register_join_matrix(chat, room_id, user_id) From 314bbba5ede0bc338a641f19634b2b5fdc2f611e Mon Sep 17 00:00:00 2001 From: Redlink Date: Thu, 22 Feb 2018 01:41:38 +0100 Subject: [PATCH 09/32] fix --- telematrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 6605f0e..a219fb0 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -243,7 +243,7 @@ async def matrix_transaction(request): elif content['msgtype'] == 'm.emote': msg, mode = format_matrix_msg('{}', content) response = await group.send_text("* {} {}".format(displayname, msg), parse_mode=mode) - elif content['msgtype'] in ['m.image', 'm.video', 'm.video', 'm.file']: + elif content['msgtype'] in ['m.image', 'm.audio', 'm.video', 'm.file']: try: url = urlparse(content['url']) From a176c5840d18bd8d16dc37f810819f1ff6579bec Mon Sep 17 00:00:00 2001 From: Redlink Date: Sat, 24 Feb 2018 11:32:09 +0100 Subject: [PATCH 10/32] test --- telematrix/__init__.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index a219fb0..ddcaef0 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -448,6 +448,22 @@ async def upload_tgfile_to_matrix(file_id, user_id, mime='image/jpeg', convert_t return None, 0 +async def upload_audiofile_to_matrix(file_id, user_id, mime='audio/mpeg'): + file_path = (await TG_BOT.get_file(file_id))['file_path'] + print(file_path) + request = await TG_BOT.download_file(file_path) + data = await request.read() + print(data) + + j = await matrix_post('media', 'upload', user_id, data, mime) + length = len(data) + + if 'content_uri' in j: + return j['content_uri'], length + else: + return None, 0 + + async def register_join_matrix(chat, room_id, user_id): name = chat.sender['first_name'] if 'last_name' in chat.sender: @@ -610,6 +626,7 @@ async def aiotg_photo(chat, photo): @TG_BOT.handle('audio') async def aiotg_audio(chat, audio): + print(audio) link = db.session.query(db.ChatLink).filter_by(tg_room=chat.id).first() if not link: print('Unknown telegram chat {}: {}'.format(chat, chat.id)) @@ -621,8 +638,12 @@ async def aiotg_audio(chat, audio): txn_id = quote('{}{}'.format(chat.message['message_id'], chat.id)) file_id = audio['file_id'] - uri, length = await upload_tgfile_to_matrix(file_id, user_id) - info = {'mimetype': 'audio/mp3', 'size': length} + try: + mime = audio['mime_type'] + except KeyError: + mime = 'audio/mp3' + uri, length = await upload_audiofile_to_matrix(file_id, user_id, mime) + info = {'mimetype': mime, 'size': length} body = 'Audio_{}.mp3'.format(int(time() * 1000)) if uri: From 072599c36ac9d0d27f5246059cbc0544eab647c7 Mon Sep 17 00:00:00 2001 From: Redlink Date: Sat, 24 Feb 2018 11:45:08 +0100 Subject: [PATCH 11/32] remove debug print --- telematrix/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index ddcaef0..ac9690b 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -450,10 +450,8 @@ async def upload_tgfile_to_matrix(file_id, user_id, mime='image/jpeg', convert_t async def upload_audiofile_to_matrix(file_id, user_id, mime='audio/mpeg'): file_path = (await TG_BOT.get_file(file_id))['file_path'] - print(file_path) request = await TG_BOT.download_file(file_path) data = await request.read() - print(data) j = await matrix_post('media', 'upload', user_id, data, mime) length = len(data) @@ -626,7 +624,6 @@ async def aiotg_photo(chat, photo): @TG_BOT.handle('audio') async def aiotg_audio(chat, audio): - print(audio) link = db.session.query(db.ChatLink).filter_by(tg_room=chat.id).first() if not link: print('Unknown telegram chat {}: {}'.format(chat, chat.id)) From 1dd8af761ba41f6c3f0954f75c005ff32ce1e4ac Mon Sep 17 00:00:00 2001 From: Redlink Date: Sat, 24 Feb 2018 12:51:03 +0100 Subject: [PATCH 12/32] test ci --- telematrix/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index ac9690b..2b00df5 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -685,8 +685,12 @@ async def aiotg_video(chat, video): user_id = USER_ID_FORMAT.format(chat.sender['id']) txn_id = quote('{}{}'.format(chat.message['message_id'], chat.id)) - file_id = video[-1]['file_id'] - uri, length = await upload_tgfile_to_matrix(file_id, user_id) + file_id = video['file_id'] + try: + mime = video['mime_type'] + except KeyError: + mime = 'video/mp4' + uri, length = await upload_tgfile_to_matrix(file_id, user_id, mime) info = {'mimetype': 'video/mp4', 'size': length, 'h': video['height'], 'w': video['width']} body = 'Video_{}.mp4'.format(int(time() * 1000)) From 807873a7fce930e1c8dcbc3fa483d37d27da209d Mon Sep 17 00:00:00 2001 From: Redlink Date: Sat, 24 Feb 2018 12:57:59 +0100 Subject: [PATCH 13/32] test video --- telematrix/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 2b00df5..1095fa6 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -675,6 +675,7 @@ async def aiotg_audio(chat, audio): @TG_BOT.handle('video') async def aiotg_video(chat, video): + print(video) link = db.session.query(db.ChatLink).filter_by(tg_room=chat.id).first() if not link: print('Unknown telegram chat {}: {}'.format(chat, chat.id)) @@ -691,7 +692,7 @@ async def aiotg_video(chat, video): except KeyError: mime = 'video/mp4' uri, length = await upload_tgfile_to_matrix(file_id, user_id, mime) - info = {'mimetype': 'video/mp4', 'size': length, 'h': video['height'], + info = {'mimetype': mime, 'size': length, 'h': video['height'], 'w': video['width']} body = 'Video_{}.mp4'.format(int(time() * 1000)) From 4bf422b5c8a64fb1b126e1c3c62ff168c9f708ba Mon Sep 17 00:00:00 2001 From: Redlink Date: Wed, 28 Feb 2018 09:29:34 +0100 Subject: [PATCH 14/32] add document transfer --- telematrix/__init__.py | 60 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 1095fa6..b8a0eca 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -532,7 +532,7 @@ async def aiotg_sticker(chat, sticker): print('Unknown telegram chat {}: {}'.format(chat, chat.id)) return - await update_matrix_displayname_avatar(chat.sender); + await update_matrix_displayname_avatar(chat.sender) room_id = link.matrix_room user_id = USER_ID_FORMAT.format(chat.sender['id']) @@ -581,7 +581,7 @@ async def aiotg_photo(chat, photo): print('Unknown telegram chat {}: {}'.format(chat, chat.id)) return - await update_matrix_displayname_avatar(chat.sender); + await update_matrix_displayname_avatar(chat.sender) room_id = link.matrix_room user_id = USER_ID_FORMAT.format(chat.sender['id']) txn_id = quote('{}{}'.format(chat.message['message_id'], chat.id)) @@ -629,7 +629,7 @@ async def aiotg_audio(chat, audio): print('Unknown telegram chat {}: {}'.format(chat, chat.id)) return - await update_matrix_displayname_avatar(chat.sender); + await update_matrix_displayname_avatar(chat.sender) room_id = link.matrix_room user_id = USER_ID_FORMAT.format(chat.sender['id']) txn_id = quote('{}{}'.format(chat.message['message_id'], chat.id)) @@ -673,15 +673,65 @@ async def aiotg_audio(chat, audio): db.session.commit() +@TG_BOT.handle('document') +async def aiotg_audio(chat, document): + link = db.session.query(db.ChatLink).filter_by(tg_room=chat.id).first() + if not link: + print('Unknown telegram chat {}: {}'.format(chat, chat.id)) + return + + await update_matrix_displayname_avatar(chat.sender) + room_id = link.matrix_room + user_id = USER_ID_FORMAT.format(chat.sender['id']) + txn_id = quote('{}{}'.format(chat.message['message_id'], chat.id)) + + file_id = document['file_id'] + try: + mime = document['mime_type'] + except KeyError: + mime = '' + uri, length = await upload_audiofile_to_matrix(file_id, user_id, mime) + info = {'mimetype': mime, 'size': length} + body = 'Document_{}'.format(int(time() * 1000)) + + if uri: + j = await send_matrix_message(room_id, user_id, txn_id, body=body, + url=uri, info=info, msgtype='m.document') + + if 'errcode' in j and j['errcode'] == 'M_FORBIDDEN': + await register_join_matrix(chat, room_id, user_id) + await send_matrix_message(room_id, user_id, txn_id + 'join', + body=body, url=uri, info=info, + msgtype='m.document') + + if 'caption' in chat.message: + await send_matrix_message(room_id, user_id, txn_id + 'caption', + body=chat.message['caption'], + msgtype='m.text') + + if 'event_id' in j: + name = chat.sender['first_name'] + if 'last_name' in chat.sender: + name += " " + chat.sender['last_name'] + name += " (Telegram)" + message = db.Message( + chat.message['chat']['id'], + chat.message['message_id'], + room_id, + j['event_id'], + name) + db.session.add(message) + db.session.commit() + + @TG_BOT.handle('video') async def aiotg_video(chat, video): - print(video) link = db.session.query(db.ChatLink).filter_by(tg_room=chat.id).first() if not link: print('Unknown telegram chat {}: {}'.format(chat, chat.id)) return - await update_matrix_displayname_avatar(chat.sender); + await update_matrix_displayname_avatar(chat.sender) room_id = link.matrix_room user_id = USER_ID_FORMAT.format(chat.sender['id']) txn_id = quote('{}{}'.format(chat.message['message_id'], chat.id)) From 4589a514ee4dca28bd0ff603aebe12b82cfef54a Mon Sep 17 00:00:00 2001 From: Redlink Date: Thu, 1 Mar 2018 08:39:53 +0100 Subject: [PATCH 15/32] fix --- telematrix/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index b8a0eca..5ce1b49 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -674,7 +674,7 @@ async def aiotg_audio(chat, audio): @TG_BOT.handle('document') -async def aiotg_audio(chat, document): +async def aiotg_document(chat, document): link = db.session.query(db.ChatLink).filter_by(tg_room=chat.id).first() if not link: print('Unknown telegram chat {}: {}'.format(chat, chat.id)) @@ -887,6 +887,7 @@ async def aiotg_message(chat, match): db.session.commit() + def main(): """ Main function to get the entire ball rolling. From 10a670bd5d180c4b9ae3465e4c7bea7b704f84ac Mon Sep 17 00:00:00 2001 From: Redlink Date: Thu, 1 Mar 2018 08:44:20 +0100 Subject: [PATCH 16/32] fix --- telematrix/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 5ce1b49..7254e50 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -581,7 +581,7 @@ async def aiotg_photo(chat, photo): print('Unknown telegram chat {}: {}'.format(chat, chat.id)) return - await update_matrix_displayname_avatar(chat.sender) + await update_matrix_displayname_avatar(chat.sender); room_id = link.matrix_room user_id = USER_ID_FORMAT.format(chat.sender['id']) txn_id = quote('{}{}'.format(chat.message['message_id'], chat.id)) @@ -629,7 +629,7 @@ async def aiotg_audio(chat, audio): print('Unknown telegram chat {}: {}'.format(chat, chat.id)) return - await update_matrix_displayname_avatar(chat.sender) + await update_matrix_displayname_avatar(chat.sender); room_id = link.matrix_room user_id = USER_ID_FORMAT.format(chat.sender['id']) txn_id = quote('{}{}'.format(chat.message['message_id'], chat.id)) @@ -674,13 +674,13 @@ async def aiotg_audio(chat, audio): @TG_BOT.handle('document') -async def aiotg_document(chat, document): +async def aiotg_document (chat, document): link = db.session.query(db.ChatLink).filter_by(tg_room=chat.id).first() if not link: print('Unknown telegram chat {}: {}'.format(chat, chat.id)) return - await update_matrix_displayname_avatar(chat.sender) + await update_matrix_displayname_avatar(chat.sender); room_id = link.matrix_room user_id = USER_ID_FORMAT.format(chat.sender['id']) txn_id = quote('{}{}'.format(chat.message['message_id'], chat.id)) @@ -731,7 +731,7 @@ async def aiotg_video(chat, video): print('Unknown telegram chat {}: {}'.format(chat, chat.id)) return - await update_matrix_displayname_avatar(chat.sender) + await update_matrix_displayname_avatar(chat.sender); room_id = link.matrix_room user_id = USER_ID_FORMAT.format(chat.sender['id']) txn_id = quote('{}{}'.format(chat.message['message_id'], chat.id)) From 70232ef8a0e50b866a0be985c5adc2c4eecfff6d Mon Sep 17 00:00:00 2001 From: Redlink Date: Thu, 1 Mar 2018 08:49:39 +0100 Subject: [PATCH 17/32] fix (m.document -> m.file) --- telematrix/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 7254e50..c0b6354 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -696,13 +696,13 @@ async def aiotg_document (chat, document): if uri: j = await send_matrix_message(room_id, user_id, txn_id, body=body, - url=uri, info=info, msgtype='m.document') + url=uri, info=info, msgtype='m.file') if 'errcode' in j and j['errcode'] == 'M_FORBIDDEN': await register_join_matrix(chat, room_id, user_id) await send_matrix_message(room_id, user_id, txn_id + 'join', body=body, url=uri, info=info, - msgtype='m.document') + msgtype='m.file') if 'caption' in chat.message: await send_matrix_message(room_id, user_id, txn_id + 'caption', From dbeff734116c3687470cab9e4c42f87c4181edad Mon Sep 17 00:00:00 2001 From: Redlink Date: Thu, 1 Mar 2018 09:15:11 +0100 Subject: [PATCH 18/32] test --- telematrix/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index c0b6354..f2a6ea9 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -23,7 +23,7 @@ # Read the configuration file try: - with open('config.json', 'r') as config_file: + with open('config.json.example', 'r') as config_file: CONFIG = json.load(config_file) HS_TOKEN = CONFIG['tokens']['hs'] @@ -674,7 +674,7 @@ async def aiotg_audio(chat, audio): @TG_BOT.handle('document') -async def aiotg_document (chat, document): +async def aiotg_document(chat, document): link = db.session.query(db.ChatLink).filter_by(tg_room=chat.id).first() if not link: print('Unknown telegram chat {}: {}'.format(chat, chat.id)) @@ -887,6 +887,10 @@ async def aiotg_message(chat, match): db.session.commit() +@TG_BOT.command(r'(.*)') +def test(chat, match): + print(match) + def main(): """ @@ -894,6 +898,7 @@ def main(): """ logging.basicConfig(level=logging.WARNING) db.initialize(DATABASE_URL) + print(TG_BOT._handlers) loop = asyncio.get_event_loop() asyncio.ensure_future(TG_BOT.loop()) From e7c5647703f9562663a834a009845486a1420565 Mon Sep 17 00:00:00 2001 From: Redlink Date: Thu, 1 Mar 2018 09:15:56 +0100 Subject: [PATCH 19/32] fix --- telematrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index f2a6ea9..7771de5 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -23,7 +23,7 @@ # Read the configuration file try: - with open('config.json.example', 'r') as config_file: + with open('config.json', 'r') as config_file: CONFIG = json.load(config_file) HS_TOKEN = CONFIG['tokens']['hs'] From 9db97183e6916dbc8137cafef6997532abe63f68 Mon Sep 17 00:00:00 2001 From: Redlink Date: Mon, 5 Mar 2018 12:48:16 +0100 Subject: [PATCH 20/32] cleaner way to use mimetypes --- telematrix/__init__.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 7771de5..599d4aa 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -58,6 +58,8 @@ MATRIX_SESS = ClientSession() SHORTEN_SESS = ClientSession() +MT = mimetypes.MimeTypes() + def create_response(code, obj): """ @@ -523,7 +525,15 @@ async def update_matrix_displayname_avatar(tg_user): await matrix_put('client', 'profile/{}/avatar_url'.format(user_id), user_id, {'avatar_url':None}) db.session.add(db_user) db.session.commit() - + + +def create_file_name(obj_type, mime): + try: + ext = MT.types_map_inv[mime][0] + except KeyError: + ext = '' + name = '{}_{}.{}'.format(obj_type, int(time() * 1000), ext) + return name @TG_BOT.handle('sticker') async def aiotg_sticker(chat, sticker): @@ -641,7 +651,7 @@ async def aiotg_audio(chat, audio): mime = 'audio/mp3' uri, length = await upload_audiofile_to_matrix(file_id, user_id, mime) info = {'mimetype': mime, 'size': length} - body = 'Audio_{}.mp3'.format(int(time() * 1000)) + body = create_file_name('Audio', mime) if uri: j = await send_matrix_message(room_id, user_id, txn_id, body=body, @@ -692,7 +702,7 @@ async def aiotg_document(chat, document): mime = '' uri, length = await upload_audiofile_to_matrix(file_id, user_id, mime) info = {'mimetype': mime, 'size': length} - body = 'Document_{}'.format(int(time() * 1000)) + body = create_file_name('File', mime) if uri: j = await send_matrix_message(room_id, user_id, txn_id, body=body, @@ -744,7 +754,7 @@ async def aiotg_video(chat, video): uri, length = await upload_tgfile_to_matrix(file_id, user_id, mime) info = {'mimetype': mime, 'size': length, 'h': video['height'], 'w': video['width']} - body = 'Video_{}.mp4'.format(int(time() * 1000)) + body = create_file_name('Video', mime) if uri: j = await send_matrix_message(room_id, user_id, txn_id, body=body, From 90267b4af0782dfd4fb3f18952a15683b5a856da Mon Sep 17 00:00:00 2001 From: Redlink Date: Mon, 5 Mar 2018 12:59:30 +0100 Subject: [PATCH 21/32] fix --- telematrix/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 599d4aa..0f008d3 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -142,13 +142,16 @@ async def shorten_url(url): else: return url + def matrix_is_telegram(user_id): username = user_id.split(':')[0][1:] return username.startswith('telegram_') + def get_username(user_id): return user_id.split(':')[0][1:] + mime_extensions = { 'image/jpeg': 'jpg', 'image/gif': 'gif', @@ -164,6 +167,7 @@ def get_username(user_id): 'x-video/mp4': 'mp4', } + async def matrix_transaction(request): """ Handle a transaction sent by the homeserver. @@ -529,9 +533,12 @@ async def update_matrix_displayname_avatar(tg_user): def create_file_name(obj_type, mime): try: - ext = MT.types_map_inv[mime][0] + ext = MT.types_map_inv[1][mime][0] except KeyError: - ext = '' + try: + ext = MT.types_map_inv[0][mime][0] + except KeyError: + ext = '' name = '{}_{}.{}'.format(obj_type, int(time() * 1000), ext) return name From 92d1b499def3bbbfe0760aaefa853abcbb4c98fa Mon Sep 17 00:00:00 2001 From: Redlink Date: Mon, 5 Mar 2018 13:34:47 +0100 Subject: [PATCH 22/32] add send_file_to_matrix function to remove redundant code --- telematrix/__init__.py | 180 ++++++++++------------------------------- 1 file changed, 41 insertions(+), 139 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 0f008d3..d87e558 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -454,7 +454,7 @@ async def upload_tgfile_to_matrix(file_id, user_id, mime='image/jpeg', convert_t return None, 0 -async def upload_audiofile_to_matrix(file_id, user_id, mime='audio/mpeg'): +async def upload_file_to_matrix(file_id, user_id, mime): file_path = (await TG_BOT.get_file(file_id))['file_path'] request = await TG_BOT.download_file(file_path) data = await request.read() @@ -539,9 +539,39 @@ def create_file_name(obj_type, mime): ext = MT.types_map_inv[0][mime][0] except KeyError: ext = '' - name = '{}_{}.{}'.format(obj_type, int(time() * 1000), ext) + name = '{}_{}{}'.format(obj_type, int(time() * 1000), ext) return name + +async def send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, msgtype): + j = await send_matrix_message(room_id, user_id, txn_id, body=body, + url=uri, info=info, msgtype=msgtype) + + if 'errcode' in j and j['errcode'] == 'M_FORBIDDEN': + await register_join_matrix(chat, room_id, user_id) + await send_matrix_message(room_id, user_id, txn_id + 'join', + body=body, url=uri, info=info, + msgtype=msgtype) + + if 'caption' in chat.message: + await send_matrix_message(room_id, user_id, txn_id + 'caption', + body=chat.message['caption'], + msgtype='m.text') + + if 'event_id' in j: + name = chat.sender['first_name'] + if 'last_name' in chat.sender: + name += " " + chat.sender['last_name'] + name += " (Telegram)" + message = db.Message( + chat.message['chat']['id'], + chat.message['message_id'], + room_id, + j['event_id'], + name) + db.session.add(message) + db.session.commit() + @TG_BOT.handle('sticker') async def aiotg_sticker(chat, sticker): link = db.session.query(db.ChatLink).filter_by(tg_room=chat.id).first() @@ -563,32 +593,7 @@ async def aiotg_sticker(chat, sticker): body = 'Sticker_{}.png'.format(int(time() * 1000)) if uri: - j = await send_matrix_message(room_id, user_id, txn_id, body=body, - url=uri, info=info, msgtype='m.image') - - if 'errcode' in j and j['errcode'] == 'M_FORBIDDEN': - await register_join_matrix(chat, room_id, user_id) - await send_matrix_message(room_id, user_id, txn_id + 'join', - body=body, url=uri, info=info, - msgtype='m.image') - - if 'caption' in chat.message: - await send_matrix_message(room_id, user_id, txn_id + 'caption', - body=chat.message['caption'], - msgtype='m.text') - if 'event_id' in j: - name = chat.sender['first_name'] - if 'last_name' in chat.sender: - name += " " + chat.sender['last_name'] - name += " (Telegram)" - message = db.Message( - chat.message['chat']['id'], - chat.message['message_id'], - room_id, - j['event_id'], - name) - db.session.add(message) - db.session.commit() + send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, 'm.image') @TG_BOT.handle('photo') @@ -604,39 +609,14 @@ async def aiotg_photo(chat, photo): txn_id = quote('{}{}'.format(chat.message['message_id'], chat.id)) file_id = photo[-1]['file_id'] + uri, length = await upload_tgfile_to_matrix(file_id, user_id) info = {'mimetype': 'image/jpeg', 'size': length, 'h': photo[-1]['height'], 'w': photo[-1]['width']} body = 'Image_{}.jpg'.format(int(time() * 1000)) if uri: - j = await send_matrix_message(room_id, user_id, txn_id, body=body, - url=uri, info=info, msgtype='m.image') - - if 'errcode' in j and j['errcode'] == 'M_FORBIDDEN': - await register_join_matrix(chat, room_id, user_id) - await send_matrix_message(room_id, user_id, txn_id + 'join', - body=body, url=uri, info=info, - msgtype='m.image') - - if 'caption' in chat.message: - await send_matrix_message(room_id, user_id, txn_id + 'caption', - body=chat.message['caption'], - msgtype='m.text') - - if 'event_id' in j: - name = chat.sender['first_name'] - if 'last_name' in chat.sender: - name += " " + chat.sender['last_name'] - name += " (Telegram)" - message = db.Message( - chat.message['chat']['id'], - chat.message['message_id'], - room_id, - j['event_id'], - name) - db.session.add(message) - db.session.commit() + send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, 'm.image') @TG_BOT.handle('audio') @@ -656,38 +636,12 @@ async def aiotg_audio(chat, audio): mime = audio['mime_type'] except KeyError: mime = 'audio/mp3' - uri, length = await upload_audiofile_to_matrix(file_id, user_id, mime) + uri, length = await upload_file_to_matrix(file_id, user_id, mime) info = {'mimetype': mime, 'size': length} body = create_file_name('Audio', mime) if uri: - j = await send_matrix_message(room_id, user_id, txn_id, body=body, - url=uri, info=info, msgtype='m.audio') - - if 'errcode' in j and j['errcode'] == 'M_FORBIDDEN': - await register_join_matrix(chat, room_id, user_id) - await send_matrix_message(room_id, user_id, txn_id + 'join', - body=body, url=uri, info=info, - msgtype='m.audio') - - if 'caption' in chat.message: - await send_matrix_message(room_id, user_id, txn_id + 'caption', - body=chat.message['caption'], - msgtype='m.text') - - if 'event_id' in j: - name = chat.sender['first_name'] - if 'last_name' in chat.sender: - name += " " + chat.sender['last_name'] - name += " (Telegram)" - message = db.Message( - chat.message['chat']['id'], - chat.message['message_id'], - room_id, - j['event_id'], - name) - db.session.add(message) - db.session.commit() + send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, 'm.audio') @TG_BOT.handle('document') @@ -707,38 +661,12 @@ async def aiotg_document(chat, document): mime = document['mime_type'] except KeyError: mime = '' - uri, length = await upload_audiofile_to_matrix(file_id, user_id, mime) + uri, length = await upload_file_to_matrix(file_id, user_id, mime) info = {'mimetype': mime, 'size': length} body = create_file_name('File', mime) if uri: - j = await send_matrix_message(room_id, user_id, txn_id, body=body, - url=uri, info=info, msgtype='m.file') - - if 'errcode' in j and j['errcode'] == 'M_FORBIDDEN': - await register_join_matrix(chat, room_id, user_id) - await send_matrix_message(room_id, user_id, txn_id + 'join', - body=body, url=uri, info=info, - msgtype='m.file') - - if 'caption' in chat.message: - await send_matrix_message(room_id, user_id, txn_id + 'caption', - body=chat.message['caption'], - msgtype='m.text') - - if 'event_id' in j: - name = chat.sender['first_name'] - if 'last_name' in chat.sender: - name += " " + chat.sender['last_name'] - name += " (Telegram)" - message = db.Message( - chat.message['chat']['id'], - chat.message['message_id'], - room_id, - j['event_id'], - name) - db.session.add(message) - db.session.commit() + send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, 'm.file') @TG_BOT.handle('video') @@ -758,39 +686,13 @@ async def aiotg_video(chat, video): mime = video['mime_type'] except KeyError: mime = 'video/mp4' - uri, length = await upload_tgfile_to_matrix(file_id, user_id, mime) + uri, length = await upload_file_to_matrix(file_id, user_id, mime) info = {'mimetype': mime, 'size': length, 'h': video['height'], 'w': video['width']} body = create_file_name('Video', mime) if uri: - j = await send_matrix_message(room_id, user_id, txn_id, body=body, - url=uri, info=info, msgtype='m.video') - - if 'errcode' in j and j['errcode'] == 'M_FORBIDDEN': - await register_join_matrix(chat, room_id, user_id) - await send_matrix_message(room_id, user_id, txn_id + 'join', - body=body, url=uri, info=info, - msgtype='m.video') - - if 'caption' in chat.message: - await send_matrix_message(room_id, user_id, txn_id + 'caption', - body=chat.message['caption'], - msgtype='m.text') - - if 'event_id' in j: - name = chat.sender['first_name'] - if 'last_name' in chat.sender: - name += " " + chat.sender['last_name'] - name += " (Telegram)" - message = db.Message( - chat.message['chat']['id'], - chat.message['message_id'], - room_id, - j['event_id'], - name) - db.session.add(message) - db.session.commit() + send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, 'm.video') @TG_BOT.command(r'/alias') From 396aa749f545f8f6c361401c0de0b0ee2ca3a194 Mon Sep 17 00:00:00 2001 From: Redlink Date: Mon, 5 Mar 2018 15:35:55 +0100 Subject: [PATCH 23/32] fix --- telematrix/__init__.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index d87e558..0613411 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -550,13 +550,11 @@ async def send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, m if 'errcode' in j and j['errcode'] == 'M_FORBIDDEN': await register_join_matrix(chat, room_id, user_id) await send_matrix_message(room_id, user_id, txn_id + 'join', - body=body, url=uri, info=info, - msgtype=msgtype) + body=body, url=uri, info=info, msgtype=msgtype) if 'caption' in chat.message: await send_matrix_message(room_id, user_id, txn_id + 'caption', - body=chat.message['caption'], - msgtype='m.text') + body=chat.message['caption'], msgtype='m.text') if 'event_id' in j: name = chat.sender['first_name'] @@ -593,7 +591,7 @@ async def aiotg_sticker(chat, sticker): body = 'Sticker_{}.png'.format(int(time() * 1000)) if uri: - send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, 'm.image') + await send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, 'm.image') @TG_BOT.handle('photo') @@ -616,7 +614,7 @@ async def aiotg_photo(chat, photo): body = 'Image_{}.jpg'.format(int(time() * 1000)) if uri: - send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, 'm.image') + await send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, 'm.image') @TG_BOT.handle('audio') @@ -641,7 +639,7 @@ async def aiotg_audio(chat, audio): body = create_file_name('Audio', mime) if uri: - send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, 'm.audio') + await send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, 'm.audio') @TG_BOT.handle('document') @@ -666,7 +664,7 @@ async def aiotg_document(chat, document): body = create_file_name('File', mime) if uri: - send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, 'm.file') + await send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, 'm.file') @TG_BOT.handle('video') @@ -692,7 +690,7 @@ async def aiotg_video(chat, video): body = create_file_name('Video', mime) if uri: - send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, 'm.video') + await send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, 'm.video') @TG_BOT.command(r'/alias') From 356ed17001c33f6654717764a2be9cd2b0cc3ae7 Mon Sep 17 00:00:00 2001 From: Redlink Date: Mon, 5 Mar 2018 15:36:53 +0100 Subject: [PATCH 24/32] debug --- telematrix/__init__.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 0613411..4d75b7a 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -804,16 +804,11 @@ async def aiotg_message(chat, match): db.session.commit() -@TG_BOT.command(r'(.*)') -def test(chat, match): - print(match) - - def main(): """ Main function to get the entire ball rolling. """ - logging.basicConfig(level=logging.WARNING) + logging.basicConfig(level=logging.DEBUG) db.initialize(DATABASE_URL) print(TG_BOT._handlers) From 38862aaa5ac20809a408cdbb912c1d2760e4106d Mon Sep 17 00:00:00 2001 From: Redlink Date: Mon, 5 Mar 2018 15:53:50 +0100 Subject: [PATCH 25/32] improve file transfer from tg to matrix --- telematrix/__init__.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 4d75b7a..9549717 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -661,10 +661,21 @@ async def aiotg_document(chat, document): mime = '' uri, length = await upload_file_to_matrix(file_id, user_id, mime) info = {'mimetype': mime, 'size': length} - body = create_file_name('File', mime) if uri: - await send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, 'm.file') + if 'image' in mime: + msgtype = 'm.image' + body = create_file_name('Image', mime) + elif 'video' in mime: + msgtype = 'm.video' + body = create_file_name('Video', mime) + elif 'audio' in mime: + msgtype = 'm.audio' + body = create_file_name('Audio', mime) + else: + msgtype = 'm.file' + body = create_file_name('File', mime) + await send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, msgtype) @TG_BOT.handle('video') From 4f5686045eb7e2b28503bdcffd369f1a44abd1df Mon Sep 17 00:00:00 2001 From: Redlink Date: Mon, 5 Mar 2018 16:39:48 +0100 Subject: [PATCH 26/32] improve gif transfer from matrix to tg --- telematrix/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 9549717..76c3a5d 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -271,8 +271,12 @@ async def matrix_transaction(request): url_str = await shorten_url(url_str) if content['msgtype'] == 'm.image': - caption = '{} sent an image'.format(displayname) - response = await group.send_photo(file, caption=caption) + if content['info']['mimetype'] == 'image/gif': + caption = '{} sent a gif'.format(displayname) + response = await group.send_video(file, caption=caption) + else: + caption = '{} sent an image'.format(displayname) + response = await group.send_photo(file, caption=caption) elif content['msgtype'] == 'm.video': caption = '{} sent a video'.format(displayname) response = await group.send_video(file, caption=caption) From ae0e398b92bb468e999b4fc1b492d1a1518eee7b Mon Sep 17 00:00:00 2001 From: Redlink Date: Mon, 5 Mar 2018 17:01:48 +0100 Subject: [PATCH 27/32] add chat actions --- telematrix/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 76c3a5d..3c04b10 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -270,21 +270,27 @@ async def matrix_transaction(request): .format(url.netloc, quote(url.path)) url_str = await shorten_url(url_str) + if content['msgtype'] == 'm.image': if content['info']['mimetype'] == 'image/gif': caption = '{} sent a gif'.format(displayname) + await group.send_chat_action('upload_video') response = await group.send_video(file, caption=caption) else: caption = '{} sent an image'.format(displayname) + await group.send_chat_action('upload_photo') response = await group.send_photo(file, caption=caption) elif content['msgtype'] == 'm.video': caption = '{} sent a video'.format(displayname) + await group.send_chat_action('upload_video') response = await group.send_video(file, caption=caption) elif content['msgtype'] == 'm.audio': caption = '{} sent an audio file'.format(displayname) + await group.send_chat_action('upload_audio') response = await group.send_audio(file, caption=caption) elif content['msgtype'] == 'm.file': caption = '{} sent a file'.format(displayname) + await group.send_chat_action('upload_document') response = await group.send_document(file, caption=caption) except: pass From 0ad5ebf6f3ed4efa4330a3fb5dcbfd016613ffa4 Mon Sep 17 00:00:00 2001 From: Redlink Date: Wed, 7 Mar 2018 11:52:01 +0100 Subject: [PATCH 28/32] use mimetypes package instead of a custom list + some comments --- telematrix/__init__.py | 71 ++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 3c04b10..81c5c52 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -144,6 +144,12 @@ async def shorten_url(url): def matrix_is_telegram(user_id): + """ + Check if the user is a virtual telegram user or a real matrix user. Returns True if it is a fake + user (telegram virtual user). + :param user_id: The matrix id of the user. + :return: True if a virtual telegram user. + """ username = user_id.split(':')[0][1:] return username.startswith('telegram_') @@ -152,22 +158,6 @@ def get_username(user_id): return user_id.split(':')[0][1:] -mime_extensions = { - 'image/jpeg': 'jpg', - 'image/gif': 'gif', - 'image/png': 'png', - 'image/tiff': 'tif', - 'image/x-tiff': 'tif', - 'image/bmp': 'bmp', - 'image/x-windows-bmp': 'bmp', - 'video/avi': 'avi', - 'msvideo/avi': 'avi', - 'x-msvideo/avi': 'avi', - 'video/mp4': 'mp4', - 'x-video/mp4': 'mp4', -} - - async def matrix_transaction(request): """ Handle a transaction sent by the homeserver. @@ -242,24 +232,27 @@ async def matrix_transaction(request): if content['msgtype'] == 'm.text': msg, mode = format_matrix_msg('{}', content) - response = await group.send_text("{}: {}".format(displayname, msg), parse_mode='HTML') + response = await group.send_text("{}: {}".format(displayname, msg), + parse_mode='HTML') elif content['msgtype'] == 'm.notice': msg, mode = format_matrix_msg('{}', content) - response = await group.send_text("[{}] {}".format(displayname, msg), parse_mode=mode) + response = await group.send_text("[{}] {}".format(displayname, msg), + parse_mode=mode) elif content['msgtype'] == 'm.emote': msg, mode = format_matrix_msg('{}', content) - response = await group.send_text("* {} {}".format(displayname, msg), parse_mode=mode) + response = await group.send_text("* {} {}".format(displayname, msg), + parse_mode=mode) elif content['msgtype'] in ['m.image', 'm.audio', 'm.video', 'm.file']: try: url = urlparse(content['url']) # Append the correct extension if it's missing or wrong try: - ext = mime_extensions[content['info']['mimetype']] + exts = MT.types_map_inv[content['info']['mimetype']] except KeyError: - ext = "" - if not content['body'].endswith(ext): - content['body'] += '.' + ext + exts = "" + if not content['body'].endswith(tuple(exts)): + content['body'] += '.' + exts[0] # Download the file await download_matrix_file(url, content['body']) @@ -270,9 +263,9 @@ async def matrix_transaction(request): .format(url.netloc, quote(url.path)) url_str = await shorten_url(url_str) - if content['msgtype'] == 'm.image': if content['info']['mimetype'] == 'image/gif': + # Send gif as a video, so telegram can display it animated caption = '{} sent a gif'.format(displayname) await group.send_chat_action('upload_video') response = await group.send_video(file, caption=caption) @@ -299,7 +292,8 @@ async def matrix_transaction(request): print(json.dumps(content, indent=4)) elif event['type'] == 'm.room.member': - if HIDE_MEMBERSHIP_CHANGES: + if HIDE_MEMBERSHIP_CHANGES: # Hide everything, could be improved to be + # more specific continue if matrix_is_telegram(event['state_key']): continue @@ -465,6 +459,13 @@ async def upload_tgfile_to_matrix(file_id, user_id, mime='image/jpeg', convert_t async def upload_file_to_matrix(file_id, user_id, mime): + """ + Upload to the matrix homeserver all kinds of files based on mime informations. + :param file_id: Telegram file id + :param user_id: Matrix user id + :param mime: mime type of the file + :return: Tuple (JSON response, data length) if success, None else + """ file_path = (await TG_BOT.get_file(file_id))['file_path'] request = await TG_BOT.download_file(file_path) data = await request.read() @@ -580,6 +581,7 @@ async def send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, m db.session.add(message) db.session.commit() + @TG_BOT.handle('sticker') async def aiotg_sticker(chat, sticker): link = db.session.query(db.ChatLink).filter_by(tg_room=chat.id).first() @@ -673,6 +675,9 @@ async def aiotg_document(chat, document): info = {'mimetype': mime, 'size': length} if uri: + # We check if the document can be sent in a better way (for example a photo or a gif) + # For gif, that's still not perfect : it's sent as a video to matrix instead of a real + # gif image if 'image' in mime: msgtype = 'm.image' body = create_file_name('Image', mime) @@ -688,6 +693,8 @@ async def aiotg_document(chat, document): await send_file_to_matrix(chat, room_id, user_id, txn_id, body, uri, info, msgtype) +# This doesn't catch video from telegram, I don't know why +# The handler is never called @TG_BOT.handle('video') async def aiotg_video(chat, video): link = db.session.query(db.ChatLink).filter_by(tg_room=chat.id).first() @@ -766,11 +773,11 @@ async def aiotg_message(chat, match): re_msg['from']['last_name']) else: msg_from = '{} (Telegram)'.format(re_msg['from']['first_name']) - date = datetime.fromtimestamp(re_msg['date']) \ - .strftime('%Y-%m-%d %H:%M:%S') + date = datetime.fromtimestamp(re_msg['date']).strftime('%Y-%m-%d %H:%M:%S') reply_mx_id = db.session.query(db.Message)\ - .filter_by(tg_group_id=chat.message['chat']['id'], tg_message_id=chat.message['reply_to_message']['message_id']).first() + .filter_by(tg_group_id=chat.message['chat']['id'], + tg_message_id=chat.message['reply_to_message']['message_id']).first() html_message = html.escape(message).replace('\n', '
') if 'text' in re_msg: @@ -787,7 +794,9 @@ async def aiotg_message(chat, match): quoted_msg = 'Reply to {}:\n{}\n\n{}' \ .format(reply_mx_id.displayname, quoted_msg, message) quoted_html = 'Reply to {}:
{}

{}

' \ - .format(html.escape(room_id), html.escape(reply_mx_id.matrix_event_id), html.escape(reply_mx_id.displayname), + .format(html.escape(room_id), + html.escape(reply_mx_id.matrix_event_id), + html.escape(reply_mx_id.displayname), quoted_html, html_message) else: quoted_msg = 'Reply to {}:\n{}\n\n{}' \ @@ -829,9 +838,9 @@ def main(): """ Main function to get the entire ball rolling. """ - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig(level=logging.DEBUG) # should be set to WARNING when debug is no + # more necessary db.initialize(DATABASE_URL) - print(TG_BOT._handlers) loop = asyncio.get_event_loop() asyncio.ensure_future(TG_BOT.loop()) From f8518cc47bf9851866147e213247d7c5bf818cc8 Mon Sep 17 00:00:00 2001 From: Redlink Date: Wed, 7 Mar 2018 13:58:59 +0100 Subject: [PATCH 29/32] for merge --- telematrix/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 81c5c52..86a4d7b 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -838,8 +838,7 @@ def main(): """ Main function to get the entire ball rolling. """ - logging.basicConfig(level=logging.DEBUG) # should be set to WARNING when debug is no - # more necessary + logging.basicConfig(level=logging.WARNING) db.initialize(DATABASE_URL) loop = asyncio.get_event_loop() From d5272ce7ac2d7490215bfa75fe5fdcabdd996b4e Mon Sep 17 00:00:00 2001 From: Redlink Date: Wed, 7 Mar 2018 21:33:22 +0100 Subject: [PATCH 30/32] fix file transfer from matrix to tg --- telematrix/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 86a4d7b..c8f79a7 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -248,11 +248,11 @@ async def matrix_transaction(request): # Append the correct extension if it's missing or wrong try: - exts = MT.types_map_inv[content['info']['mimetype']] + exts = MT.types_map_inv[1][content['info']['mimetype']] + if not content['body'].endswith(tuple(exts)): + content['body'] += '.' + exts[0] except KeyError: - exts = "" - if not content['body'].endswith(tuple(exts)): - content['body'] += '.' + exts[0] + pass # Download the file await download_matrix_file(url, content['body']) From e08a6daa41c26a8540d140fb51bc0b60cefeae0a Mon Sep 17 00:00:00 2001 From: Redlink Date: Sun, 6 May 2018 22:46:32 +0200 Subject: [PATCH 31/32] add m.sticker event support --- telematrix/__init__.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index c8f79a7..17c5658 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -291,6 +291,37 @@ async def matrix_transaction(request): print('Unsupported message type {}'.format(content['msgtype'])) print(json.dumps(content, indent=4)) + elif event['type'] == 'm.sticker': + user_id = event['user_id'] + if matrix_is_telegram(user_id): + continue + + sender = db.session.query(db.MatrixUser) \ + .filter_by(matrix_id=user_id).first() + + if not sender: + response = await matrix_get('client', 'profile/{}/displayname' + .format(user_id), None) + try: + displayname = response['displayname'] + except KeyError: + displayname = get_username(user_id) + sender = db.MatrixUser(user_id, displayname) + db.session.add(sender) + else: + displayname = sender.name or get_username(user_id) + content = event['content'] + + try: + url = urlparse(content['url']) + await download_matrix_file(url, content['body']) + + with open('/tmp/{}'.format(content['body']), 'rb') as file: + response = await group.send_sticker(file) + + except: + pass + elif event['type'] == 'm.room.member': if HIDE_MEMBERSHIP_CHANGES: # Hide everything, could be improved to be # more specific From 1276b04380c7179c3ebdacc5df15fe288b0b92d2 Mon Sep 17 00:00:00 2001 From: Romain Poirot Date: Thu, 31 May 2018 21:14:04 +0200 Subject: [PATCH 32/32] stickers are now sent correctly to tg --- telematrix/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/telematrix/__init__.py b/telematrix/__init__.py index 17c5658..96c4257 100644 --- a/telematrix/__init__.py +++ b/telematrix/__init__.py @@ -316,8 +316,10 @@ async def matrix_transaction(request): url = urlparse(content['url']) await download_matrix_file(url, content['body']) - with open('/tmp/{}'.format(content['body']), 'rb') as file: - response = await group.send_sticker(file) + png_image = Image.open('/tmp/{}'.format(content['body'])) + png_image.save('/tmp/{}.webp'.format(content['body']), 'WEBP') + with open('/tmp/{}.webp'.format(content['body']), 'rb') as file: + response = await group.send_document(file) except: pass