From 922da85bd51e5223dd229ae998209c074da47c8a Mon Sep 17 00:00:00 2001 From: darmiel <71837281+darmiel@users.noreply.github.com> Date: Mon, 30 Nov 2020 17:50:15 +0100 Subject: [PATCH] Implements #23 --- .../mods/updaters/ChatsUpdateModule.java | 31 ++- .../schwurbelwatch/tgcrawler/api/SwApi.java | 2 +- .../tgcrawler/api/routes/chats/ApiChat.java | 2 +- .../api/routes/chats/ChatService.java | 17 +- .../src/controller/chatsController.js | 244 +++++++----------- apps/rest-api/src/routes/chats.js | 9 +- 6 files changed, 139 insertions(+), 166 deletions(-) diff --git a/apps/message-crawler/telegram-crawler-core/src/main/java/io/d2a/schwurbelwatch/mods/updaters/ChatsUpdateModule.java b/apps/message-crawler/telegram-crawler-core/src/main/java/io/d2a/schwurbelwatch/mods/updaters/ChatsUpdateModule.java index f093e52..9e34f77 100644 --- a/apps/message-crawler/telegram-crawler-core/src/main/java/io/d2a/schwurbelwatch/mods/updaters/ChatsUpdateModule.java +++ b/apps/message-crawler/telegram-crawler-core/src/main/java/io/d2a/schwurbelwatch/mods/updaters/ChatsUpdateModule.java @@ -54,15 +54,6 @@ public void onEnable() { } } - private void addOrUpdateChat(@Nullable final Chat chat, @Nullable final ApiChat apiChat) { - if (chat != null) { - // Logger.debug(chat); - } - //Logger.debug(apiChat); - - SwApi.callDatabaseResult(SwApi.CHAT_SERVICE.addChat(apiChat)); - } - @Subscribe public void onUpdateNewChat(final TdApi.UpdateNewChat event) { final Chat chat = event.chat; @@ -70,9 +61,11 @@ public void onUpdateNewChat(final TdApi.UpdateNewChat event) { .chatId(chat.id) .title(chat.title) .type(ChatType.getType(chat.type)) - .lastUpdated(System.currentTimeMillis()) .build(); - addOrUpdateChat(chat, build); + + // uppdate new chat + // -> chatId + SwApi.callDatabaseResult(SwApi.CHAT_SERVICE.addChat(build)); } @Subscribe @@ -81,7 +74,10 @@ public void onChatTitleUpdate(final UpdateChatTitle event) { .chatId(event.chatId) .title(event.title) .build(); - addOrUpdateChat(null, build); + + // uppdate chat title + // -> chatId + SwApi.callDatabaseResult(SwApi.CHAT_SERVICE.addChat(build)); } @Subscribe @@ -111,7 +107,7 @@ public void onNewMessage(final UpdateNewMessage event) { } @RequiredArgsConstructor - public class ChatInfoResultHandler implements ResultHandler { + public static class ChatInfoResultHandler implements ResultHandler { private final Client client; @@ -148,10 +144,11 @@ public void onResult(final Object object) { .groupId(groupId) .title(chat.title) .type(ChatType.getType(chat.type)) - .lastUpdated(System.currentTimeMillis()) .build(); - addOrUpdateChat(null, build); + // uppdate chat title + // -> chatId + SwApi.callDatabaseResult(SwApi.CHAT_SERVICE.addChat(build)); return; } @@ -184,7 +181,9 @@ public void onResult(final Object object) { } if (apiChat != null) { - addOrUpdateChat(null, apiChat); + // uppdate chat title + // -> chatId + SwApi.callDatabaseResult(SwApi.CHAT_SERVICE.addChat(apiChat)); } } } diff --git a/apps/message-crawler/telegram-crawler-core/src/main/java/io/d2a/schwurbelwatch/tgcrawler/api/SwApi.java b/apps/message-crawler/telegram-crawler-core/src/main/java/io/d2a/schwurbelwatch/tgcrawler/api/SwApi.java index c8a8044..152c4f0 100644 --- a/apps/message-crawler/telegram-crawler-core/src/main/java/io/d2a/schwurbelwatch/tgcrawler/api/SwApi.java +++ b/apps/message-crawler/telegram-crawler-core/src/main/java/io/d2a/schwurbelwatch/tgcrawler/api/SwApi.java @@ -27,7 +27,7 @@ public class SwApi { public static final Gson GSON = new GsonBuilder() .setPrettyPrinting() - .serializeNulls() + // .serializeNulls() .create(); private static final Retrofit retrofit = new Retrofit.Builder() diff --git a/apps/message-crawler/telegram-crawler-core/src/main/java/io/d2a/schwurbelwatch/tgcrawler/api/routes/chats/ApiChat.java b/apps/message-crawler/telegram-crawler-core/src/main/java/io/d2a/schwurbelwatch/tgcrawler/api/routes/chats/ApiChat.java index cbe5488..54aba54 100644 --- a/apps/message-crawler/telegram-crawler-core/src/main/java/io/d2a/schwurbelwatch/tgcrawler/api/routes/chats/ApiChat.java +++ b/apps/message-crawler/telegram-crawler-core/src/main/java/io/d2a/schwurbelwatch/tgcrawler/api/routes/chats/ApiChat.java @@ -29,7 +29,7 @@ public class ApiChat { public final Byte isScam; @SerializedName("last_updated") - public final long lastUpdated; + public final Long lastUpdated; public final Byte monitor; diff --git a/apps/message-crawler/telegram-crawler-core/src/main/java/io/d2a/schwurbelwatch/tgcrawler/api/routes/chats/ChatService.java b/apps/message-crawler/telegram-crawler-core/src/main/java/io/d2a/schwurbelwatch/tgcrawler/api/routes/chats/ChatService.java index 67087d9..0645293 100644 --- a/apps/message-crawler/telegram-crawler-core/src/main/java/io/d2a/schwurbelwatch/tgcrawler/api/routes/chats/ChatService.java +++ b/apps/message-crawler/telegram-crawler-core/src/main/java/io/d2a/schwurbelwatch/tgcrawler/api/routes/chats/ChatService.java @@ -15,8 +15,11 @@ public interface ChatService { @GET("chats") Call> getChats(@Query("offset") long offset); - @POST("chats") - Call addChat(@Body ApiChat chat); + @POST("chats/chatId") + Call addChatByChatId(@Body ApiChat chat); + + @POST("chats/groupId") + Call addChatByGroupId(@Body ApiChat chat); @GET("chats/:id") Call getChat(@Path("id") long chatId); @@ -27,4 +30,14 @@ public interface ChatService { @POST("chats/:id/count/:count") Call updateMemberCount(@Path("id") long chatId, @Path("count") int count); + default Call addChat(ApiChat chat) { + if (chat.chatId != null) { + return addChatByChatId(chat); + } else if (chat.groupId != null) { + return addChatByGroupId(chat); + } else { + return null; + } + } + } \ No newline at end of file diff --git a/apps/rest-api/src/controller/chatsController.js b/apps/rest-api/src/controller/chatsController.js index b4251ab..80412ed 100644 --- a/apps/rest-api/src/controller/chatsController.js +++ b/apps/rest-api/src/controller/chatsController.js @@ -15,7 +15,7 @@ const chatSchema = Joi.object({ title: Joi.string().default(null).allow("").allow(null).optional(), description: Joi.string().default(null).allow("").allow(null).optional(), member_count: Joi.number().min(0).default(0).allow("").allow(null).optional(), - type: Joi.string().default("unknown").allow("").allow(null).optional(), + type: Joi.string().default(null).allow("").allow(null).optional(), is_verified: Joi.number() .min(0) .max(1) @@ -30,7 +30,6 @@ const chatSchema = Joi.object({ .allow("") .allow(null) .optional(), - last_updated: Joi.number().default(-1).allow("").allow(null).optional(), // -1 = current time monitor: Joi.number() .min(0) .max(1) @@ -48,9 +47,8 @@ module.exports.getChats = async (limit = 200, offset = 0) => { return dbController.selectPaged("chats", "*", 1, limit, offset); }; -// POST /chats -module.exports.addChat = async (chat) => { - // validate user schema +// Template +async function updateChatByX(idField, chat, additionalFields) { const { error, value } = chatSchema.validate(chat); if (error) { return { @@ -59,174 +57,124 @@ module.exports.addChat = async (chat) => { }; } - if (value.chatId == null && value.groupId == null) { + if (chat[idField] == null) { return { error: true, - message: "chatId or groupId missing.", + message: idField + " required.", }; } - // get chat id and check - const chatId = value.chatId; - const groupId = value.groupId; + const fields = [ + "username", + "date", + "title", + "description", + "member_count", + "type", + "is_verified", + "is_scam", + ]; + // add additional fields + for (const idx in additionalFields) { + fields.push(additionalFields[idx]); + } - // current date in milliseconds const date = Date.now(); - - // get a connection from the database const connection = await dbController.pool.getConnection(); try { - // get existing chats if any exists - let rows = []; - if (value.chatId != null) { - rows = await connection.query( - "SELECT * FROM chats WHERE chatId = ? LIMIT 1;", - [parseInt(chatId)] - ); - } else if (value.groupId != null) { - rows = await connection.query( - "SELECT * FROM chats WHERE groupId = ? LIMIT 1;", - [groupId] - ); - } - - // there is an chat existing with the same chatId - if (rows.length >= 1) { - const oldChat = rows[0]; - console.log(oldChat); - - // check if we monitor this channel, if not, return with an error - if (oldChat.monitor == 0) { - return { - error: true, - message: "Not monitoring", - }; - } - - // we'll compare the following mysql & object keys - const fields = [ - "groupId", - "username", - "date", - "title", - "description", - "member_count", - "type", - "is_verified", - "is_scam", - "last_updated", - ]; + const rows = await connection.query( + `SELECT * FROM chats WHERE \`${idField}\` = ? LIMIT 1;`, + [parseInt(value[idField])] + ); + // no old chat found -> insert + if (rows.length !== 1) { + return await connection.query( + "INSERT INTO chats VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", + [ + value.chatId, + value.groupId, + value.username, + value.date, + value.title, + value.description, + value.member_count, + value.type, + value.is_verified, + value.is_scam, + date, + 1, + ] + ); + // old chat found -> update + } else { + const old = rows[0]; const update = { - query: "", - params: [], + // default: last_updated + last_updated: { + param: date, // new value + store: false, // save to update table + old: null, // old value + }, }; - for (let i = 0; i < fields.length; i++) { - const field = fields[i]; - - if (!(field in chat)) { + for (const indx in fields) { + const field = fields[indx]; + if (!(field in old) || !(field in value)) { continue; } - - const _old = oldChat[field]; - const _new = value[field]; - - if (_new == null) { + // check for null values + const val = value[field]; + if (val == null || val == undefined) { continue; } - - if (_old !== _new) { - console.log( - `[Chat Update | ${chatId}] Updating field ${field} from ${_old} to ${_new}` - ); - - update.query += - (update.query.length == 0 ? "" : ", ") + field + " = ?"; - update.params.push(_new); - - if (field == "last_updated") { - continue; - } - - // add to updates - await connection.query( - "INSERT INTO chats_updates (`chatId`, `key`, `old_value`, `new_value`, `date`) VALUES (?, ?, ?, ?, ?);", - [chatId, field, _old, _new, date] - ); + // check for equal values + const oldval = old[field]; + if (oldval === val) { + continue; } + // mark field as updated + update[field] = { + param: val, + old: oldval, + store: oldval != null + }; + } + + // check if something was updated + if (Object.keys(update).length === 1) { + return { + error: false, + message: "Nothing updated.", + }; } - // update ?! - if (update.query.length > 0) { - console.log(value); + console.log(update); - if (!update.query.includes("last_updated")) { - update.query += ", last_updated = ?"; - update.params.push(date); - } else if (update.query.length == 1) { - return { - error: true, - message: "Only updating last_updated.", - }; - } + let result = {}; - console.log( - "[Chat Update | " + - chatId + - "] Update chat " + - chatId + - " | " + - groupId + for (const key in update) { + const param = update[key]["param"]; + const store = update[key]["store"]; + const oldParam = update[key]["old"]; + + result = await connection.query( + `UPDATE chats SET \`${key}\` = ? WHERE \`${idField}\` = ?;`, + [param, value[idField]] ); - // update chat - if (value.chatId != null) { - console.log("by chat id"); - console.log(update); - update.params.push(chatId); // this is used for the where clause and should be here. - return await connection.query( - "UPDATE chats SET " + update.query + " WHERE chatId = ?;", - update.params - ); - } else if (value.groupId != null) { - console.log("by group id"); - console.log(update); - update.params.push(groupId); // this is used for the where clause and should be here. - return await connection.query( - "UPDATE chats SET " + update.query + " WHERE groupId = ?;", - update.params + if (store) { + // add to updates + await connection.query( + "INSERT INTO chats_updates (`" + idField + "`, `key`, `old_value`, `new_value`, `date`) VALUES (?, ?, ?, ?, ?);", + [parseInt(value[idField]), key, oldParam, param, date] ); } - } else { - console.log("[Chat Update | " + chatId + "] No need to update"); } - } else { - // insert new chat - return await connection.query( - "INSERT INTO chats VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - [ - value.chatId, - value.groupId, - value.username, - value.date, - value.title, - value.description, - value.member_count, - value.type, - value.is_verified, - value.is_scam, - value.last_updated, - 1, - ] - ); - } - return { - error: false, - message: "Nothing updated.", - }; + return result; + } } catch (exception) { return { error: true, @@ -237,6 +185,14 @@ module.exports.addChat = async (chat) => { connection.end(); } } +} + +module.exports.addChatChatId = async (chat) => { + return updateChatByX("chatId", chat, ["groupId"]); +}; + +module.exports.addChatGroupId = async (chat) => { + return updateChatByX("groupId", chat, []); }; // GET /chats/:id diff --git a/apps/rest-api/src/routes/chats.js b/apps/rest-api/src/routes/chats.js index 1789966..889571e 100644 --- a/apps/rest-api/src/routes/chats.js +++ b/apps/rest-api/src/routes/chats.js @@ -30,8 +30,13 @@ router.get("/", async (req, res) => { * POST /chats * Stores a new chat */ -router.post("/", async (req, res) => { - const controllerResult = await controller.addChat(req.body); +router.post("/chatId", async (req, res) => { + const controllerResult = await controller.addChatChatId(req.body); + return res.status(controllerResult.error ? 400 : 200).json(controllerResult); +}); + +router.post("/groupId", async (req, res) => { + const controllerResult = await controller.addChatGroupId(req.body); return res.status(controllerResult.error ? 400 : 200).json(controllerResult); });