From 456273be96e99cc608c12d93ce443aea172e7cd4 Mon Sep 17 00:00:00 2001 From: Eyre_S Date: Wed, 7 Feb 2024 00:12:13 +0800 Subject: [PATCH] add event sources statistics, --- build.sbt | 4 +- project/MornyConfiguration.scala | 2 +- .../morny/core/bot/api/EventListener.scala | 11 +++++ .../core/bot/api/EventListenerManager.scala | 1 + .../cono/morny/reporter/MornyReport.scala | 40 +++++++++++++++-- .../event_hack/HackerEventHandler.scala | 44 +++---------------- .../cono/morny/util/CommonEncrypt.scala | 10 +++++ .../morny/util/statistics/UniqueCounter.scala | 27 ++++++++++++ .../morny/util/tgapi/TelegramExtensions.scala | 38 ++++++++++++++++ .../tgapi/formatting/TelegramFormatter.scala | 25 ++++++++--- 10 files changed, 153 insertions(+), 49 deletions(-) create mode 100644 src/main/scala/cc/sukazyo/cono/morny/util/statistics/UniqueCounter.scala diff --git a/build.sbt b/build.sbt index 561166e..7e298be 100644 --- a/build.sbt +++ b/build.sbt @@ -68,9 +68,7 @@ lazy val root = (project in file(".")) .withClassifier(Some("fat")), if (MornyProject.publishWithFatJar) { addArtifact(assembly / artifact, assembly) - } else { - Nil - }, + } else Nil, if (System.getenv("DOCKER_BUILD") != null) { assembly / assemblyJarName := { sLog.value info "environment DOCKER_BUILD checked" diff --git a/project/MornyConfiguration.scala b/project/MornyConfiguration.scala index 4958e9e..ff2184c 100644 --- a/project/MornyConfiguration.scala +++ b/project/MornyConfiguration.scala @@ -8,7 +8,7 @@ object MornyConfiguration { val MORNY_CODE_STORE = "https://github.com/Eyre-S/Coeur-Morny-Cono" val MORNY_COMMIT_PATH = "https://github.com/Eyre-S/Coeur-Morny-Cono/commit/%s" - val VERSION = "2.0.0-alpha11" + val VERSION = "2.0.0-alpha12" val VERSION_DELTA: Option[String] = None val CODENAME = "guanggu" diff --git a/src/main/scala/cc/sukazyo/cono/morny/core/bot/api/EventListener.scala b/src/main/scala/cc/sukazyo/cono/morny/core/bot/api/EventListener.scala index a8da606..a01e9ce 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/core/bot/api/EventListener.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/core/bot/api/EventListener.scala @@ -28,6 +28,17 @@ trait EventListener () { */ def atEventPost (using EventEnv): Unit = {} + /** A overall event listener that can listen every types that supported + * by the bot API. + * + * This method will runs before the specific event listener methods. + * + * [[executeFilter]] will affect this method. + * + * @since 2.0.0 + */ + def on (using EventEnv): Unit = {} + def onMessage (using EventEnv): Unit = {} def onEditedMessage (using EventEnv): Unit = {} def onChannelPost (using EventEnv): Unit = {} diff --git a/src/main/scala/cc/sukazyo/cono/morny/core/bot/api/EventListenerManager.scala b/src/main/scala/cc/sukazyo/cono/morny/core/bot/api/EventListenerManager.scala index a2e4fa4..78466a7 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/core/bot/api/EventListenerManager.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/core/bot/api/EventListenerManager.scala @@ -48,6 +48,7 @@ class EventListenerManager (using coeur: MornyCoeur) extends UpdatesListener { private def runEventListener (i: EventListener)(using EventEnv): Unit = { try { + i.on updateThreadName("message") if update.message ne null then i.onMessage updateThreadName("edited-message") diff --git a/src/main/scala/cc/sukazyo/cono/morny/reporter/MornyReport.scala b/src/main/scala/cc/sukazyo/cono/morny/reporter/MornyReport.scala index c0e4b59..1d26cdb 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/reporter/MornyReport.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/reporter/MornyReport.scala @@ -4,19 +4,22 @@ import cc.sukazyo.cono.morny.core.{MornyCoeur, MornyConfig} import cc.sukazyo.cono.morny.core.Log.{exceptionLog, logger} import cc.sukazyo.cono.morny.core.bot.api.{EventEnv, EventListener} import cc.sukazyo.cono.morny.data.MornyInformation.getVersionAllFullTagHTML -import cc.sukazyo.cono.morny.util.statistics.NumericStatistics +import cc.sukazyo.cono.morny.util.statistics.{NumericStatistics, UniqueCounter} import cc.sukazyo.cono.morny.util.tgapi.event.EventRuntimeException import cc.sukazyo.cono.morny.util.tgapi.formatting.TelegramFormatter.* import cc.sukazyo.cono.morny.util.tgapi.formatting.TelegramParseEscape.escapeHtml as h import cc.sukazyo.cono.morny.util.tgapi.TelegramExtensions.Bot.exec import cc.sukazyo.cono.morny.util.EpochDateTime.DurationMillis import cc.sukazyo.cono.morny.util.schedule.CronTask +import cc.sukazyo.cono.morny.util.tgapi.TelegramExtensions.Update.{extractSourceChat, extractSourceUser} +import cc.sukazyo.cono.morny.util.CommonEncrypt.hashId +import cc.sukazyo.cono.morny.util.ConvertByteHex.toHex import com.cronutils.builder.CronBuilder import com.cronutils.model.Cron import com.cronutils.model.definition.CronDefinitionBuilder import com.google.gson.GsonBuilder +import com.pengrad.telegrambot.model.{Chat, User} import com.pengrad.telegrambot.model.request.ParseMode -import com.pengrad.telegrambot.model.User import com.pengrad.telegrambot.request.{BaseRequest, SendMessage} import com.pengrad.telegrambot.response.BaseResponse import com.pengrad.telegrambot.TelegramException @@ -30,7 +33,7 @@ class MornyReport (using coeur: MornyCoeur) { logger info "Morny Report is disabled : report chat is set to -1" private def executeReport[T <: BaseRequest[T, R], R<: BaseResponse] (report: T): Unit = { - if !enabled then return; + if !enabled then return try { coeur.account exec report } catch case e: EventRuntimeException => { @@ -148,11 +151,23 @@ class MornyReport (using coeur: MornyCoeur) { private var eventTotal = 0 private var eventCanceled = 0 private val runningTime: NumericStatistics[DurationMillis] = NumericStatistics() + /** The event which is from a private chat (mostly message) */ + private val event_from_private = UniqueCounter[String]() + /** The event which is from a group (message, or member join etc.) */ + private val event_from_group = UniqueCounter[String]() + /** The event which is from a channel (message, or member join etc.) */ + private val event_from_channel = UniqueCounter[String]() + /** The event which is from a user's action (inline queries etc. which have a executor but not belongs to a chat.) */ + private val event_from_user_action = UniqueCounter[String]() def reset (): Unit = { eventTotal = 0 eventCanceled = 0 runningTime.reset() + event_from_private.reset() + event_from_group.reset() + event_from_channel.reset() + event_from_user_action.reset() } private def runningTimeStatisticsHTML: String = @@ -168,11 +183,16 @@ class MornyReport (using coeur: MornyCoeur) { def eventStatisticsHTML: String = import cc.sukazyo.cono.morny.util.UseMath.percentageOf as p + import cc.sukazyo.cono.morny.util.tgapi.formatting.TelegramFormatter.ChatTypeTag.* val processed = runningTime.count val canceled = eventCanceled val ignored = eventTotal - processed - canceled // language=html s""" - total event received: $eventTotal + | - from ${event_from_channel.count} $CHANNEL channels + | - from ${event_from_group.count} $SUPERGROUP groups/supergroups + | - from ${event_from_private.count} $PRIVATE private chats + | - from ${event_from_user_action.count} 😼 user actions | - event ignored: (${eventTotal p ignored}%) $ignored | - event canceled: (${eventTotal p canceled}%) $canceled | - event processed: (${eventTotal p processed}%) $processed @@ -186,6 +206,20 @@ class MornyReport (using coeur: MornyCoeur) { override def atEventPost (using event: EventEnv): Unit = { import event.* eventTotal += 1 + event.update.extractSourceChat match + case None => + event.update.extractSourceUser match + case None => + case Some(user) => + event_from_user_action << hashId(user.id).toHex + case Some(chat) => + chat.`type` match + case Chat.Type.Private => + event_from_private << hashId(chat.id).toHex + case Chat.Type.group | Chat.Type.supergroup => + event_from_group << hashId(chat.id).toHex + case Chat.Type.channel => + event_from_channel << hashId(chat.id).toHex event.state match case State.OK(from) => val timeUsed = EventTimeUsed(System.currentTimeMillis - event.timeStartup) diff --git a/src/main/scala/cc/sukazyo/cono/morny/tele_utils/event_hack/HackerEventHandler.scala b/src/main/scala/cc/sukazyo/cono/morny/tele_utils/event_hack/HackerEventHandler.scala index 58ae45b..e6f46eb 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/tele_utils/event_hack/HackerEventHandler.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/tele_utils/event_hack/HackerEventHandler.scala @@ -1,50 +1,20 @@ package cc.sukazyo.cono.morny.tele_utils.event_hack -import cc.sukazyo.cono.morny.core.Log.logger import cc.sukazyo.cono.morny.core.MornyCoeur import cc.sukazyo.cono.morny.core.bot.api.{EventEnv, EventListener} -import com.google.gson.GsonBuilder +import cc.sukazyo.cono.morny.util.tgapi.TelegramExtensions.Update.* import com.pengrad.telegrambot.model.Update -import com.pengrad.telegrambot.model.request.ParseMode -import com.pengrad.telegrambot.request.SendMessage -import scala.collection.mutable import scala.language.postfixOps class HackerEventHandler (using hacker: EventHacker)(using coeur: MornyCoeur) extends EventListener { - private def trigger (chat_id: Long, from_id: Long)(using event: EventEnv): Unit = - given Update = event.update - if hacker.trigger(chat_id, from_id) then + override def on (using event: EventEnv): Unit = + given update: Update = event.update + if hacker.trigger( + update.extractSourceChat.map[Long](_.id).getOrElse(0), + update.extractSourceUser.map[Long](_.id).getOrElse(0) + ) then event.setEventOk - override def onMessage (using event: EventEnv): Unit = - trigger(event.update.message.chat.id, event.update.message.from.id) - override def onEditedMessage (using event: EventEnv): Unit = - trigger(event.update.editedMessage.chat.id, event.update.editedMessage.from.id) - override def onChannelPost (using event: EventEnv): Unit = - trigger(event.update.channelPost.chat.id, 0) - override def onEditedChannelPost (using event: EventEnv): Unit = - trigger(event.update.editedChannelPost.chat.id, 0) - override def onInlineQuery (using event: EventEnv): Unit = - trigger(0, event.update.inlineQuery.from.id) - override def onChosenInlineResult (using event: EventEnv): Unit = - trigger(0, event.update.chosenInlineResult.from.id) - override def onCallbackQuery (using event: EventEnv): Unit = - trigger(0, event.update.callbackQuery.from.id) - override def onShippingQuery (using event: EventEnv): Unit = - trigger(0, event.update.shippingQuery.from.id) - override def onPreCheckoutQuery (using event: EventEnv): Unit = - trigger(0, event.update.preCheckoutQuery.from.id) - override def onPoll (using event: EventEnv): Unit = - trigger(0, 0) - override def onPollAnswer (using event: EventEnv): Unit = - trigger(0, event.update.pollAnswer.user.id) - override def onMyChatMemberUpdated (using event: EventEnv): Unit = - trigger(event.update.myChatMember.chat.id, event.update.myChatMember.from.id) - override def onChatMemberUpdated (using event: EventEnv): Unit = - trigger(event.update.chatMember.chat.id, event.update.chatMember.from.id) - override def onChatJoinRequest (using event: EventEnv): Unit = - trigger(event.update.chatJoinRequest.chat.id, event.update.chatJoinRequest.from.id) - } diff --git a/src/main/scala/cc/sukazyo/cono/morny/util/CommonEncrypt.scala b/src/main/scala/cc/sukazyo/cono/morny/util/CommonEncrypt.scala index 740e048..dfc4752 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/util/CommonEncrypt.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/util/CommonEncrypt.scala @@ -95,4 +95,14 @@ object CommonEncrypt { case lx if lx endsWith ".base64.txt" => lx dropRight ".base64.txt".length case u => u + /** Hash a [[Long]] id to [[Bin]] using [[MD5]] algorithm. + * + * For some privacy cases, this method can provide a standard way to hash a ID to a MD5 hash value. + * + * @param id The [[Long]] number typed id. + * @return The hash value of the id. + */ + def hashId (id: Long): Bin = + MD5(id.toString) + } diff --git a/src/main/scala/cc/sukazyo/cono/morny/util/statistics/UniqueCounter.scala b/src/main/scala/cc/sukazyo/cono/morny/util/statistics/UniqueCounter.scala new file mode 100644 index 0000000..3d88658 --- /dev/null +++ b/src/main/scala/cc/sukazyo/cono/morny/util/statistics/UniqueCounter.scala @@ -0,0 +1,27 @@ +package cc.sukazyo.cono.morny.util.statistics + +import scala.collection.mutable + +/** Count unique elements progressively. + * + * Use [[<<]] to add a element to this counter. Use [[count]] to get current + * count in this counter, and use [[reset()]] to reset this counter. + * + * Behind it is a [[scala.collection.mutable.Set]]. + * + * @tparam T The element type. + */ +class UniqueCounter [T] { + + private var set: mutable.Set[T] = mutable.Set.empty + + def << (t: T): Unit = + set += t + + def count: Int = + set.size + + def reset(): Unit = + set.clear() + +} diff --git a/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/TelegramExtensions.scala b/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/TelegramExtensions.scala index 1a8da95..5d48f76 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/TelegramExtensions.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/TelegramExtensions.scala @@ -29,6 +29,44 @@ object TelegramExtensions { }} + object Update { extension (update: Update) { + + def extractSourceChat: Option[Chat] = + if (update.message != null) Some(update.message.chat) + else if (update.editedMessage != null) Some(update.editedMessage.chat) + else if (update.channelPost != null) Some(update.channelPost.chat) + else if (update.editedChannelPost != null) Some(update.editedChannelPost.chat) + else if (update.inlineQuery != null) None + else if (update.chosenInlineResult != null) None + else if (update.callbackQuery != null) Some(update.callbackQuery.message.chat) + else if (update.shippingQuery != null) None + else if (update.preCheckoutQuery != null) None + else if (update.poll != null) None + else if (update.pollAnswer != null) None + else if (update.myChatMember != null) Some(update.myChatMember.chat) + else if (update.chatMember != null) Some(update.chatMember.chat) + else if (update.chatJoinRequest != null) Some(update.chatJoinRequest.chat) + else None + + def extractSourceUser: Option[User] = + if (update.message != null) Some(update.message.from) + else if (update.editedMessage != null) Some(update.editedMessage.from) + else if (update.channelPost != null) None + else if (update.editedChannelPost != null) None + else if (update.inlineQuery != null) Some(update.inlineQuery.from) + else if (update.chosenInlineResult != null) Some(update.chosenInlineResult.from) + else if (update.callbackQuery != null) Some(update.callbackQuery.from) + else if (update.shippingQuery != null) Some(update.shippingQuery.from) + else if (update.preCheckoutQuery != null) Some(update.preCheckoutQuery.from) + else if (update.poll != null) None + else if (update.pollAnswer != null) Some(update.pollAnswer.user) + else if (update.myChatMember != null) Some(update.myChatMember.from) + else if (update.chatMember != null) Some(update.chatMember.from) + else if (update.chatJoinRequest != null) Some(update.chatJoinRequest.from) + else None + + }} + object Chat { extension (chat: Chat) { def hasMember (user: User) (using TelegramBot): Boolean = diff --git a/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/formatting/TelegramFormatter.scala b/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/formatting/TelegramFormatter.scala index 48fd6ef..6ed891a 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/formatting/TelegramFormatter.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/formatting/TelegramFormatter.scala @@ -34,11 +34,26 @@ object TelegramFormatter { def id_tdLib: Long = if chat.id < 0 then (chat.id - MASK_BOTAPI_ID)abs else chat.id - def typeTag: String = chat.`type` match - case Type.Private => "🔒" - case Type.group => "💭" - case Type.supergroup => "💬" - case Type.channel => "📢" + def typeTag: String = + import ChatTypeTag.tag + chat.`type`.tag + + } + + object ChatTypeTag { + + inline val PRIVATE = "🔒" + inline val GROUP = "💭" + inline val SUPERGROUP = "💬" + inline val CHANNEL = "📢" + + extension (t: Type) { + def tag: String = t match + case Type.Private => this.PRIVATE + case Type.group => this.GROUP + case Type.supergroup => this.SUPERGROUP + case Type.channel => this.CHANNEL + } }