From a9767ec1b07f8a667fad51cf36e0c320a803d323 Mon Sep 17 00:00:00 2001 From: Eyre_S Date: Mon, 27 Nov 2023 18:58:35 +0800 Subject: [PATCH] change /tweet to /get and added support for weibo content --- build.gradle | 1 + gradle.properties | 2 +- .../command/{Tweet.scala => GetSocial.scala} | 70 ++++++++++++-- .../morny/bot/command/MornyCommands.scala | 2 +- .../cono/morny/data/twitter/FXApi.scala | 10 +- .../sukazyo/cono/morny/data/weibo/MApi.scala | 66 +++++++++++++ .../sukazyo/cono/morny/data/weibo/MPic.scala | 33 +++++++ .../cono/morny/data/weibo/MStatus.scala | 92 +++++++++++++++++++ .../sukazyo/cono/morny/data/weibo/MUser.scala | 13 +++ .../cono/morny/data/weibo/package.scala | 40 ++++++++ .../morny/util/tgapi/TelegramExtensions.scala | 6 +- .../formatting/TelegramParseEscape.scala | 57 ++++++++++++ 12 files changed, 374 insertions(+), 18 deletions(-) rename src/main/scala/cc/sukazyo/cono/morny/bot/command/{Tweet.scala => GetSocial.scala} (53%) create mode 100644 src/main/scala/cc/sukazyo/cono/morny/data/weibo/MApi.scala create mode 100644 src/main/scala/cc/sukazyo/cono/morny/data/weibo/MPic.scala create mode 100644 src/main/scala/cc/sukazyo/cono/morny/data/weibo/MStatus.scala create mode 100644 src/main/scala/cc/sukazyo/cono/morny/data/weibo/MUser.scala create mode 100644 src/main/scala/cc/sukazyo/cono/morny/data/weibo/package.scala diff --git a/build.gradle b/build.gradle index cc32c16..a436ee8 100644 --- a/build.gradle +++ b/build.gradle @@ -92,6 +92,7 @@ dependencies { implementation group: 'io.circe', name: scala('circe-core'), version: lib_circe_v implementation group: 'io.circe', name: scala('circe-generic'), version: lib_circe_v implementation group: 'io.circe', name: scala('circe-parser'), version: lib_circe_v + implementation group: 'org.jsoup', name: 'jsoup', version: '1.16.2' implementation group: 'com.cronutils', name: 'cron-utils', version: lib_cron_utils_v // used for disable slf4j diff --git a/gradle.properties b/gradle.properties index aab8e75..51fff15 100644 --- a/gradle.properties +++ b/gradle.properties @@ -5,7 +5,7 @@ MORNY_ARCHIVE_NAME = morny-coeur MORNY_CODE_STORE = https://github.com/Eyre-S/Coeur-Morny-Cono MORNY_COMMIT_PATH = https://github.com/Eyre-S/Coeur-Morny-Cono/commit/%s -VERSION = 1.3.0-dev10 +VERSION = 1.3.0-dev11 USE_DELTA = false VERSION_DELTA = diff --git a/src/main/scala/cc/sukazyo/cono/morny/bot/command/Tweet.scala b/src/main/scala/cc/sukazyo/cono/morny/bot/command/GetSocial.scala similarity index 53% rename from src/main/scala/cc/sukazyo/cono/morny/bot/command/Tweet.scala rename to src/main/scala/cc/sukazyo/cono/morny/bot/command/GetSocial.scala index 247819e..3b96200 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/bot/command/Tweet.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/bot/command/GetSocial.scala @@ -1,20 +1,23 @@ package cc.sukazyo.cono.morny.bot.command -import cc.sukazyo.cono.morny.data.{twitter, TelegramStickers} +import cc.sukazyo.cono.morny.data.{twitter, weibo, TelegramStickers} import cc.sukazyo.cono.morny.util.tgapi.InputCommand import cc.sukazyo.cono.morny.MornyCoeur import cc.sukazyo.cono.morny.data.twitter.{FXApi, TweetUrlInformation} import cc.sukazyo.cono.morny.util.tgapi.TelegramExtensions.Bot.exec import cc.sukazyo.cono.morny.Log.{exceptionLog, logger} +import cc.sukazyo.cono.morny.data.weibo.StatusUrlInfo import com.pengrad.telegrambot.model.Update import com.pengrad.telegrambot.model.request.{InputMedia, InputMediaPhoto, InputMediaVideo, ParseMode} import com.pengrad.telegrambot.request.{SendMediaGroup, SendMessage, SendSticker} +import io.circe.{DecodingFailure, ParsingFailure} +import sttp.client3.{HttpError, SttpClientException} -class Tweet (using coeur: MornyCoeur) extends ITelegramCommand { +class GetSocial (using coeur: MornyCoeur) extends ITelegramCommand { - override val name: String = "tweet" + override val name: String = "get" override val aliases: Array[ICommandAlias] | Null = null - override val paramRule: String = "" - override val description: String = "获取 Twitter(X) Tweet 内容" + override val paramRule: String = "" + override val description: String = "从社交媒体分享链接获取其内容" override def execute (using command: InputCommand, event: Update): Unit = { @@ -26,9 +29,11 @@ class Tweet (using coeur: MornyCoeur) extends ITelegramCommand { if command.args.length < 1 then { do404(); return } + var succeed = 0 twitter.parseTweetUrl(command.args(0)) match - case None => do404() + case None => case Some(TweetUrlInformation(_, _, screenName, statusId, _, _)) => + succeed += 1 try { val api = FXApi.Fetch.status(Some(screenName), statusId) import cc.sukazyo.cono.morny.util.tgapi.formatting.TelegramParseEscape.escapeHtml as h @@ -72,15 +77,64 @@ class Tweet (using coeur: MornyCoeur) extends ITelegramCommand { event.message.chat.id, mediaGroup:_* ).replyToMessageId(event.message.messageId) - } catch case e: Exception => + } catch case e: (SttpClientException|ParsingFailure|DecodingFailure) => coeur.account exec SendSticker( event.message.chat.id, TelegramStickers.ID_NETWORK_ERR ).replyToMessageId(event.message.messageId) - logger attention + logger error "Error on requesting FixTweet API\n" + exceptionLog(e) coeur.daemons.reporter.exception(e, "Error on requesting FixTweet API") + weibo.parseWeiboStatusUrl(command.args(0)) match + case None => + case Some(StatusUrlInfo(_, id)) => + succeed += 1 + try { + val api = weibo.MApi.Fetch.statuses_show(id) + import cc.sukazyo.cono.morny.util.tgapi.formatting.TelegramParseEscape.{cleanupHtml as ch, escapeHtml as h} + val content = + // language=html + s"""🔸${h(api.data.user.screen_name)} + | + |${ch(api.data.text)} + | + |${h(api.data.created_at)}""".stripMargin + api.data.pics match + case None => + coeur.account exec SendMessage( + event.message.chat.id, + content + ).replyToMessageId(event.message.messageId).parseMode(ParseMode.HTML) + case Some(pics) => +// val mediaGroup = pics.map(f => +// InputMediaPhoto(weibo.PicUrl(weibo.randomPicCdn, "large", f.pid).toUrl)) + val mediaGroup = pics.map(f => InputMediaPhoto(weibo.MApi.Fetch.pic(f.large.url))) + mediaGroup.head.caption(content) + mediaGroup.head.parseMode(ParseMode.HTML) + coeur.account exec SendMediaGroup( + event.message.chat.id, + mediaGroup:_* + ).replyToMessageId(event.message.messageId) + } catch + case e: HttpError[?] => + coeur.account exec SendMessage( + event.message.chat.id, + // language=html + s"""Weibo Request Error ${e.statusCode} + |
${e.body}
""".stripMargin + ).replyToMessageId(event.message.messageId).parseMode(ParseMode.HTML) + case e: (SttpClientException|ParsingFailure|DecodingFailure) => + coeur.account exec SendSticker( + event.message.chat.id, + TelegramStickers.ID_NETWORK_ERR + ).replyToMessageId(event.message.messageId) + logger error + "Error on requesting Weibo m.API\n" + exceptionLog(e) + coeur.daemons.reporter.exception(e, "Error on requesting Weibo m.API") + + if succeed == 0 then do404() + } } diff --git a/src/main/scala/cc/sukazyo/cono/morny/bot/command/MornyCommands.scala b/src/main/scala/cc/sukazyo/cono/morny/bot/command/MornyCommands.scala index 160c811..d46b48a 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/bot/command/MornyCommands.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/bot/command/MornyCommands.scala @@ -44,7 +44,7 @@ class MornyCommands (using coeur: MornyCoeur) { $IP186Query.Whois, Encryptor(), MornyOldJrrp(), - Tweet(), + GetSocial(), $MornyManagers.SaveData, $MornyInformation, diff --git a/src/main/scala/cc/sukazyo/cono/morny/data/twitter/FXApi.scala b/src/main/scala/cc/sukazyo/cono/morny/data/twitter/FXApi.scala index 828aba1..7f5ce51 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/data/twitter/FXApi.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/data/twitter/FXApi.scala @@ -91,18 +91,16 @@ object FXApi { @throws[SttpClientException|ParsingFailure|DecodingFailure] def status (screen_name: Option[String], id: String, translate_to: Option[String] = None): FXApi = val get = mornyBasicRequest - .header(SttpPublic.Headers.UserAgent.MORNY_CURRENT) .get(uri_status(screen_name, id, translate_to)) .response(asString) .send(httpClient) val body = get.body match case Left(error) => error case Right(success) => success - parser.parse(body) match - case Left(error) => throw error - case Right(value) => value.as[FXApi] match - case Left(error) => throw error - case Right(value) => value + parser.parse(body) + .toTry.get + .as[FXApi] + .toTry.get } diff --git a/src/main/scala/cc/sukazyo/cono/morny/data/weibo/MApi.scala b/src/main/scala/cc/sukazyo/cono/morny/data/weibo/MApi.scala new file mode 100644 index 0000000..a576db5 --- /dev/null +++ b/src/main/scala/cc/sukazyo/cono/morny/data/weibo/MApi.scala @@ -0,0 +1,66 @@ +package cc.sukazyo.cono.morny.data.weibo + +case class MApi [D] ( + ok: Int, + data: D +) + +object MApi { + + object CirceADTs { + import io.circe.Decoder + import io.circe.generic.semiauto.deriveDecoder + given Decoder[MUser] = deriveDecoder + given given_Decoder_largeType_getType: Decoder[MPic.largeType.geoType] = deriveDecoder + given Decoder[MPic.largeType] = deriveDecoder + given Decoder[MPic.geoType] = deriveDecoder + given Decoder[MPic] = deriveDecoder + given Decoder[MStatus] = deriveDecoder + given Decoder[MApi[MStatus]] = deriveDecoder + } + + object Fetch { + + import cc.sukazyo.cono.morny.util.SttpPublic + import cc.sukazyo.cono.morny.util.SttpPublic.mornyBasicRequest + import io.circe.{parser, DecodingFailure, ParsingFailure} + import sttp.client3.{HttpError, SttpClientException, UriContext} + import sttp.client3.okhttp.OkHttpSyncBackend + + val uri_base = uri"https://m.weibo.cn/" + val uri_statuses_show = + (id: String) => uri"$uri_base/statuses/show?id=$id" + + private val httpClient = OkHttpSyncBackend() + + @throws[HttpError[_]|SttpClientException|ParsingFailure|DecodingFailure] + def statuses_show (id: String): MApi[MStatus] = + import sttp.client3.asString + import MApi.CirceADTs.given + val response = mornyBasicRequest + .get(uri_statuses_show(id)) + .response(asString.getRight) + .send(httpClient) + parser.parse(response.body) + .toTry.get + .as[MApi[MStatus]] + .toTry.get + + @throws[HttpError[_] | SttpClientException | ParsingFailure | DecodingFailure] + def pic (picUrl: String): Array[Byte] = + import sttp.client3.* + import sttp.model.{MediaType, Uri} + mornyBasicRequest + .acceptEncoding(MediaType.ImageJpeg.toString) + .get(Uri.unsafeParse(picUrl)) + .response(asByteArray.getRight) + .send(httpClient) + .body + +// @throws[HttpError[_] | SttpClientException | ParsingFailure | DecodingFailure] +// def pic (info: PicUrl): Array[Byte] = +// pic(info.toUrl) + + } + +} diff --git a/src/main/scala/cc/sukazyo/cono/morny/data/weibo/MPic.scala b/src/main/scala/cc/sukazyo/cono/morny/data/weibo/MPic.scala new file mode 100644 index 0000000..9701728 --- /dev/null +++ b/src/main/scala/cc/sukazyo/cono/morny/data/weibo/MPic.scala @@ -0,0 +1,33 @@ +package cc.sukazyo.cono.morny.data.weibo + +case class MPic ( + pid: String, + url: String, + size: String, + geo: MPic.geoType, + large: MPic.largeType +) + +object MPic { + + case class geoType ( +// width: Int, +// height: Int, + croped: Boolean + ) + + case class largeType ( + size: String, + url: String, + geo: largeType.geoType + ) + + object largeType { + case class geoType ( +// width: String, +// height: String, + croped: Boolean + ) + } + +} diff --git a/src/main/scala/cc/sukazyo/cono/morny/data/weibo/MStatus.scala b/src/main/scala/cc/sukazyo/cono/morny/data/weibo/MStatus.scala new file mode 100644 index 0000000..b7571ab --- /dev/null +++ b/src/main/scala/cc/sukazyo/cono/morny/data/weibo/MStatus.scala @@ -0,0 +1,92 @@ +package cc.sukazyo.cono.morny.data.weibo + +case class MStatus ( + + id: String, + mid: String, + bid: String, + + created_at: String, + text: String, + raw_text: Option[String], + + user: MUser, + + retweeted_status: Option[MStatus], + + pic_ids: List[String], + pics: Option[List[MPic]], + thumbnail_pic: Option[String], + bmiddle_pic: Option[String], + original_pic: Option[String], + +// visible: Nothing, +// created_at: String, +// id: String, +// mid: String, +// bid: String, +// can_edit: Boolean, +// show_additional_indication: Int, +// text: String, +// textLength: Option[Int], +// source: String, +// favorited: Boolean, +// pic_ids: List[String], +// pic_focus_point: Option[List[Nothing]], +// falls_pic_focus_point: Option[List[Nothing]], +// pic_rectangle_object: Option[List[Nothing]], +// pic_flag: Option[Int], +// thumbnail_pic: Option[String], +// bmiddle_pic: Option[String], +// original_pic: Option[String], +// is_paid: Boolean, +// mblog_vip_type: Int, +// user: Nothing, +// picStatus: Option[String], +// retweeted_status: Option[Nothing], +// reposts_count: Int, +// comments_count: Int, +// reprint_cmt_count: Int, +// attitudes_count: Int, +// pending_approval_count: Int, +// isLongText: Boolean, +// show_mlevel: Int, +// topic_id: Option[String], +// sync_mblog: Option[Boolean], +// is_imported_topic: Option[Boolean], +// darwin_tags: List[Nothing], +// ad_marked: Boolean, +// mblogtype: Int, +// item_category: String, +// rid: String, +// number_display_strategy: Nothing, +// content_auth: Int, +// safe_tags: Option[Int], +// comment_manage_info: Nothing, +// repost_type: Option[Int], +// pic_num: Int, +// jump_type: Option[Int], +// hot_page: Nothing, +// new_comment_style: Int, +// ab_switcher: Int, +// mlevel: Int, +// region_name: String, +// region_opt: 1, +// page_info: Option[Nothing], +// pics: Option[List[Nothing]], +// raw_text: Option[String], +// buttons: List[Nothing], +// status_title: Option[String], +// ok: Int, + + +// pid: Long, +// pidstr: String, +// pic_types: String, +// alchemy_params: Nothing, +// ad_state: Int, +// cardid: String, +// hide_flag: Int, +// mark: String, +// more_info_type: Int, +) diff --git a/src/main/scala/cc/sukazyo/cono/morny/data/weibo/MUser.scala b/src/main/scala/cc/sukazyo/cono/morny/data/weibo/MUser.scala new file mode 100644 index 0000000..8a8d8d7 --- /dev/null +++ b/src/main/scala/cc/sukazyo/cono/morny/data/weibo/MUser.scala @@ -0,0 +1,13 @@ +package cc.sukazyo.cono.morny.data.weibo + +case class MUser ( + + id: Long, + screen_name: String, + profile_url: String, + profile_image_url: Option[String], + avatar_hd: Option[String], + description: Option[String], + cover_image_phone: Option[String], + +) diff --git a/src/main/scala/cc/sukazyo/cono/morny/data/weibo/package.scala b/src/main/scala/cc/sukazyo/cono/morny/data/weibo/package.scala new file mode 100644 index 0000000..19b9dcb --- /dev/null +++ b/src/main/scala/cc/sukazyo/cono/morny/data/weibo/package.scala @@ -0,0 +1,40 @@ +package cc.sukazyo.cono.morny.data + +package object weibo { + + /** Information in weibo status url. + * + * @param uid Status owner's user id. should be a number. + * @param id Status id. Should be unique in the whole weibo.com + * globe. Maybe a number format mid, or a base58-like + * bid. + */ + case class StatusUrlInfo ( + uid: String, + id: String + ) + +// case class PicUrl ( +// cdn: String, +// mode: String, +// pid: String +// ) { +// def toUrl: String = +// s"https://$cdn.singimg.cn/$mode/$pid.jpg" +// } + + private val REGEX_WEIBO_STATUS_URL = "^(?:https?://)?((?:www\\.|m.)?weibo\\.(?:com|cn))/(\\d+)/([0-9a-zA-Z]+)/?(?:\\?([\\w&=-]+))?$"r + + def parseWeiboStatusUrl (url: String): Option[StatusUrlInfo] = + url match + case REGEX_WEIBO_STATUS_URL(_, uid, id, _) => Some(StatusUrlInfo(uid, id)) + case _ => None + + def genWeiboStatusUrl (url: StatusUrlInfo): String = + s"https://weibo.com/${url.uid}/${url.id}" + +// def randomPicCdn: String = +// import scala.util.Random +// s"wx${Random.nextInt(4)+1}" + +} diff --git a/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/TelegramExtensions.scala b/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/TelegramExtensions.scala index f066792..d4b9bb3 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/TelegramExtensions.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/TelegramExtensions.scala @@ -21,8 +21,10 @@ object TelegramExtensions { if onError_message isEmpty then response.errorCode toString else onError_message, response ) - } catch case e: RuntimeException => - throw EventRuntimeException.ClientFailed(e) + } catch + case e: EventRuntimeException.ActionFailed => throw e + case e: RuntimeException => + throw EventRuntimeException.ClientFailed(e) } }} diff --git a/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/formatting/TelegramParseEscape.scala b/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/formatting/TelegramParseEscape.scala index 4b27795..208646a 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/formatting/TelegramParseEscape.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/formatting/TelegramParseEscape.scala @@ -1,5 +1,11 @@ package cc.sukazyo.cono.morny.util.tgapi.formatting +import org.jsoup.Jsoup +import org.jsoup.nodes.Node + +import scala.collection.mutable +import scala.jdk.CollectionConverters.* + object TelegramParseEscape { def escapeHtml (input: String): String = @@ -9,4 +15,55 @@ object TelegramParseEscape { process = process.replaceAll(">", ">") process + def cleanupHtml (input: String): String = + import org.jsoup.nodes.* + val source = Jsoup.parse(input) + val x = cleanupHtml(source.body.childNodes.asScala.toSeq) + val doc = Document("") + doc.outputSettings + .prettyPrint(false) + x.map(f => doc.appendChild(f)) + x.mkString("") + +// def toHtmlRaw (input: Node): String = +// import org.jsoup.nodes.* +// input match +// case text: TextNode => text.getWholeText +// case _: (DataNode | XmlDeclaration | DocumentType | Comment) => "" +// case elem: Element => elem.childNodes.asScala.map(f => toHtmlRaw(f)).mkString("") + + def cleanupHtml (input: Seq[Node]): List[Node] = + val result = mutable.ListBuffer.empty[Node] + for (i <- input) { + import org.jsoup.nodes.* + def produceChildNodes (curr: Element): Element = + val newOne = Element(curr.tagName) + curr.attributes.forEach(attr => newOne.attr(attr.getKey, attr.getValue)) + for (i <- cleanupHtml(curr.childNodes.asScala.toSeq)) newOne.appendChild(i) + newOne + i match + case text_cdata: CDataNode => result += CDataNode(text_cdata.text) + case text: TextNode => result += TextNode(text.getWholeText) + case _: (DataNode | XmlDeclaration | DocumentType | Comment) => + case elem: Element => elem match + case _: Document => // should not exists here + case _: FormElement => // ignored due to Telegram do not support form + case elem => elem.tagName match + case "a"|"b"|"strong"|"i"|"em"|"u"|"ins"|"s"|"strike"|"del"|"tg-spoiler"|"code"|"pre" => + result += produceChildNodes(elem) + case "br" => + result += TextNode("\n") + case "tg-emoji" => + if elem.attributes.hasKey("emoji-id") then + result += produceChildNodes(elem) + else + result += TextNode(elem.text) + case "img" => + if elem.attributes hasKey "alt" then + result += TextNode(s"[${elem attr "alt"}]") + case _ => + for (i <- cleanupHtml(elem.childNodes.asScala.toSeq)) result += i + } + result.toList + }