FixTweet api implement, with a /tweet command

This commit is contained in:
A.C.Sukazyo Eyre 2023-11-21 23:35:12 +08:00
parent 2687c3be88
commit f8b2d056cc
Signed by: Eyre_S
GPG Key ID: C17CE40291207874
17 changed files with 536 additions and 5 deletions

View File

@ -89,6 +89,9 @@ dependencies {
implementation group: 'com.softwaremill.sttp.client3', name: scala('okhttp-backend'), version: lib_sttp_v
runtimeOnly group: 'com.squareup.okhttp3', name: 'okhttp', version: lib_okhttp_v
implementation group: 'com.google.code.gson', name: 'gson', version: lib_gson_v
implementation group: 'io.circe', name: scala('circe-core'), version: lib_circe_v
implementation group: 'io.circe', name: scala('circe-generic'), version: lib_circe_v
implementation group: 'io.circe', name: scala('circe-parser'), version: lib_circe_v
implementation group: 'com.cronutils', name: 'cron-utils', version: lib_cron_utils_v
// used for disable slf4j
@ -139,6 +142,7 @@ tasks.withType(ScalaCompile).configureEach {
targetCompatibility proj_java.getMajorVersion()
scalaCompileOptions.additionalParameters.add "-language:postfixOps"
scalaCompileOptions.additionalParameters.addAll ("-Xmax-inlines", "256")
scalaCompileOptions.encoding = proj_file_encoding.name()
options.encoding = proj_file_encoding.name()

View File

@ -5,7 +5,7 @@ MORNY_ARCHIVE_NAME = morny-coeur
MORNY_CODE_STORE = https://github.com/Eyre-S/Coeur-Morny-Cono
MORNY_COMMIT_PATH = https://github.com/Eyre-S/Coeur-Morny-Cono/commit/%s
VERSION = 1.3.0-dev7
VERSION = 1.3.0-dev8
USE_DELTA = false
VERSION_DELTA =
@ -26,6 +26,7 @@ lib_javatelegramapi_v = 6.2.0
lib_sttp_v = 3.9.0
lib_okhttp_v = 4.11.0
lib_gson_v = 2.10.1
lib_circe_v = 0.14.6
lib_cron_utils_v = 9.2.0
lib_scalatest_v = 3.2.17

View File

@ -43,11 +43,13 @@ class MornyCommands (using coeur: MornyCoeur) {
$IP186Query.IP,
$IP186Query.Whois,
Encryptor(),
MornyOldJrrp(),
Tweet(),
$MornyManagers.SaveData,
$MornyInformation,
$MornyInformationOlds.Version,
$MornyInformationOlds.Runtime,
MornyOldJrrp(),
$MornyManagers.Exit,
Testing(),

View File

@ -0,0 +1,86 @@
package cc.sukazyo.cono.morny.bot.command
import cc.sukazyo.cono.morny.data.{twitter, TelegramStickers}
import cc.sukazyo.cono.morny.util.tgapi.InputCommand
import cc.sukazyo.cono.morny.MornyCoeur
import cc.sukazyo.cono.morny.data.twitter.{FXApi, TweetUrlInformation}
import cc.sukazyo.cono.morny.util.tgapi.TelegramExtensions.Bot.exec
import cc.sukazyo.cono.morny.Log.{exceptionLog, logger}
import com.pengrad.telegrambot.model.Update
import com.pengrad.telegrambot.model.request.{InputMedia, InputMediaPhoto, InputMediaVideo, ParseMode}
import com.pengrad.telegrambot.request.{SendMediaGroup, SendMessage, SendSticker}
class Tweet (using coeur: MornyCoeur) extends ITelegramCommand {
override val name: String = "tweet"
override val aliases: Array[ICommandAlias] | Null = null
override val paramRule: String = "<tweet-url>"
override val description: String = "获取 Twitter(X) Tweet 内容"
override def execute (using command: InputCommand, event: Update): Unit = {
def do404 (): Unit =
coeur.account exec SendSticker(
event.message.chat.id,
TelegramStickers.ID_404
).replyToMessageId(event.message.messageId())
if command.args.length < 1 then { do404(); return }
twitter.parseTweetUrl(command.args(0)) match
case None => do404()
case Some(TweetUrlInformation(_, _, screenName, statusId, _, _)) =>
try {
val api = FXApi.Fetch.status(Some(screenName), statusId)
import cc.sukazyo.cono.morny.util.tgapi.formatting.TelegramParseEscape.escapeHtml as h
api.tweet match
case None =>
coeur.account exec SendMessage(
event.message.chat.id,
// language=html
s"""❌ Fix-Tweet <code>${api.code}</code>
|<i>${h(api.message)}</i>""".stripMargin
).replyToMessageId(event.message.messageId).parseMode(ParseMode.HTML)
case Some(tweet) =>
val content: String =
// language=html
s"""⚪️ <b>${h(tweet.author.name)} <a href="${tweet.author.url}">@${h(tweet.author.screen_name)}</a></b>
|
|${h(tweet.text)}
|
|<i>💬${tweet.replies} 🔗${tweet.retweets} ${tweet.likes}</i>
|<i><a href="${tweet.url}">${h(tweet.created_at)}</a></i>""".stripMargin
tweet.media match
case None =>
coeur.account exec SendMessage(
event.message.chat.id,
content
).replyToMessageId(event.message.messageId).parseMode(ParseMode.HTML)
case Some(media) =>
val mediaGroup: List[InputMedia[?]] =
(
media.photos match
case None => List.empty
case Some(photos) => for i <- photos yield InputMediaPhoto(i.url)
) ::: (
media.videos match
case None => List.empty
case Some(videos) => for i <- videos yield InputMediaVideo(i.url)
)
mediaGroup.head.caption(content)
mediaGroup.head.parseMode(ParseMode.HTML)
coeur.account exec SendMediaGroup(
event.message.chat.id,
mediaGroup:_*
).replyToMessageId(event.message.messageId)
} catch case e: Exception =>
coeur.account exec SendSticker(
event.message.chat.id,
TelegramStickers.ID_NETWORK_ERR
).replyToMessageId(event.message.messageId)
logger attention
"Error on requesting FixTweet API\n" + exceptionLog(e)
coeur.daemons.reporter.exception(e, "Error on requesting FixTweet API")
}
}

View File

@ -1,5 +1,7 @@
package cc.sukazyo.cono.morny.bot.query
import cc.sukazyo.cono.morny.data.twitter
import cc.sukazyo.cono.morny.data.twitter.TweetUrlInformation
import cc.sukazyo.cono.morny.util.tgapi.formatting.NamingUtils.inlineQueryId
import com.pengrad.telegrambot.model.Update
import com.pengrad.telegrambot.model.request.InlineQueryResultArticle
@ -13,15 +15,14 @@ class ShareToolTwitter extends ITelegramQuery {
private val ID_PREFIX_VX = "[morny/share/twitter/vxtwi]"
private val TITLE_FX = "[tweet] Share as Fix-Tweet"
private val ID_PREFIX_FX = "[morny/share/twitter/fxtwi]"
private val REGEX_TWEET_LINK: Regex = "^(?:https?://)?((?:(?:c\\.)?vx|fx|www\\.)?twitter|(?:www\\.|fixup)?x)\\.com/((\\w+)/status/(\\d+)(?:/photo/(\\d+))?)/?(\\?[\\w&=-]+)?$"r
override def query (event: Update): List[InlineQueryUnit[_]] | Null = {
if (event.inlineQuery.query == null) return null
event.inlineQuery.query match
twitter.parseTweetUrl(event.inlineQuery.query) match
case REGEX_TWEET_LINK(_, _path_data, _, _, _, _) =>
case Some(TweetUrlInformation(_, _path_data, _, _, _, _)) =>
List(
InlineQueryUnit(InlineQueryResultArticle(
inlineQueryId(ID_PREFIX_FX + event.inlineQuery.query), TITLE_FX,

View File

@ -0,0 +1,108 @@
package cc.sukazyo.cono.morny.data.twitter
import cc.sukazyo.cono.morny.util.SttpPublic
import io.circe.{DecodingFailure, ParsingFailure}
/** The struct of FixTweet Status-Fetch-API response.
*
* It may have some issues due to the API reference from FixTweet
* project is very outdated and inaccurate.
*
* @see [[https://github.com/FixTweet/FixTweet/wiki/Status-Fetch-API]]
*
* @param code Status code, normally be [[200]], but can be 401
* or [[404]] or [[500]] due to different reasons.
*
* Related to [[message]]
* @param message Status message.
*
* - When [[code]] is [[200]], it should be `OK`
* - When [[code]] is [[401]], it should be `PRIVATE_TWEET`,
* while in practice, it seems PRIVATE_TWEET will
* just return [[404]].
* - When [[code]] is [[404]], it should be `NOT_FOUND`
* - When [[code]] is [[500]], it should be `API_FILE`
* @param tweet [[FXTweet]] content.
* @since 1.3.0
* @version 2023.11.21
*/
case class FXApi (
code: Int,
message: String,
tweet: Option[FXTweet]
)
object FXApi {
object CirceADTs {
import io.circe.Decoder
import io.circe.generic.semiauto.deriveDecoder
implicit val decoderForAny: Decoder[Any] = _ => Right(None)
implicit val decoder_FXAuthor_website: Decoder[FXAuthor.websiteType] = deriveDecoder
implicit val decoder_FXAuthor: Decoder[FXAuthor] = deriveDecoder
implicit val decoder_FXExternalMedia: Decoder[FXExternalMedia] = deriveDecoder
implicit val decoder_FXMosaicPhoto_formats: Decoder[FXMosaicPhoto.formatsType] = deriveDecoder
implicit val decoder_FXMosaicPhoto: Decoder[FXMosaicPhoto] = deriveDecoder
implicit val decoder_FXPhoto: Decoder[FXPhoto] = deriveDecoder
implicit val decoder_FXVideo: Decoder[FXVideo] = deriveDecoder
implicit val decoder_FXPoolChoice: Decoder[FXPoolChoice] = deriveDecoder
implicit val decoder_FXPool: Decoder[FXPool] = deriveDecoder
implicit val decoder_FXTranslate: Decoder[FXTranslate] = deriveDecoder
implicit val decoder_FXTweet_media: Decoder[FXTweet.mediaType] = deriveDecoder
implicit val decoder_FXTweet: Decoder[FXTweet] = deriveDecoder
implicit val decoder_FXApi: Decoder[FXApi] = deriveDecoder
}
object Fetch {
import io.circe.parser
import CirceADTs.*
import sttp.client3.*
import sttp.client3.okhttp.OkHttpSyncBackend
val uri_base = uri"https://api.fxtwitter.com/"
/** Endpoint URI of [[https://github.com/FixTweet/FixTweet/wiki/Status-Fetch-API FixTweet Status Fetch API]]. */
val uri_status =
(screen_name: Option[String], id: String, translate_to: Option[String]) =>
uri"$uri_base/$screen_name/status/$id/$translate_to"
private val httpClient = OkHttpSyncBackend()
/** Get tweet data from [[uri_status FixTweet Status Fetch API]].
*
* This method uses [[SttpPublic.Headers.UserAgent.MORNY_CURRENT Morny HTTP User-Agent]]
*
* @param screen_name The screen name (@ handle) (aka. user id) of the
* tweet, which is ignored.
* @param id The ID of the status (tweet)
* @param translate_to 2 letter ISO language code of the language you
* want to translate the tweet into.
* @throws SttpClientException When HTTP Request fails due to network
* or else HTTP client related problem.
* @throws ParsingFailure When the response from API is not a regular JSON
* so cannot be parsed. It mostly due to some problem
* or breaking changes from the API serverside.
* @throws DecodingFailure When cannot decode the API response to a [[FXApi]]
* object. It might be some wrong with the [[FXApi]]
* model, or the remote API spec changes.
* @return a [[FXApi]] response object, with [[200]] or any other response code.
*/
@throws[SttpClientException|ParsingFailure|DecodingFailure]
def status (screen_name: Option[String], id: String, translate_to: Option[String] = None): FXApi =
val get = basicRequest
.header(SttpPublic.Headers.UserAgent.MORNY_CURRENT)
.get(uri_status(screen_name, id, translate_to))
.response(asString)
.send(httpClient)
val body = get.body match
case Left(error) => error
case Right(success) => success
parser.parse(body) match
case Left(error) => throw error
case Right(value) => value.as[FXApi] match
case Left(error) => throw error
case Right(value) => value
}
}

View File

@ -0,0 +1,33 @@
package cc.sukazyo.cono.morny.data.twitter
/** Information about the author of a tweet.
*
* @param name Name of the user, set on their profile
* @param screen_name Screen name or @ handle of the user.
* @param avatar_url URL for the user's avatar (profile picture)
* @param avatar_color Palette color corresponding to the user's avatar (profile picture). Value is a hex, including `#`.
* @param banner_url URL for the banner of the user
*/
case class FXAuthor (
name: String,
url: String,
screen_name: String,
avatar_url: Option[String],
avatar_color: Option[String],
banner_url: Option[String],
description: String, // todo
location: String, // todo
website: Option[FXAuthor.websiteType], // todo
followers: Int, // todo
following: Int, // todo
joined: String, // todo
likes: Int, // todo
tweets: Int // todo
)
object FXAuthor {
case class websiteType (
url: String,
display_url: String
)
}

View File

@ -0,0 +1,17 @@
package cc.sukazyo.cono.morny.data.twitter
/** Data for external media, currently only video.
*
* @param `type` Embed type, currently always `video`
* @param url Video URL
* @param height Video height in pixels
* @param width Video width in pixels
* @param duration Video duration in seconds
*/
case class FXExternalMedia (
`type`: String,
url: String,
height: Int,
width: Int,
duration: Int
)

View File

@ -0,0 +1,24 @@
package cc.sukazyo.cono.morny.data.twitter
import cc.sukazyo.cono.morny.data.twitter.FXMosaicPhoto.formatsType
/** Data for the mosaic service, which stitches photos together
*
* @param `type` This can help compare items in a pool of media
* @param formats Pool of formats, only `jpeg` and `webp` are returned currently
*/
case class FXMosaicPhoto (
`type`: "mosaic_photo",
formats: formatsType
)
object FXMosaicPhoto {
/** Pool of formats, only `jpeg` and `webp` are returned currently.
* @param webp URL for webp resource
* @param jpeg URL for jpeg resource
*/
case class formatsType (
webp: String,
jpeg: String
)
}

View File

@ -0,0 +1,16 @@
package cc.sukazyo.cono.morny.data.twitter
/** This can help compare items in a pool of media
*
* @param `type` This can help compare items in a pool of media
* @param url URL of the photo
* @param width Width of the photo, in pixels
* @param height Height of the photo, in pixels
*/
case class FXPhoto (
`type`: "photo",
url: String,
width: Int,
height: Int,
altText: String // todo
)

View File

@ -0,0 +1,15 @@
package cc.sukazyo.cono.morny.data.twitter
/** Data for a poll on a given Tweet.
*
* @param choices Array of the poll choices
* @param total_votes Total votes in poll
* @param ends_at Date of which the poll ends
* @param time_left_en Time remaining counter in English (i.e. **9 hours left**)
*/
case class FXPool (
choices: List[FXPoolChoice],
total_votes: Int,
ends_at: String,
time_left_en: String
)

View File

@ -0,0 +1,13 @@
package cc.sukazyo.cono.morny.data.twitter
/** Data for a single choice in a poll
*
* @param label What this choice in the poll is called
* @param count How many people voted in this poll
* @param percentage Percentage of total people who voted for this option (0 - 100, rounded to nearest tenth)
*/
case class FXPoolChoice (
label: String,
count: Int,
percentage: Int
)

View File

@ -0,0 +1,13 @@
package cc.sukazyo.cono.morny.data.twitter
/** Information about a requested translation for a Tweet, when asked.
*
* @param text Translated Tweet text
* @param source_lang 2-letter ISO language code of source language
* @param target_lang 2-letter ISO language code of target language
*/
case class FXTranslate (
text: String,
source_lang: String,
target_lang: String
)

View File

@ -0,0 +1,90 @@
package cc.sukazyo.cono.morny.data.twitter
import cc.sukazyo.cono.morny.data.twitter.FXTweet.mediaType
import cc.sukazyo.cono.morny.util.EpochDateTime.EpochSeconds
/** The container of all the information for a Tweet.
*
* @param id Status (Tweet) ID
* @param url Link to original Tweet
* @param text Text of Tweet
* @param created_at Date/Time in UTC when the Tweet was created
* @param created_timestamp Date/Time in UTC when the Tweet was created
* @param color Dominant color pulled from either Tweet media or from the author's profile picture.
* @param lang Language that Twitter detects a Tweet is. May be null is unknown.
* @param replying_to Screen name of person being replied to, or null
* @param replying_to_status Tweet ID snowflake being replied to, or null
* @param twitter_card Corresponds to proper embed container for Tweet, which is used by
* FixTweet for our official embeds.<br>
* Notice that this should be of type [[]]
* but due to circe parser does not support it well so alternative
* [[String]] type is used.
* @param author Author of the tweet
* @param source Tweet source (i.e. Twitter for iPhone)
* @param likes Like count
* @param retweets Retweet count
* @param replies Reply count
* @param views View count, returns null if view count is not available (i.e. older Tweets)
* @param quote Nested Tweet corresponding to the tweet which this tweet is quoting, if applicable
* @param pool Poll attached to Tweet
* @param translation Translation results, only provided if explicitly asked
* @param media Containing object containing references to photos, videos, or external media
*/
case class FXTweet (
///====================
/// Core
///====================
id: String,
url: String,
text: String,
created_at: String,
created_timestamp: EpochSeconds,
is_note_tweet: Boolean, // todo
possibly_sensitive: Option[Boolean], // todo
color: Option[String],
lang: Option[String],
replying_to: Option[String],
replying_to_status: Option[String],
// twitter_card: "tweet"|"summary"|"summary_large_image"|"player",
twitter_card: String,
author: FXAuthor,
source: String,
///====================
/// Interaction counts
///====================
likes: Int,
retweets: Int,
replies: Int,
views: Option[Int],
///====================
/// Embeds
///====================
quote: Option[FXTweet],
pool: Option[FXPool],
translation: Option[FXTranslate],
media: Option[mediaType]
)
object FXTweet {
/** Containing object containing references to photos, videos, or external media.
*
* @param external Refers to external media, such as YouTube embeds
* @param photos An Array of photos from a Tweet
* @param videos An Array of videos from a Tweet
* @param mosaic Corresponding Mosaic information for a Tweet
*/
case class mediaType (
all: Option[List[Any]], // todo
external: Option[FXExternalMedia],
photos: Option[List[FXPhoto]],
videos: Option[List[FXVideo]],
mosaic: Option[FXMosaicPhoto]
)
}

View File

@ -0,0 +1,21 @@
package cc.sukazyo.cono.morny.data.twitter
/** Data for a Tweet's video
*
* @param `type` Returns video if video, or gif if gif. Note that on Twitter, all GIFs are MP4s.
* @param url URL corresponding to the video file
* @param thumbnail_url URL corresponding to the thumbnail for the video
* @param width Width of the video, in pixels
* @param height Height of the video, in pixels
* @param format Video format, usually `video/mp4`
*/
case class FXVideo (
// `type`: "video"|"gif",
`type`: String,
url: String,
thumbnail_url: String,
width: Int,
height: Int,
duration: Float, // todo
format: String
)

View File

@ -0,0 +1,72 @@
package cc.sukazyo.cono.morny.data
import scala.util.matching.Regex
package object twitter {
private val REGEX_TWEET_URL: Regex = "^(?:https?://)?((?:(?:(?:c\\.)?vx|fx|www\\.)?twitter|(?:www\\.|fixup)?x)\\.com)/((\\w+)/status/(\\d+)(?:/photo/(\\d+))?)/?(\\?[\\w&=-]+)?$"r
/** Messages that can contains on a tweet url.
*
* A tweet url is like `https://twitter.com/pj_sekai/status/1726526899982352557?s=20`
* which can be found in address bar of tweet page or tweet's share link.
*
* @param domain Domain of the tweet url. Normally `twitter.com` or `x.com`
* (can be with `www.` or without). But [[parseTweetUrl]] also
* supports to parse some third-party tweet share url domain
* includes `fx.twitter.com`, `vxtwitter.com`(or `c.vxtwitter.com`
* though it have been deprecated), or `fixupx.com`.
* @param statusPath Full path of the status. It should be like
* `$screenName/status/$statusId`, with or without photo param
* like `/photo/$subPhotoId`. It does not contains tracking
* or any else params.
* @param screenName Screen name of the tweet author, aka. author's user id.
* For most case this section is useless in processing at
* the backend (because [[statusId]] along is accurate enough)
* so it may not be right, but it should always exists.
* @param statusId Unique ID of the status. It is unique in whole Twitter globe.
* Should be a number.
* @param subPhotoId photo id or serial number in the status. Unique in the status
* globe, only exists when specific a photo in the status. It should
* be a number of 0~3 (because twitter supports 4 image at most in
* one tweet).
* @param trackingParam All of encoded url params. Normally no data here is something
* important.
*/
case class TweetUrlInformation (
domain: String,
statusPath: String,
screenName: String,
statusId: String,
subPhotoId: Option[String],
trackingParam: Option[String]
)
/** Parse a url to [[TweetUrlInformation]] for future processing.
*
* Supports following url:
*
* - `twitter.com` or `www.twitter.com`
* - `x.com` or `www.x.com`
* - `fxtwitter.com` or `fixupx.com`
* - `vxtwitter.com` or `c.vxtwitter.com`
* - should be the path of `/:screenName/status/:id`
* - can contains `./photo/:photoId`
* - url param non-sensitive
* - http or https non-sensitive
*
* @param url a supported tweet URL or not.
* @return [[Option]] with [[TweetUrlInformation]] if the input url is a supported
* tweet url, or [[None]] if it's not.
*/
def parseTweetUrl (url: String): Option[TweetUrlInformation] =
url match
case REGEX_TWEET_URL(_1, _2, _3, _4, _5, _6) =>
Some(TweetUrlInformation(
_1, _2, _3, _4,
Option(_5),
Option(_6)
))
case _ => None
}

View File

@ -1,5 +1,8 @@
package cc.sukazyo.cono.morny.util
import cc.sukazyo.cono.morny.MornySystem
import sttp.model.Header
object SttpPublic {
object Schemes {
@ -7,4 +10,16 @@ object SttpPublic {
val HTTPS = "https"
}
object Headers {
object UserAgent {
private val key = "User-Agent"
val MORNY_CURRENT = Header(key, s"MornyCoeur / ${MornySystem.VERSION}")
}
}
}