Eyre_S/Coeur-Morny-Cono
1
0
Fork 0
mirror of https://github.com/Eyre-S/Coeur-Morny-Cono.git synced 2024-11-22 11:14:55 +08:00
Code Issues Actions Packages Releases Wiki Activity

set morny UA for all HTTP req, add twitter tests

Browse Source
This commit is contained in:
A.C.Sukazyo Eyre 2023-11-23 17:57:29 +08:00
parent 43cdf221d9
commit d602e1b366
Signed by: Eyre_S
GPG Key ID: C17CE40291207874
11 changed files with 231 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
gradle.properties
View File

@ -5,7 +5,7 @@ MORNY_ARCHIVE_NAME = morny-coeur
MORNY_CODE_STORE = https://github.com/Eyre-S/Coeur-Morny-Cono
MORNY_COMMIT_PATH = https://github.com/Eyre-S/Coeur-Morny-Cono/commit/%s
VERSION = 1.3.0-dev9
VERSION = 1.3.0-dev10
USE_DELTA = false
VERSION_DELTA =

7
src/main/scala/cc/sukazyo/cono/morny/data/BilibiliForms.scala
View File

@ -1,9 +1,9 @@
package cc.sukazyo.cono.morny.data
import cc.sukazyo.cono.morny.util.BiliTool
import cc.sukazyo.cono.morny.util.SttpPublic.Schemes
import cc.sukazyo.cono.morny.util.SttpPublic.{mornyBasicRequest, Schemes}
import cc.sukazyo.cono.morny.util.UseSelect.select
import sttp.client3.{basicRequest, ignore, HttpError, SttpClientException}
import sttp.client3.{HttpError, SttpClientException}
import sttp.client3.okhttp.OkHttpSyncBackend
import sttp.model.Uri
@ -77,7 +77,8 @@ object BilibiliForms {
throw IllegalArgumentException(s"is a b23 video link: $uri . (use parse_videoUrl instead)")
try {
val response = basicRequest
import sttp.client3.ignore
val response = mornyBasicRequest
.get(uri)
.followRedirects(false)
.response(ignore)

5
src/main/scala/cc/sukazyo/cono/morny/data/NbnhhshQuery.scala
View File

@ -1,7 +1,8 @@
package cc.sukazyo.cono.morny.data
import cc.sukazyo.cono.morny.util.SttpPublic.mornyBasicRequest
import com.google.gson.Gson
import sttp.client3.{asString, basicRequest, HttpError, SttpClientException, UriContext}
import sttp.client3.{asString, HttpError, SttpClientException, UriContext}
import sttp.client3.okhttp.OkHttpSyncBackend
import sttp.model.MediaType
@ -22,7 +23,7 @@ object NbnhhshQuery {
@throws[HttpError[_]|SttpClientException]
def sendGuess (text: String): GuessResult = {
case class GuessRequest (text: String)
val http = basicRequest
val http = mornyBasicRequest
.body(Gson().toJson(GuessRequest(text))).contentType(MediaType.ApplicationJson)
.post(API_GUESS_METHOD)
.response(asString.getRight)

6
src/main/scala/cc/sukazyo/cono/morny/data/ip186/IP186QueryHandler.scala
View File

@ -1,7 +1,7 @@
package cc.sukazyo.cono.morny.data.ip186
import cc.sukazyo.cono.morny.util.SttpPublic.Schemes
import sttp.client3.{asString, basicRequest, HttpError, SttpClientException, UriContext}
import cc.sukazyo.cono.morny.util.SttpPublic.{mornyBasicRequest, Schemes}
import sttp.client3.{asString, HttpError, SttpClientException, UriContext}
import sttp.client3.okhttp.OkHttpSyncBackend
import sttp.model.Uri
@ -36,7 +36,7 @@ object IP186QueryHandler {
val uri = requestPath.scheme(Schemes.HTTPS).host(SITE_HOST)
IP186Response(
uri.toString,
basicRequest
mornyBasicRequest
.get(uri)
.response(asString.getRight)
.send(httpClient)

3
src/main/scala/cc/sukazyo/cono/morny/data/twitter/FXApi.scala
View File

@ -1,6 +1,7 @@
package cc.sukazyo.cono.morny.data.twitter
import cc.sukazyo.cono.morny.util.SttpPublic
import cc.sukazyo.cono.morny.util.SttpPublic.mornyBasicRequest
import io.circe.{DecodingFailure, ParsingFailure}
/** The struct of FixTweet Status-Fetch-API response.
@ -89,7 +90,7 @@ object FXApi {
*/
@throws[SttpClientException|ParsingFailure|DecodingFailure]
def status (screen_name: Option[String], id: String, translate_to: Option[String] = None): FXApi =
val get = basicRequest
val get = mornyBasicRequest
.header(SttpPublic.Headers.UserAgent.MORNY_CURRENT)
.get(uri_status(screen_name, id, translate_to))
.response(asString)

4
src/main/scala/cc/sukazyo/cono/morny/data/twitter/FXTweet.scala
View File

@ -16,7 +16,7 @@ import cc.sukazyo.cono.morny.util.EpochDateTime.EpochSeconds
* @param replying_to_status Tweet ID snowflake being replied to, or null
* @param twitter_card Corresponds to proper embed container for Tweet, which is used by
* FixTweet for our official embeds.<br>
* Notice that this should be of type [[]]
* Notice that this should be of type [["tweet"|"summary"|"summary_large_image"|"player"]]
* but due to circe parser does not support it well so alternative
* [[String]] type is used.
* @param author Author of the tweet
@ -48,7 +48,7 @@ case class FXTweet (
replying_to: Option[String],
replying_to_status: Option[String],
// twitter_card: "tweet"|"summary"|"summary_large_image"|"player",
twitter_card: String,
twitter_card: Option[String],
author: FXAuthor,
source: String,

4
src/main/scala/cc/sukazyo/cono/morny/data/twitter/package.scala
View File

@ -4,7 +4,7 @@ import scala.util.matching.Regex
package object twitter {
private val REGEX_TWEET_URL: Regex = "^(?:https?://)?((?:(?:(?:c\\.)?vx|fx|www\\.)?twitter|(?:www\\.|fixup)?x)\\.com)/((\\w+)/status/(\\d+)(?:/photo/(\\d+))?)/?(\\?[\\w&=-]+)?$"r
private val REGEX_TWEET_URL: Regex = "^(?:https?://)?((?:(?:(?:c\\.)?vx|fx|www\\.)?twitter|(?:www\\.|fixup|fixv)?x)\\.com)/((\\w+)/status/(\\d+)(?:/photo/(\\d+))?)/?(?:\\?([\\w&=-]+))?$"r
/** Messages that can contains on a tweet url.
*
@ -49,7 +49,7 @@ package object twitter {
* - `twitter.com` or `www.twitter.com`
* - `x.com` or `www.x.com`
* - `fxtwitter.com` or `fixupx.com`
* - `vxtwitter.com` or `c.vxtwitter.com`
* - `vxtwitter.com` or `c.vxtwitter.com` or `fixvx.com`
* - should be the path of `/:screenName/status/:id`
* - can contains `./photo/:photoId`
* - url param non-sensitive

7
src/main/scala/cc/sukazyo/cono/morny/util/SttpPublic.scala
View File

@ -1,6 +1,7 @@
package cc.sukazyo.cono.morny.util
import cc.sukazyo.cono.morny.MornySystem
import sttp.client3.basicRequest
import sttp.model.Header
object SttpPublic {
@ -16,10 +17,14 @@ object SttpPublic {
private val key = "User-Agent"
val MORNY_CURRENT = Header(key, s"MornyCoeur / ${MornySystem.VERSION}")
val MORNY_CURRENT: Header = Header(key, s"MornyCoeur / ${MornySystem.VERSION}")
}
}
val mornyBasicRequest =
basicRequest
.header(Headers.UserAgent.MORNY_CURRENT, true)
}

5
src/main/scala/cc/sukazyo/cono/morny/util/tgapi/formatting/TelegramUserInformation.scala
View File

@ -1,7 +1,8 @@
package cc.sukazyo.cono.morny.util.tgapi.formatting
import cc.sukazyo.cono.morny.util.SttpPublic.mornyBasicRequest
import com.pengrad.telegrambot.model.User
import sttp.client3.{asString, basicRequest, HttpError, SttpClientException, UriContext}
import sttp.client3.{asString, HttpError, SttpClientException, UriContext}
import sttp.client3.okhttp.OkHttpSyncBackend
import java.io.IOException
@ -17,7 +18,7 @@ object TelegramUserInformation {
def getDataCenterFromUser (username: String): String = {
try
val body = basicRequest
val body = mornyBasicRequest
.get(uri"https://t.me/$username")
.response(asString.getRight)
.send(httpClient)

82
src/test/scala/cc/sukazyo/cono/morny/test/data/twitter/FXApiTest.scala Normal file
View File

@ -0,0 +1,82 @@
package cc.sukazyo.cono.morny.test.data.twitter
import cc.sukazyo.cono.morny.data.twitter.FXApi
import cc.sukazyo.cono.morny.data.twitter.FXApi.Fetch
import cc.sukazyo.cono.morny.test.MornyTests
import org.scalatest.prop.TableDrivenPropertyChecks
import org.scalatest.tagobjects.{Network, Slow}
//noinspection ScalaUnusedExpression
class FXApiTest extends MornyTests with TableDrivenPropertyChecks {
"while fetch status (tweet) :" - {
"non exists tweet id should return 404" taggedAs (Slow, Network) in {
val api = Fetch.status(Some("some_non_exists"), "-1")
api.code shouldEqual 404
api.message shouldEqual "NOT_FOUND"
api.tweet shouldBe empty
}
/** It should return 401, but in practice it seems will only
* return 404.
*/
"private tweet should return 410 or 404" taggedAs (Slow, Network) in {
val api = Fetch.status(Some("_takiChan"), "1586671758999924736")
api.code should (equal (404) or equal (401))
api.code match
case 401 =>
api.message shouldEqual "PRIVATE_TWEET"
note("from private tweet got 401 PRIVATE_TWEET")
case 404 =>
api.message shouldEqual "NOT_FOUND"
note("from private tweet got 404 NOT_FOUND")
api.tweet shouldBe empty
}
val examples = Table[(Option[String], String), FXApi =>Unit](
("id", "checking"),
((Some("_Eyre_S"), "1669362743332438019"), api => {
api.tweet shouldBe defined
api.tweet.get.text shouldEqual "猫头猫头鹰头猫头鹰头猫头鹰"
api.tweet.get.quote shouldBe defined
api.tweet.get.quote.get.id shouldEqual "1669302279386828800"
}),
((None, "1669362743332438019"), api => {
api.tweet shouldBe defined
api.tweet.get.text shouldEqual "猫头猫头鹰头猫头鹰头猫头鹰"
api.tweet.get.quote shouldBe defined
api.tweet.get.quote.get.id shouldEqual "1669302279386828800"
}),
((None, "1654080016802807809"), api => {
api.tweet shouldBe defined
api.tweet.get.media shouldBe defined
api.tweet.get.media.get.videos shouldBe empty
api.tweet.get.media.get.photos shouldBe defined
api.tweet.get.media.get.photos.get.length shouldBe 1
api.tweet.get.media.get.photos.get.head.width shouldBe 2048
api.tweet.get.media.get.photos.get.head.height shouldBe 1536
api.tweet.get.media.get.mosaic shouldBe empty
}),
((None, "1538536152093044736"), api => {
api.tweet shouldBe defined
api.tweet.get.media shouldBe defined
api.tweet.get.media.get.videos shouldBe empty
api.tweet.get.media.get.photos shouldBe defined
api.tweet.get.media.get.photos.get.length shouldBe 2
api.tweet.get.media.get.photos.get.head.width shouldBe 2894
api.tweet.get.media.get.photos.get.head.height shouldBe 4093
api.tweet.get.media.get.photos.get(1).width shouldBe 2894
api.tweet.get.media.get.photos.get(1).height shouldBe 4093
api.tweet.get.media.get.mosaic shouldBe defined
})
)
forAll(examples) { (data, assertion) =>
s"tweet $data should be fetched successful" taggedAs (Slow, Network) in {
assertion(Fetch.status(data._1, data._2))
}
}
}
}

123
src/test/scala/cc/sukazyo/cono/morny/test/data/twitter/PackageTest.scala Normal file
View File

@ -0,0 +1,123 @@
package cc.sukazyo.cono.morny.test.data.twitter
import cc.sukazyo.cono.morny.data.twitter.{parseTweetUrl, TweetUrlInformation}
import cc.sukazyo.cono.morny.test.MornyTests
class PackageTest extends MornyTests {
"while parsing tweet url :" - {
"normal twitter tweet share url should be parsed" in {
parseTweetUrl("https://twitter.com/ps_urine/status/1727614825755505032?s=20")
.shouldEqual(Some(TweetUrlInformation(
"twitter.com", "ps_urine/status/1727614825755505032",
"ps_urine", "1727614825755505032",
None, Some("s=20")
)))
}
"normal X.com tweet share url should be parsed" in {
parseTweetUrl("https://x.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(defined)
}
"X.com or twitter tweet share url should not www.sensitive" in {
parseTweetUrl("https://www.twitter.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(defined)
parseTweetUrl("https://www.x.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(defined)
}
"fxtwitter and fixupx url should be parsed" in {
parseTweetUrl("https://fxtwitter.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(defined)
parseTweetUrl("https://fixupx.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(defined)
}
"vxtwitter should be parsed and can be with c." in {
parseTweetUrl("https://vxtwitter.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(defined)
parseTweetUrl("https://c.vxtwitter.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(defined)
}
"fixvx should be parsed and cannot be with c." in {
parseTweetUrl("https://fixvx.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(defined)
parseTweetUrl("https://c.fixvx.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(empty)
}
"fxtwitter and vxtwitter should not contains www." in {
parseTweetUrl("https://www.fxtwitter.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(empty)
parseTweetUrl("https://www.fixupx.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(empty)
parseTweetUrl("https://www.vxtwitter.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(empty)
parseTweetUrl("https://www.fixvx.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(empty)
}
"url should be http/s non-sensitive" in {
parseTweetUrl("twitter.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(defined)
parseTweetUrl("http://x.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(defined)
parseTweetUrl("http://fxtwitter.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(defined)
parseTweetUrl("http://fixupx.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(defined)
parseTweetUrl("vxtwitter.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(defined)
parseTweetUrl("fixvx.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(defined)
}
"url param should be non-sensitive" in {
parseTweetUrl("twitter.com/ps_urine/status/1727614825755505032")
.shouldBe(defined)
parseTweetUrl("http://x.com/ps_urine/status/1727614825755505032/?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a")
.shouldBe(defined)
parseTweetUrl("http://fxtwitter.com/ps_urine/status/1727614825755505032/?s=20")
.shouldBe(defined)
parseTweetUrl("http://fixupx.com/ps_urine/status/1727614825755505032?s=20")
.shouldBe(defined)
parseTweetUrl("vxtwitter.com/ps_urine/status/1727614825755505032")
.shouldBe(defined)
parseTweetUrl("fixvx.com/ps_urine/status/1727614825755505032?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a")
.shouldBe(defined)
}
"screen name should not be non-exists" in {
parseTweetUrl("twitter.com/status/1727614825755505032")
.shouldBe(empty)
parseTweetUrl("http://x.com/status/1727614825755505032/?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a")
.shouldBe(empty)
parseTweetUrl("http://fxtwitter.com/status/1727614825755505032/?s=20")
.shouldBe(empty)
parseTweetUrl("http://fixupx.com/status/1727614825755505032?s=20")
.shouldBe(empty)
parseTweetUrl("vxtwitter.com/status/1727614825755505032")
.shouldBe(empty)
parseTweetUrl("fixvx.com/status/1727614825755505032?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a")
.shouldBe(empty)
}
"url with photo id should be parsed" in {
parseTweetUrl("twitter.com/ps_urine/status/1727614825755505032/photo/2")
.should(matchPattern { case Some(TweetUrlInformation(_, _, _, _, Some("2"), _)) => })
parseTweetUrl("http://x.com/ps_urine/status/1727614825755505032/photo/1/?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a")
.should(matchPattern { case Some(TweetUrlInformation(_, _, _, _, Some("1"), _)) => })
parseTweetUrl("http://fxtwitter.com/ps_urine/status/1727614825755505032/photo/4/?s=20")
.should(matchPattern { case Some(TweetUrlInformation(_, _, _, _, Some("4"), _)) => })
parseTweetUrl("http://fixupx.com/ps_urine/status/1727614825755505032/photo/7?s=20")
.should(matchPattern { case Some(TweetUrlInformation(_, _, _, _, Some("7"), _)) => })
parseTweetUrl("vxtwitter.com/ps_urine/status/1727614825755505032/photo/114514")
.should(matchPattern { case Some(TweetUrlInformation(_, _, _, _, Some("114514"), _)) => })
parseTweetUrl("fixvx.com/ps_urine/status/1727614825755505032/photo/unavailable-id?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a")
.shouldBe(empty)
}
}
}