From d602e1b366af3213c8b48ce5c2b82d6a1baf0a56 Mon Sep 17 00:00:00 2001 From: Eyre_S Date: Thu, 23 Nov 2023 17:57:29 +0800 Subject: [PATCH] set morny UA for all HTTP req, add twitter tests --- gradle.properties | 2 +- .../cono/morny/data/BilibiliForms.scala | 7 +- .../cono/morny/data/NbnhhshQuery.scala | 5 +- .../morny/data/ip186/IP186QueryHandler.scala | 6 +- .../cono/morny/data/twitter/FXApi.scala | 3 +- .../cono/morny/data/twitter/FXTweet.scala | 4 +- .../cono/morny/data/twitter/package.scala | 4 +- .../sukazyo/cono/morny/util/SttpPublic.scala | 7 +- .../formatting/TelegramUserInformation.scala | 5 +- .../morny/test/data/twitter/FXApiTest.scala | 82 ++++++++++++ .../morny/test/data/twitter/PackageTest.scala | 123 ++++++++++++++++++ 11 files changed, 231 insertions(+), 17 deletions(-) create mode 100644 src/test/scala/cc/sukazyo/cono/morny/test/data/twitter/FXApiTest.scala create mode 100644 src/test/scala/cc/sukazyo/cono/morny/test/data/twitter/PackageTest.scala diff --git a/gradle.properties b/gradle.properties index 1e4944c..aab8e75 100644 --- a/gradle.properties +++ b/gradle.properties @@ -5,7 +5,7 @@ MORNY_ARCHIVE_NAME = morny-coeur MORNY_CODE_STORE = https://github.com/Eyre-S/Coeur-Morny-Cono MORNY_COMMIT_PATH = https://github.com/Eyre-S/Coeur-Morny-Cono/commit/%s -VERSION = 1.3.0-dev9 +VERSION = 1.3.0-dev10 USE_DELTA = false VERSION_DELTA = diff --git a/src/main/scala/cc/sukazyo/cono/morny/data/BilibiliForms.scala b/src/main/scala/cc/sukazyo/cono/morny/data/BilibiliForms.scala index c2bee8e..d018119 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/data/BilibiliForms.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/data/BilibiliForms.scala @@ -1,9 +1,9 @@ package cc.sukazyo.cono.morny.data import cc.sukazyo.cono.morny.util.BiliTool -import cc.sukazyo.cono.morny.util.SttpPublic.Schemes +import cc.sukazyo.cono.morny.util.SttpPublic.{mornyBasicRequest, Schemes} import cc.sukazyo.cono.morny.util.UseSelect.select -import sttp.client3.{basicRequest, ignore, HttpError, SttpClientException} +import sttp.client3.{HttpError, SttpClientException} import sttp.client3.okhttp.OkHttpSyncBackend import sttp.model.Uri @@ -77,7 +77,8 @@ object BilibiliForms { throw IllegalArgumentException(s"is a b23 video link: $uri . (use parse_videoUrl instead)") try { - val response = basicRequest + import sttp.client3.ignore + val response = mornyBasicRequest .get(uri) .followRedirects(false) .response(ignore) diff --git a/src/main/scala/cc/sukazyo/cono/morny/data/NbnhhshQuery.scala b/src/main/scala/cc/sukazyo/cono/morny/data/NbnhhshQuery.scala index 4322739..1fd00fb 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/data/NbnhhshQuery.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/data/NbnhhshQuery.scala @@ -1,7 +1,8 @@ package cc.sukazyo.cono.morny.data +import cc.sukazyo.cono.morny.util.SttpPublic.mornyBasicRequest import com.google.gson.Gson -import sttp.client3.{asString, basicRequest, HttpError, SttpClientException, UriContext} +import sttp.client3.{asString, HttpError, SttpClientException, UriContext} import sttp.client3.okhttp.OkHttpSyncBackend import sttp.model.MediaType @@ -22,7 +23,7 @@ object NbnhhshQuery { @throws[HttpError[_]|SttpClientException] def sendGuess (text: String): GuessResult = { case class GuessRequest (text: String) - val http = basicRequest + val http = mornyBasicRequest .body(Gson().toJson(GuessRequest(text))).contentType(MediaType.ApplicationJson) .post(API_GUESS_METHOD) .response(asString.getRight) diff --git a/src/main/scala/cc/sukazyo/cono/morny/data/ip186/IP186QueryHandler.scala b/src/main/scala/cc/sukazyo/cono/morny/data/ip186/IP186QueryHandler.scala index 178e21d..3bc351c 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/data/ip186/IP186QueryHandler.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/data/ip186/IP186QueryHandler.scala @@ -1,7 +1,7 @@ package cc.sukazyo.cono.morny.data.ip186 -import cc.sukazyo.cono.morny.util.SttpPublic.Schemes -import sttp.client3.{asString, basicRequest, HttpError, SttpClientException, UriContext} +import cc.sukazyo.cono.morny.util.SttpPublic.{mornyBasicRequest, Schemes} +import sttp.client3.{asString, HttpError, SttpClientException, UriContext} import sttp.client3.okhttp.OkHttpSyncBackend import sttp.model.Uri @@ -36,7 +36,7 @@ object IP186QueryHandler { val uri = requestPath.scheme(Schemes.HTTPS).host(SITE_HOST) IP186Response( uri.toString, - basicRequest + mornyBasicRequest .get(uri) .response(asString.getRight) .send(httpClient) diff --git a/src/main/scala/cc/sukazyo/cono/morny/data/twitter/FXApi.scala b/src/main/scala/cc/sukazyo/cono/morny/data/twitter/FXApi.scala index 8e48aec..828aba1 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/data/twitter/FXApi.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/data/twitter/FXApi.scala @@ -1,6 +1,7 @@ package cc.sukazyo.cono.morny.data.twitter import cc.sukazyo.cono.morny.util.SttpPublic +import cc.sukazyo.cono.morny.util.SttpPublic.mornyBasicRequest import io.circe.{DecodingFailure, ParsingFailure} /** The struct of FixTweet Status-Fetch-API response. @@ -89,7 +90,7 @@ object FXApi { */ @throws[SttpClientException|ParsingFailure|DecodingFailure] def status (screen_name: Option[String], id: String, translate_to: Option[String] = None): FXApi = - val get = basicRequest + val get = mornyBasicRequest .header(SttpPublic.Headers.UserAgent.MORNY_CURRENT) .get(uri_status(screen_name, id, translate_to)) .response(asString) diff --git a/src/main/scala/cc/sukazyo/cono/morny/data/twitter/FXTweet.scala b/src/main/scala/cc/sukazyo/cono/morny/data/twitter/FXTweet.scala index 62f7417..2bc75e3 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/data/twitter/FXTweet.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/data/twitter/FXTweet.scala @@ -16,7 +16,7 @@ import cc.sukazyo.cono.morny.util.EpochDateTime.EpochSeconds * @param replying_to_status Tweet ID snowflake being replied to, or null * @param twitter_card Corresponds to proper embed container for Tweet, which is used by * FixTweet for our official embeds.
- * Notice that this should be of type [[]] + * Notice that this should be of type [["tweet"|"summary"|"summary_large_image"|"player"]] * but due to circe parser does not support it well so alternative * [[String]] type is used. * @param author Author of the tweet @@ -48,7 +48,7 @@ case class FXTweet ( replying_to: Option[String], replying_to_status: Option[String], // twitter_card: "tweet"|"summary"|"summary_large_image"|"player", - twitter_card: String, + twitter_card: Option[String], author: FXAuthor, source: String, diff --git a/src/main/scala/cc/sukazyo/cono/morny/data/twitter/package.scala b/src/main/scala/cc/sukazyo/cono/morny/data/twitter/package.scala index ee58a65..73e6259 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/data/twitter/package.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/data/twitter/package.scala @@ -4,7 +4,7 @@ import scala.util.matching.Regex package object twitter { - private val REGEX_TWEET_URL: Regex = "^(?:https?://)?((?:(?:(?:c\\.)?vx|fx|www\\.)?twitter|(?:www\\.|fixup)?x)\\.com)/((\\w+)/status/(\\d+)(?:/photo/(\\d+))?)/?(\\?[\\w&=-]+)?$"r + private val REGEX_TWEET_URL: Regex = "^(?:https?://)?((?:(?:(?:c\\.)?vx|fx|www\\.)?twitter|(?:www\\.|fixup|fixv)?x)\\.com)/((\\w+)/status/(\\d+)(?:/photo/(\\d+))?)/?(?:\\?([\\w&=-]+))?$"r /** Messages that can contains on a tweet url. * @@ -49,7 +49,7 @@ package object twitter { * - `twitter.com` or `www.twitter.com` * - `x.com` or `www.x.com` * - `fxtwitter.com` or `fixupx.com` - * - `vxtwitter.com` or `c.vxtwitter.com` + * - `vxtwitter.com` or `c.vxtwitter.com` or `fixvx.com` * - should be the path of `/:screenName/status/:id` * - can contains `./photo/:photoId` * - url param non-sensitive diff --git a/src/main/scala/cc/sukazyo/cono/morny/util/SttpPublic.scala b/src/main/scala/cc/sukazyo/cono/morny/util/SttpPublic.scala index 230e5a0..1f338b9 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/util/SttpPublic.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/util/SttpPublic.scala @@ -1,6 +1,7 @@ package cc.sukazyo.cono.morny.util import cc.sukazyo.cono.morny.MornySystem +import sttp.client3.basicRequest import sttp.model.Header object SttpPublic { @@ -16,10 +17,14 @@ object SttpPublic { private val key = "User-Agent" - val MORNY_CURRENT = Header(key, s"MornyCoeur / ${MornySystem.VERSION}") + val MORNY_CURRENT: Header = Header(key, s"MornyCoeur / ${MornySystem.VERSION}") } } + val mornyBasicRequest = + basicRequest + .header(Headers.UserAgent.MORNY_CURRENT, true) + } diff --git a/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/formatting/TelegramUserInformation.scala b/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/formatting/TelegramUserInformation.scala index c02c84a..cd8a896 100644 --- a/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/formatting/TelegramUserInformation.scala +++ b/src/main/scala/cc/sukazyo/cono/morny/util/tgapi/formatting/TelegramUserInformation.scala @@ -1,7 +1,8 @@ package cc.sukazyo.cono.morny.util.tgapi.formatting +import cc.sukazyo.cono.morny.util.SttpPublic.mornyBasicRequest import com.pengrad.telegrambot.model.User -import sttp.client3.{asString, basicRequest, HttpError, SttpClientException, UriContext} +import sttp.client3.{asString, HttpError, SttpClientException, UriContext} import sttp.client3.okhttp.OkHttpSyncBackend import java.io.IOException @@ -17,7 +18,7 @@ object TelegramUserInformation { def getDataCenterFromUser (username: String): String = { try - val body = basicRequest + val body = mornyBasicRequest .get(uri"https://t.me/$username") .response(asString.getRight) .send(httpClient) diff --git a/src/test/scala/cc/sukazyo/cono/morny/test/data/twitter/FXApiTest.scala b/src/test/scala/cc/sukazyo/cono/morny/test/data/twitter/FXApiTest.scala new file mode 100644 index 0000000..b687f86 --- /dev/null +++ b/src/test/scala/cc/sukazyo/cono/morny/test/data/twitter/FXApiTest.scala @@ -0,0 +1,82 @@ +package cc.sukazyo.cono.morny.test.data.twitter + +import cc.sukazyo.cono.morny.data.twitter.FXApi +import cc.sukazyo.cono.morny.data.twitter.FXApi.Fetch +import cc.sukazyo.cono.morny.test.MornyTests +import org.scalatest.prop.TableDrivenPropertyChecks +import org.scalatest.tagobjects.{Network, Slow} + +//noinspection ScalaUnusedExpression +class FXApiTest extends MornyTests with TableDrivenPropertyChecks { + + "while fetch status (tweet) :" - { + + "non exists tweet id should return 404" taggedAs (Slow, Network) in { + val api = Fetch.status(Some("some_non_exists"), "-1") + api.code shouldEqual 404 + api.message shouldEqual "NOT_FOUND" + api.tweet shouldBe empty + } + + /** It should return 401, but in practice it seems will only + * return 404. + */ + "private tweet should return 410 or 404" taggedAs (Slow, Network) in { + val api = Fetch.status(Some("_takiChan"), "1586671758999924736") + api.code should (equal (404) or equal (401)) + api.code match + case 401 => + api.message shouldEqual "PRIVATE_TWEET" + note("from private tweet got 401 PRIVATE_TWEET") + case 404 => + api.message shouldEqual "NOT_FOUND" + note("from private tweet got 404 NOT_FOUND") + api.tweet shouldBe empty + } + + val examples = Table[(Option[String], String), FXApi =>Unit]( + ("id", "checking"), + ((Some("_Eyre_S"), "1669362743332438019"), api => { + api.tweet shouldBe defined + api.tweet.get.text shouldEqual "猫头猫头鹰头猫头鹰头猫头鹰" + api.tweet.get.quote shouldBe defined + api.tweet.get.quote.get.id shouldEqual "1669302279386828800" + }), + ((None, "1669362743332438019"), api => { + api.tweet shouldBe defined + api.tweet.get.text shouldEqual "猫头猫头鹰头猫头鹰头猫头鹰" + api.tweet.get.quote shouldBe defined + api.tweet.get.quote.get.id shouldEqual "1669302279386828800" + }), + ((None, "1654080016802807809"), api => { + api.tweet shouldBe defined + api.tweet.get.media shouldBe defined + api.tweet.get.media.get.videos shouldBe empty + api.tweet.get.media.get.photos shouldBe defined + api.tweet.get.media.get.photos.get.length shouldBe 1 + api.tweet.get.media.get.photos.get.head.width shouldBe 2048 + api.tweet.get.media.get.photos.get.head.height shouldBe 1536 + api.tweet.get.media.get.mosaic shouldBe empty + }), + ((None, "1538536152093044736"), api => { + api.tweet shouldBe defined + api.tweet.get.media shouldBe defined + api.tweet.get.media.get.videos shouldBe empty + api.tweet.get.media.get.photos shouldBe defined + api.tweet.get.media.get.photos.get.length shouldBe 2 + api.tweet.get.media.get.photos.get.head.width shouldBe 2894 + api.tweet.get.media.get.photos.get.head.height shouldBe 4093 + api.tweet.get.media.get.photos.get(1).width shouldBe 2894 + api.tweet.get.media.get.photos.get(1).height shouldBe 4093 + api.tweet.get.media.get.mosaic shouldBe defined + }) + ) + forAll(examples) { (data, assertion) => + s"tweet $data should be fetched successful" taggedAs (Slow, Network) in { + assertion(Fetch.status(data._1, data._2)) + } + } + + } + +} diff --git a/src/test/scala/cc/sukazyo/cono/morny/test/data/twitter/PackageTest.scala b/src/test/scala/cc/sukazyo/cono/morny/test/data/twitter/PackageTest.scala new file mode 100644 index 0000000..1510dc4 --- /dev/null +++ b/src/test/scala/cc/sukazyo/cono/morny/test/data/twitter/PackageTest.scala @@ -0,0 +1,123 @@ +package cc.sukazyo.cono.morny.test.data.twitter + +import cc.sukazyo.cono.morny.data.twitter.{parseTweetUrl, TweetUrlInformation} +import cc.sukazyo.cono.morny.test.MornyTests + +class PackageTest extends MornyTests { + + "while parsing tweet url :" - { + + "normal twitter tweet share url should be parsed" in { + parseTweetUrl("https://twitter.com/ps_urine/status/1727614825755505032?s=20") + .shouldEqual(Some(TweetUrlInformation( + "twitter.com", "ps_urine/status/1727614825755505032", + "ps_urine", "1727614825755505032", + None, Some("s=20") + ))) + } + + "normal X.com tweet share url should be parsed" in { + parseTweetUrl("https://x.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(defined) + } + + "X.com or twitter tweet share url should not www.sensitive" in { + parseTweetUrl("https://www.twitter.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(defined) + parseTweetUrl("https://www.x.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(defined) + } + + "fxtwitter and fixupx url should be parsed" in { + parseTweetUrl("https://fxtwitter.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(defined) + parseTweetUrl("https://fixupx.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(defined) + } + "vxtwitter should be parsed and can be with c." in { + parseTweetUrl("https://vxtwitter.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(defined) + parseTweetUrl("https://c.vxtwitter.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(defined) + } + "fixvx should be parsed and cannot be with c." in { + parseTweetUrl("https://fixvx.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(defined) + parseTweetUrl("https://c.fixvx.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(empty) + } + + "fxtwitter and vxtwitter should not contains www." in { + parseTweetUrl("https://www.fxtwitter.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(empty) + parseTweetUrl("https://www.fixupx.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(empty) + parseTweetUrl("https://www.vxtwitter.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(empty) + parseTweetUrl("https://www.fixvx.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(empty) + } + + "url should be http/s non-sensitive" in { + parseTweetUrl("twitter.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(defined) + parseTweetUrl("http://x.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(defined) + parseTweetUrl("http://fxtwitter.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(defined) + parseTweetUrl("http://fixupx.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(defined) + parseTweetUrl("vxtwitter.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(defined) + parseTweetUrl("fixvx.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(defined) + } + + "url param should be non-sensitive" in { + parseTweetUrl("twitter.com/ps_urine/status/1727614825755505032") + .shouldBe(defined) + parseTweetUrl("http://x.com/ps_urine/status/1727614825755505032/?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a") + .shouldBe(defined) + parseTweetUrl("http://fxtwitter.com/ps_urine/status/1727614825755505032/?s=20") + .shouldBe(defined) + parseTweetUrl("http://fixupx.com/ps_urine/status/1727614825755505032?s=20") + .shouldBe(defined) + parseTweetUrl("vxtwitter.com/ps_urine/status/1727614825755505032") + .shouldBe(defined) + parseTweetUrl("fixvx.com/ps_urine/status/1727614825755505032?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a") + .shouldBe(defined) + } + + "screen name should not be non-exists" in { + parseTweetUrl("twitter.com/status/1727614825755505032") + .shouldBe(empty) + parseTweetUrl("http://x.com/status/1727614825755505032/?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a") + .shouldBe(empty) + parseTweetUrl("http://fxtwitter.com/status/1727614825755505032/?s=20") + .shouldBe(empty) + parseTweetUrl("http://fixupx.com/status/1727614825755505032?s=20") + .shouldBe(empty) + parseTweetUrl("vxtwitter.com/status/1727614825755505032") + .shouldBe(empty) + parseTweetUrl("fixvx.com/status/1727614825755505032?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a") + .shouldBe(empty) + } + + "url with photo id should be parsed" in { + parseTweetUrl("twitter.com/ps_urine/status/1727614825755505032/photo/2") + .should(matchPattern { case Some(TweetUrlInformation(_, _, _, _, Some("2"), _)) => }) + parseTweetUrl("http://x.com/ps_urine/status/1727614825755505032/photo/1/?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a") + .should(matchPattern { case Some(TweetUrlInformation(_, _, _, _, Some("1"), _)) => }) + parseTweetUrl("http://fxtwitter.com/ps_urine/status/1727614825755505032/photo/4/?s=20") + .should(matchPattern { case Some(TweetUrlInformation(_, _, _, _, Some("4"), _)) => }) + parseTweetUrl("http://fixupx.com/ps_urine/status/1727614825755505032/photo/7?s=20") + .should(matchPattern { case Some(TweetUrlInformation(_, _, _, _, Some("7"), _)) => }) + parseTweetUrl("vxtwitter.com/ps_urine/status/1727614825755505032/photo/114514") + .should(matchPattern { case Some(TweetUrlInformation(_, _, _, _, Some("114514"), _)) => }) + parseTweetUrl("fixvx.com/ps_urine/status/1727614825755505032/photo/unavailable-id?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a") + .shouldBe(empty) + } + + } + +}