set morny UA for all HTTP req, add twitter tests

gradle.properties

@@ -5,7 +5,7 @@ MORNY_ARCHIVE_NAME = morny-coeur
 MORNY_CODE_STORE = https://github.com/Eyre-S/Coeur-Morny-Cono
 MORNY_COMMIT_PATH = https://github.com/Eyre-S/Coeur-Morny-Cono/commit/%s
 
-VERSION = 1.3.0-dev9
+VERSION = 1.3.0-dev10
 
 USE_DELTA = false
 VERSION_DELTA =

src/main/scala/cc/sukazyo/cono/morny/data/BilibiliForms.scala

@@ -1,9 +1,9 @@
 package cc.sukazyo.cono.morny.data
 
 import cc.sukazyo.cono.morny.util.BiliTool
-import cc.sukazyo.cono.morny.util.SttpPublic.Schemes
+import cc.sukazyo.cono.morny.util.SttpPublic.{mornyBasicRequest, Schemes}
 import cc.sukazyo.cono.morny.util.UseSelect.select
-import sttp.client3.{basicRequest, ignore, HttpError, SttpClientException}
+import sttp.client3.{HttpError, SttpClientException}
 import sttp.client3.okhttp.OkHttpSyncBackend
 import sttp.model.Uri
 
@@ -77,7 +77,8 @@ object BilibiliForms {
 			throw IllegalArgumentException(s"is a b23 video link: $uri . (use parse_videoUrl instead)")
 		
 		try {
-			val response = basicRequest
+			import sttp.client3.ignore
+			val response = mornyBasicRequest
 				.get(uri)
 				.followRedirects(false)
 				.response(ignore)

src/main/scala/cc/sukazyo/cono/morny/data/NbnhhshQuery.scala

@@ -1,7 +1,8 @@
 package cc.sukazyo.cono.morny.data
 
+import cc.sukazyo.cono.morny.util.SttpPublic.mornyBasicRequest
 import com.google.gson.Gson
-import sttp.client3.{asString, basicRequest, HttpError, SttpClientException, UriContext}
+import sttp.client3.{asString, HttpError, SttpClientException, UriContext}
 import sttp.client3.okhttp.OkHttpSyncBackend
 import sttp.model.MediaType
 
@@ -22,7 +23,7 @@ object NbnhhshQuery {
 	@throws[HttpError[_]|SttpClientException]
 	def sendGuess (text: String): GuessResult = {
 		case class GuessRequest (text: String)
-		val http = basicRequest
+		val http = mornyBasicRequest
 			.body(Gson().toJson(GuessRequest(text))).contentType(MediaType.ApplicationJson)
 			.post(API_GUESS_METHOD)
 			.response(asString.getRight)

src/main/scala/cc/sukazyo/cono/morny/data/ip186/IP186QueryHandler.scala

@@ -1,7 +1,7 @@
 package cc.sukazyo.cono.morny.data.ip186
 
-import cc.sukazyo.cono.morny.util.SttpPublic.Schemes
+import cc.sukazyo.cono.morny.util.SttpPublic.{mornyBasicRequest, Schemes}
-import sttp.client3.{asString, basicRequest, HttpError, SttpClientException, UriContext}
+import sttp.client3.{asString, HttpError, SttpClientException, UriContext}
 import sttp.client3.okhttp.OkHttpSyncBackend
 import sttp.model.Uri
 
@@ -36,7 +36,7 @@ object IP186QueryHandler {
 			val uri = requestPath.scheme(Schemes.HTTPS).host(SITE_HOST)
 			IP186Response(
 				uri.toString,
-				basicRequest
+				mornyBasicRequest
 					.get(uri)
 					.response(asString.getRight)
 					.send(httpClient)

src/main/scala/cc/sukazyo/cono/morny/data/twitter/FXApi.scala

@@ -1,6 +1,7 @@
 package cc.sukazyo.cono.morny.data.twitter
 
 import cc.sukazyo.cono.morny.util.SttpPublic
+import cc.sukazyo.cono.morny.util.SttpPublic.mornyBasicRequest
 import io.circe.{DecodingFailure, ParsingFailure}
 
 /** The struct of FixTweet Status-Fetch-API response.
@@ -89,7 +90,7 @@ object FXApi {
 		  */
 		@throws[SttpClientException|ParsingFailure|DecodingFailure]
 		def status (screen_name: Option[String], id: String, translate_to: Option[String] = None): FXApi =
-			val get = basicRequest
+			val get = mornyBasicRequest
 				.header(SttpPublic.Headers.UserAgent.MORNY_CURRENT)
 				.get(uri_status(screen_name, id, translate_to))
 				.response(asString)

src/main/scala/cc/sukazyo/cono/morny/data/twitter/FXTweet.scala

@@ -16,7 +16,7 @@ import cc.sukazyo.cono.morny.util.EpochDateTime.EpochSeconds
   * @param replying_to_status Tweet ID snowflake being replied to, or null
   * @param twitter_card Corresponds to proper embed container for Tweet, which is used by
   *                     FixTweet for our official embeds.<br>
-  *                     Notice that this should be of type [[]]
+  *                     Notice that this should be of type [["tweet"|"summary"|"summary_large_image"|"player"]]
   *                     but due to circe parser does not support it well so alternative
   *                     [[String]] type is used.
   * @param author Author of the tweet
@@ -48,7 +48,7 @@ case class FXTweet (
 	replying_to: Option[String],
 	replying_to_status: Option[String],
 //	twitter_card: "tweet"|"summary"|"summary_large_image"|"player",
-	twitter_card: String,
+	twitter_card: Option[String],
 	author: FXAuthor,
 	source: String,
 	

src/main/scala/cc/sukazyo/cono/morny/data/twitter/package.scala

@@ -4,7 +4,7 @@ import scala.util.matching.Regex
 
 package object twitter {
 	
-	private val REGEX_TWEET_URL: Regex = "^(?:https?://)?((?:(?:(?:c\\.)?vx|fx|www\\.)?twitter|(?:www\\.|fixup)?x)\\.com)/((\\w+)/status/(\\d+)(?:/photo/(\\d+))?)/?(\\?[\\w&=-]+)?$"r
+	private val REGEX_TWEET_URL: Regex = "^(?:https?://)?((?:(?:(?:c\\.)?vx|fx|www\\.)?twitter|(?:www\\.|fixup|fixv)?x)\\.com)/((\\w+)/status/(\\d+)(?:/photo/(\\d+))?)/?(?:\\?([\\w&=-]+))?$"r
 	
 	/** Messages that can contains on a tweet url.
 	  *
@@ -49,7 +49,7 @@ package object twitter {
 	  *  - `twitter.com` or `www.twitter.com`
 	  *  - `x.com` or `www.x.com`
 	  *  - `fxtwitter.com` or `fixupx.com`
-	  *  - `vxtwitter.com` or `c.vxtwitter.com`
+	  *  - `vxtwitter.com` or `c.vxtwitter.com` or `fixvx.com`
 	  *  - should be the path of `/:screenName/status/:id`
 	  *  - can contains `./photo/:photoId`
 	  *  - url param non-sensitive

src/main/scala/cc/sukazyo/cono/morny/util/SttpPublic.scala

@@ -1,6 +1,7 @@
 package cc.sukazyo.cono.morny.util
 
 import cc.sukazyo.cono.morny.MornySystem
+import sttp.client3.basicRequest
 import sttp.model.Header
 
 object SttpPublic {
@@ -16,10 +17,14 @@ object SttpPublic {
 			
 			private val key = "User-Agent"
 			
-			val MORNY_CURRENT = Header(key, s"MornyCoeur / ${MornySystem.VERSION}")
+			val MORNY_CURRENT: Header = Header(key, s"MornyCoeur / ${MornySystem.VERSION}")
 			
 		}
 		
 	}
 	
+	val mornyBasicRequest =
+		basicRequest
+			.header(Headers.UserAgent.MORNY_CURRENT, true)
+	
 }

src/main/scala/cc/sukazyo/cono/morny/util/tgapi/formatting/TelegramUserInformation.scala

@@ -1,7 +1,8 @@
 package cc.sukazyo.cono.morny.util.tgapi.formatting
 
+import cc.sukazyo.cono.morny.util.SttpPublic.mornyBasicRequest
 import com.pengrad.telegrambot.model.User
-import sttp.client3.{asString, basicRequest, HttpError, SttpClientException, UriContext}
+import sttp.client3.{asString, HttpError, SttpClientException, UriContext}
 import sttp.client3.okhttp.OkHttpSyncBackend
 
 import java.io.IOException
@@ -17,7 +18,7 @@ object TelegramUserInformation {
 	def getDataCenterFromUser (username: String): String = {
 		
 		try
-			val body = basicRequest
+			val body = mornyBasicRequest
 				.get(uri"https://t.me/$username")
 				.response(asString.getRight)
 				.send(httpClient)

src/test/scala/cc/sukazyo/cono/morny/test/data/twitter/FXApiTest.scala

@@ -0,0 +1,82 @@
+package cc.sukazyo.cono.morny.test.data.twitter
+
+import cc.sukazyo.cono.morny.data.twitter.FXApi
+import cc.sukazyo.cono.morny.data.twitter.FXApi.Fetch
+import cc.sukazyo.cono.morny.test.MornyTests
+import org.scalatest.prop.TableDrivenPropertyChecks
+import org.scalatest.tagobjects.{Network, Slow}
+
+//noinspection ScalaUnusedExpression
+class FXApiTest extends MornyTests with TableDrivenPropertyChecks {
+	
+	"while fetch status (tweet) :"  - {
+		
+		"non exists tweet id should return 404" taggedAs (Slow, Network) in {
+			val api = Fetch.status(Some("some_non_exists"), "-1")
+			api.code shouldEqual 404
+			api.message shouldEqual "NOT_FOUND"
+			api.tweet shouldBe empty
+		}
+		
+		/** It should return 401, but in practice it seems will only
+		  * return 404.
+		  */
+		"private tweet should return 410 or 404" taggedAs (Slow, Network) in {
+			val api = Fetch.status(Some("_takiChan"), "1586671758999924736")
+			api.code should (equal (404) or equal (401))
+			api.code match
+				case 401 =>
+					api.message shouldEqual "PRIVATE_TWEET"
+					note("from private tweet got 401 PRIVATE_TWEET")
+				case 404 =>
+					api.message shouldEqual "NOT_FOUND"
+					note("from private tweet got 404 NOT_FOUND")
+			api.tweet shouldBe empty
+		}
+		
+		val examples = Table[(Option[String], String), FXApi =>Unit](
+			("id", "checking"),
+			((Some("_Eyre_S"), "1669362743332438019"), api => {
+				api.tweet shouldBe defined
+				api.tweet.get.text shouldEqual "猫头猫头鹰头猫头鹰头猫头鹰"
+				api.tweet.get.quote shouldBe defined
+				api.tweet.get.quote.get.id shouldEqual "1669302279386828800"
+			}),
+			((None, "1669362743332438019"), api => {
+				api.tweet shouldBe defined
+				api.tweet.get.text shouldEqual "猫头猫头鹰头猫头鹰头猫头鹰"
+				api.tweet.get.quote shouldBe defined
+				api.tweet.get.quote.get.id shouldEqual "1669302279386828800"
+			}),
+			((None, "1654080016802807809"), api => {
+				api.tweet shouldBe defined
+				api.tweet.get.media shouldBe defined
+				api.tweet.get.media.get.videos shouldBe empty
+				api.tweet.get.media.get.photos shouldBe defined
+				api.tweet.get.media.get.photos.get.length shouldBe 1
+				api.tweet.get.media.get.photos.get.head.width shouldBe 2048
+				api.tweet.get.media.get.photos.get.head.height shouldBe 1536
+				api.tweet.get.media.get.mosaic shouldBe empty
+			}),
+			((None, "1538536152093044736"), api => {
+				api.tweet shouldBe defined
+				api.tweet.get.media shouldBe defined
+				api.tweet.get.media.get.videos shouldBe empty
+				api.tweet.get.media.get.photos shouldBe defined
+				api.tweet.get.media.get.photos.get.length shouldBe 2
+				api.tweet.get.media.get.photos.get.head.width shouldBe 2894
+				api.tweet.get.media.get.photos.get.head.height shouldBe 4093
+				api.tweet.get.media.get.photos.get(1).width shouldBe 2894
+				api.tweet.get.media.get.photos.get(1).height shouldBe 4093
+				api.tweet.get.media.get.mosaic shouldBe defined
+			})
+		)
+		forAll(examples) { (data, assertion) =>
+			s"tweet $data should be fetched successful" taggedAs (Slow, Network) in {
+				assertion(Fetch.status(data._1, data._2))
+			}
+		}
+		
+	}
+	
+}

src/test/scala/cc/sukazyo/cono/morny/test/data/twitter/PackageTest.scala

@@ -0,0 +1,123 @@
+package cc.sukazyo.cono.morny.test.data.twitter
+
+import cc.sukazyo.cono.morny.data.twitter.{parseTweetUrl, TweetUrlInformation}
+import cc.sukazyo.cono.morny.test.MornyTests
+
+class PackageTest extends MornyTests {
+	
+	"while parsing tweet url :" - {
+		
+		"normal twitter tweet share url should be parsed" in {
+			parseTweetUrl("https://twitter.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldEqual(Some(TweetUrlInformation(
+					"twitter.com", "ps_urine/status/1727614825755505032",
+					"ps_urine", "1727614825755505032",
+					None, Some("s=20")
+				)))
+		}
+		
+		"normal X.com tweet share url should be parsed" in {
+			parseTweetUrl("https://x.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(defined)
+		}
+		
+		"X.com or twitter tweet share url should not www.sensitive" in {
+			parseTweetUrl("https://www.twitter.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(defined)
+			parseTweetUrl("https://www.x.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(defined)
+		}
+		
+		"fxtwitter and fixupx url should be parsed" in {
+			parseTweetUrl("https://fxtwitter.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(defined)
+			parseTweetUrl("https://fixupx.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(defined)
+		}
+		"vxtwitter should be parsed and can be with c." in {
+			parseTweetUrl("https://vxtwitter.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(defined)
+			parseTweetUrl("https://c.vxtwitter.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(defined)
+		}
+		"fixvx should be parsed and cannot be with c." in {
+			parseTweetUrl("https://fixvx.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(defined)
+			parseTweetUrl("https://c.fixvx.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(empty)
+		}
+		
+		"fxtwitter and vxtwitter should not contains www." in {
+			parseTweetUrl("https://www.fxtwitter.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(empty)
+			parseTweetUrl("https://www.fixupx.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(empty)
+			parseTweetUrl("https://www.vxtwitter.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(empty)
+			parseTweetUrl("https://www.fixvx.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(empty)
+		}
+		
+		"url should be http/s non-sensitive" in {
+			parseTweetUrl("twitter.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(defined)
+			parseTweetUrl("http://x.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(defined)
+			parseTweetUrl("http://fxtwitter.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(defined)
+			parseTweetUrl("http://fixupx.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(defined)
+			parseTweetUrl("vxtwitter.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(defined)
+			parseTweetUrl("fixvx.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(defined)
+		}
+		
+		"url param should be non-sensitive" in {
+			parseTweetUrl("twitter.com/ps_urine/status/1727614825755505032")
+				.shouldBe(defined)
+			parseTweetUrl("http://x.com/ps_urine/status/1727614825755505032/?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a")
+				.shouldBe(defined)
+			parseTweetUrl("http://fxtwitter.com/ps_urine/status/1727614825755505032/?s=20")
+				.shouldBe(defined)
+			parseTweetUrl("http://fixupx.com/ps_urine/status/1727614825755505032?s=20")
+				.shouldBe(defined)
+			parseTweetUrl("vxtwitter.com/ps_urine/status/1727614825755505032")
+				.shouldBe(defined)
+			parseTweetUrl("fixvx.com/ps_urine/status/1727614825755505032?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a")
+				.shouldBe(defined)
+		}
+		
+		"screen name should not be non-exists" in {
+			parseTweetUrl("twitter.com/status/1727614825755505032")
+				.shouldBe(empty)
+			parseTweetUrl("http://x.com/status/1727614825755505032/?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a")
+				.shouldBe(empty)
+			parseTweetUrl("http://fxtwitter.com/status/1727614825755505032/?s=20")
+				.shouldBe(empty)
+			parseTweetUrl("http://fixupx.com/status/1727614825755505032?s=20")
+				.shouldBe(empty)
+			parseTweetUrl("vxtwitter.com/status/1727614825755505032")
+				.shouldBe(empty)
+			parseTweetUrl("fixvx.com/status/1727614825755505032?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a")
+				.shouldBe(empty)
+		}
+		
+		"url with photo id should be parsed" in {
+			parseTweetUrl("twitter.com/ps_urine/status/1727614825755505032/photo/2")
+				.should(matchPattern { case Some(TweetUrlInformation(_, _, _, _, Some("2"), _)) => })
+			parseTweetUrl("http://x.com/ps_urine/status/1727614825755505032/photo/1/?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a")
+				.should(matchPattern { case Some(TweetUrlInformation(_, _, _, _, Some("1"), _)) => })
+			parseTweetUrl("http://fxtwitter.com/ps_urine/status/1727614825755505032/photo/4/?s=20")
+				.should(matchPattern { case Some(TweetUrlInformation(_, _, _, _, Some("4"), _)) => })
+			parseTweetUrl("http://fixupx.com/ps_urine/status/1727614825755505032/photo/7?s=20")
+				.should(matchPattern { case Some(TweetUrlInformation(_, _, _, _, Some("7"), _)) => })
+			parseTweetUrl("vxtwitter.com/ps_urine/status/1727614825755505032/photo/114514")
+				.should(matchPattern { case Some(TweetUrlInformation(_, _, _, _, Some("114514"), _)) => })
+			parseTweetUrl("fixvx.com/ps_urine/status/1727614825755505032/photo/unavailable-id?q=ajisdl&form=ANNNB1&refig=5883b79c966b4881b79b50cb6f1c6c6a")
+				.shouldBe(empty)
+		}
+		
+	}
+	
+}