Switch to TweetDetail for tweets

zedeus · Aug 25, 2023 · ae9fa02 · ae9fa02
1 parent 88b005c
commit ae9fa02
Show file tree

Hide file tree

Showing 6 changed files with 31 additions and 21 deletions.
diff --git a/src/consts.nim b/src/consts.nim
@@ -17,7 +17,7 @@ const
   graphUserTweets* = graphql / "3JNH4e9dq1BifLxAa3UMWg/UserWithProfileTweetsQueryV2"
   graphUserTweetsAndReplies* = graphql / "8IS8MaO-2EN6GZZZb8jF0g/UserWithProfileTweetsAndRepliesQueryV2"
   graphUserMedia* = graphql / "PDfFf8hGeJvUCiTyWtw4wQ/MediaTimelineV2"
-  graphTweet* = graphql / "83h5UyHZ9wEKBVzALX8R_g/ConversationTimelineV2"
+  graphTweet* = graphql / "q94uRCEn65LZThakYcPT6g/TweetDetail"
   graphTweetResult* = graphql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery"
   graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline"
   graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
@@ -89,8 +89,12 @@ const
   tweetVariables* = """{
   "focalTweetId": "$1",
   $2
-  "includeHasBirdwatchNotes": false
-}"""
+  "includeHasBirdwatchNotes": false,
+  "includePromotedContent": false,
+  "withBirdwatchNotes": false,
+  "withVoice": false,
+  "withV2Timeline": true
+}""".replace(" ", "").replace("\n", "")
 
 #   oldUserTweetsVariables* = """{
 #   "userId": "$1", $2

diff --git a/src/parser.nim b/src/parser.nim
@@ -324,7 +324,7 @@ proc parseGraphTweet(js: JsonNode): Tweet =
   of "TweetWithVisibilityResults":
     return parseGraphTweet(js{"tweet"})
 
-  var jsCard = copy(js{"tweet_card", "legacy"})
+  var jsCard = copy(js{"card", "legacy"})
   if jsCard.kind != JNull:
     var values = newJObject()
     for val in jsCard["binding_values"]:
@@ -342,19 +342,23 @@ proc parseGraphTweet(js: JsonNode): Tweet =
     result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}))
 
 proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
-  let thread = js{"content", "items"}
   for t in js{"content", "items"}:
     let entryId = t{"entryId"}.getStr
     if "cursor-showmore" in entryId:
       let cursor = t{"item", "content", "value"}
       result.thread.cursor = cursor.getStr
       result.thread.hasMore = true
     elif "tweet" in entryId:
-      let tweet = parseGraphTweet(t{"item", "content", "tweetResult", "result"})
-      result.thread.content.add tweet
+      let
+        isLegacy = t{"item"}.hasKey("itemContent")
+        (contentKey, resultKey) = if isLegacy: ("itemContent", "tweet_results")
+                                  else: ("content", "tweetResult")
 
-      if t{"item", "content", "tweetDisplayType"}.getStr == "SelfThread":
-        result.self = true
+      with content, t{"item", contentKey}:
+        result.thread.content.add parseGraphTweet(content{resultKey, "result"})
+
+        if content{"tweetDisplayType"}.getStr == "SelfThread":
+          result.self = true
 
 proc parseGraphTweetResult*(js: JsonNode): Tweet =
   with tweet, js{"data", "tweet_result", "result"}:
@@ -363,14 +367,14 @@ proc parseGraphTweetResult*(js: JsonNode): Tweet =
 proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
   result = Conversation(replies: Result[Chain](beginning: true))
 
-  let instructions = ? js{"data", "timeline_response", "instructions"}
+  let instructions = ? js{"data", "threaded_conversation_with_injections_v2", "instructions"}
   if instructions.len == 0:
     return
 
   for e in instructions[0]{"entries"}:
     let entryId = e{"entryId"}.getStr
     if entryId.startsWith("tweet"):
-      with tweetResult, e{"content", "content", "tweetResult", "result"}:
+      with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
         let tweet = parseGraphTweet(tweetResult)
 
         if not tweet.available:
@@ -385,7 +389,7 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
       let tweet = Tweet(
         id: parseBiggestInt(id),
         available: false,
-        text: e{"content", "content", "tombstoneInfo", "richText"}.getTombstone
+        text: e{"content", "itemContent", "tombstoneInfo", "richText"}.getTombstone
       )
 
       if id == tweetId:
@@ -397,9 +401,10 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
       if self:
         result.after = thread
       else:
+        echo "adding thread: ", thread.content.len
         result.replies.content.add thread
     elif entryId.startsWith("cursor-bottom"):
-      result.replies.bottom = e{"content", "content", "value"}.getStr
+      result.replies.bottom = e{"content", "itemContent", "value"}.getStr
 
 proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile =
   result = Profile(tweets: Timeline(beginning: after.len == 0))

diff --git a/src/tokens.nim b/src/tokens.nim
@@ -52,10 +52,11 @@ proc getPoolJson*(): JsonNode =
         maxReqs =
           case api
           of Api.search: 50
+          of Api.tweetDetail: 150
           of Api.photoRail: 180
           of Api.userTweets, Api.userTweetsAndReplies, Api.userMedia,
              Api.userRestId, Api.userScreenName,
-             Api.tweetResult, Api.tweetDetail,
+             Api.tweetResult,
              Api.list, Api.listTweets, Api.listMembers, Api.listBySlug: 500
           of Api.userSearch: 900
         reqs = maxReqs - apiStatus.remaining

diff --git a/tests/test_card.py b/tests/test_card.py
@@ -13,18 +13,18 @@
      'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim',
      'gist.github.com', True],
 
-    ['FluentAI/status/1116417904831029248',
-     'Amazon’s Alexa isn’t just AI — thousands of humans are listening',
-     'One of the only ways to improve Alexa is to have human beings check it for errors',
-     'theverge.com', True],
-
     ['nim_lang/status/1082989146040340480',
      'Nim in 2018: A short recap',
      'There were several big news in the Nim world in 2018 – two new major releases, partnership with Status, and much more. But let us go chronologically.',
      'nim-lang.org', True]
 ]
 
 no_thumb = [
+    ['FluentAI/status/1116417904831029248',
+     'Amazon’s Alexa isn’t just AI — thousands of humans are listening',
+     'One of the only ways to improve Alexa is to have human beings check it for errors',
+     'theverge.com'],
+
     ['Thom_Wolf/status/1122466524860702729',
      'facebookresearch/fairseq',
      'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',

diff --git a/tests/test_timeline.py b/tests/test_timeline.py
@@ -6,7 +6,7 @@
 after = [['jack', '1681686036294803456'],
          ['elonmusk', '1681686036294803456']]
 
-no_more = [['mobile_test_8?cursor=1000']]
+no_more = [['mobile_test_8?cursor=DAABCgABF4YVAqN___kKAAICNn_4msIQAAgAAwAAAAIAAA']]
 
 empty = [['emptyuser'], ['mobile_test_10']]
 

diff --git a/tests/test_tweet_media.py b/tests/test_tweet_media.py
@@ -14,7 +14,7 @@
 
 image = [
     ['mobile_test/status/519364660823207936', 'BzUnaDFCUAAmrjs'],
-    ['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj']
+    #['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj']
 ]
 
 gif = [