From f213b5cf701f3a1b4e0418b073fbc64bf755c524 Mon Sep 17 00:00:00 2001 From: zernonia Date: Wed, 1 Jun 2022 11:11:43 +0800 Subject: [PATCH 01/42] test: using cheerio to extract tweets --- package.json | 1 + pages/test.vue | 16 ++++ server/_lib/extractor.ts | 196 +++++++++++++++++++++++++++++++++++++++ server/api/test.ts | 10 ++ yarn.lock | 98 +++++++++++++++++++- 5 files changed, 318 insertions(+), 3 deletions(-) create mode 100644 pages/test.vue create mode 100644 server/_lib/extractor.ts create mode 100644 server/api/test.ts diff --git a/package.json b/package.json index c088e20..d73b138 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "@supabase/supabase-js": "^1.35.3", "@vueuse/core": "^8.4.2", "@yeger/vue-masonry-wall": "^3.0.30", + "cheerio": "^1.0.0-rc.11", "highlight.js": "^11.5.1", "lodash-es": "^4.17.21", "node-html-parser": "^5.3.3", diff --git a/pages/test.vue b/pages/test.vue new file mode 100644 index 0000000..65c108c --- /dev/null +++ b/pages/test.vue @@ -0,0 +1,16 @@ + + + diff --git a/server/_lib/extractor.ts b/server/_lib/extractor.ts new file mode 100644 index 0000000..fb64c72 --- /dev/null +++ b/server/_lib/extractor.ts @@ -0,0 +1,196 @@ +import { CheerioAPI, load } from "cheerio" + +export const extractTweetContent = (data: string, id: string) => { + const $ = load(data[id], { + decodeEntities: false, + xmlMode: false, + }) + return getTweetContent($) +} + +export const getTweetContent = ($: CheerioAPI) => { + const container = $(".EmbeddedTweet-tweetContainer") + + if (!container.length) return + + const meta: any = {} + const content: any = { meta } + + // This is the blockquote with the tweet + const subject = container.find('[data-scribe="section:subject"]') + + // Tweet header with the author info + const header = subject.children(".Tweet-header") + const avatar = header.find('[data-scribe="element:avatar"]') + const author = header.find('[data-scribe="component:author"]') + const name = author.find('[data-scribe="element:name"]') + const screenName = author.find('[data-scribe="element:screen_name"]') + + // Tweet body + const tweet = subject.children('[data-scribe="component:tweet"]') + const tweetContent = tweet.children("p") + const card = tweet.children(".Tweet-card") + const tweetInfo = tweet.children(".TweetInfo") + const fullTimestamp = tweetInfo.find('[data-scribe="element:full_timestamp"]') + const heartCount = tweetInfo.find('[data-scribe="element:heart_count"]') + + // Tweet footer + const callToAction = container.children('[data-scribe="section:cta component:news"]') + const profileText = callToAction.children('[data-scribe="element:profile_text"]') + const conversationText = callToAction.children('[data-scribe="element:conversation_text"]') + + let quotedTweet: { id: string; url: string } + let mediaHtml: string + + meta.id = subject.attr("data-tweet-id") + meta.avatar = { + normal: avatar.attr("data-src-1x"), + } + meta.name = name.text() + meta.username = screenName.text().substring(1) // Omit the initial @ + meta.createdAt = new Date(fullTimestamp.attr("data-datetime")).getTime() + meta.heartCount = heartCount.text() + meta.ctaType = profileText.length ? "profile" : "conversation" + + if (conversationText.length) { + // Get the formatted count and skip the rest + meta.ctaCount = conversationText.text().match(/^[^\s]+/)[0] + } + + // If some text ends without a trailing space, it's missing a
+ tweetContent.contents().each(function () { + const el = $(this) + const type = el[0].type + + if (type !== "text") return + + const text = el.text() + + if (text.length && text.trim() === "") { + if (el.next().children().length) { + el.after($("
")) + } + } else if (!/\s$/.test(el.text()) && el.next().children().length && !/^[#@]/.test(el.next().text())) { + el.after($("
")) + } + }) + + card.children().each(function () { + const props = this.attribs + const scribe = props["data-scribe"] + const el = $(this) + + if (scribe === "section:quote") { + const tweetCard = el.children("a") + const id = tweetCard.attr("data-tweet-id") + const url = tweetCard.attr("href") + + quotedTweet = { id, url } + return + } + + const media = $("
") + + if (scribe === "component:card") { + const photo = el.children('[data-scribe="element:photo"]') + const photoGrid = el.children('[data-scribe="element:photo_grid"]') + const photos = photo.length ? photo : photoGrid + + if (photos.length) { + const images = photos.find("img") + + images.each(function () { + const img = $(this) + const alt = img.attr("alt") + const url = img.attr("data-image") + const format = img.attr("data-image-format") + const height = img.attr("height") + const width = img.attr("width") + + this.attribs = { + "data-type": "media-image", + src: `${url}?format=${format}`, + height, + width, + } + if (alt) { + this.attribs.alt = alt + } + // Move the media img to a new container + media.append(img) + }) + media.attr("data-type", `image-container ${images.length}`) + mediaHtml = $("
").append(media).html() + } + } + }) + + tweetContent.children("img").each(function () { + const props = this.attribs + + // Handle emojis inside the text + if (props.class?.includes("Emoji--forText")) { + this.attribs = { + class: "emoji", + "data-type": "emoji-for-text", + src: props.src, + alt: props.alt, + } + return + } + + console.error("An image with the following props is not being handled:", props) + }) + + tweetContent.children("a").each(function () { + const props = this.attribs + const scribe = props["data-scribe"] + const el = $(this) + const asTwitterLink = (type) => { + this.attribs = { + "data-type": type, + href: props.href, + target: "_blank", + } + // Replace custom tags inside the anchor with text + el.text(el.text()) + } + + // @mention + if (scribe === "element:mention") { + return asTwitterLink("mention") + } + + // #hashtag + if (scribe === "element:hashtag") { + // A hashtag may be a $cashtag too + const type = props["data-query-source"] === "cashtag_click" ? "cashtag" : "hashtag" + return asTwitterLink(type) + } + + if (scribe === "element:url") { + // const quotedTweetId = props['data-tweet-id'] + + // Remove link to quoted tweet to leave the card only + // if (quotedTweetId && quotedTweetId === quotedTweet?.id) { + // el.remove(); + // return; + // } + + this.attribs = { + "data-type": "url", + href: props.href, + target: "_blank", + } + el.children(".u-hiddenVisually").remove() + return + } + }) + + content.html = tweetContent.html() + + if (quotedTweet) content.quotedTweet = quotedTweet + if (mediaHtml) content.mediaHtml = mediaHtml + + return content +} diff --git a/server/api/test.ts b/server/api/test.ts new file mode 100644 index 0000000..b6dc4b1 --- /dev/null +++ b/server/api/test.ts @@ -0,0 +1,10 @@ +import { extractTweetContent } from "../_lib/extractor" + +export default defineEventHandler(async (event) => { + const { url, layout, css, show_original_link, enable_twemoji } = useQuery(event) + const id = url.toString().split("/")[5] + + const data = await $fetch(`https://syndication.twitter.com/tweets.json?ids=${id}`) + + return { ...extractTweetContent(data, id), data } +}) diff --git a/yarn.lock b/yarn.lock index bf5ba9b..26d1ae6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1242,6 +1242,32 @@ cheerio-select@^1.5.0: domhandler "^4.3.1" domutils "^2.8.0" +cheerio-select@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/cheerio-select/-/cheerio-select-2.1.0.tgz#4d8673286b8126ca2a8e42740d5e3c4884ae21b4" + integrity sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g== + dependencies: + boolbase "^1.0.0" + css-select "^5.1.0" + css-what "^6.1.0" + domelementtype "^2.3.0" + domhandler "^5.0.3" + domutils "^3.0.1" + +cheerio@^1.0.0-rc.11: + version "1.0.0-rc.11" + resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-1.0.0-rc.11.tgz#1be84be1a126958366bcc57a11648cd9b30a60c2" + integrity sha512-bQwNaDIBKID5ts/DsdhxrjqFXYfLw4ste+wMKqWA8DyKcS4qwsPP4Bk8ZNaTJjvpiX/qW3BT4sU7d6Bh5i+dag== + dependencies: + cheerio-select "^2.1.0" + dom-serializer "^2.0.0" + domhandler "^5.0.3" + domutils "^3.0.1" + htmlparser2 "^8.0.1" + parse5 "^7.0.0" + parse5-htmlparser2-tree-adapter "^7.0.0" + tslib "^2.4.0" + cheerio@^1.0.0-rc.3: version "1.0.0-rc.10" resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-1.0.0-rc.10.tgz#2ba3dcdfcc26e7956fc1f440e61d51c643379f3e" @@ -1496,6 +1522,17 @@ css-select@^4.1.3, css-select@^4.2.1, css-select@^4.3.0: domutils "^2.8.0" nth-check "^2.0.1" +css-select@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/css-select/-/css-select-5.1.0.tgz#b8ebd6554c3637ccc76688804ad3f6a6fdaea8a6" + integrity sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg== + dependencies: + boolbase "^1.0.0" + css-what "^6.1.0" + domhandler "^5.0.2" + domutils "^3.0.1" + nth-check "^2.0.1" + css-tree@^1.1.2, css-tree@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-1.1.3.tgz#eb4870fb6fd7707327ec95c2ff2ab09b5e8db91d" @@ -1504,7 +1541,7 @@ css-tree@^1.1.2, css-tree@^1.1.3: mdn-data "2.0.14" source-map "^0.6.1" -css-what@^6.0.1: +css-what@^6.0.1, css-what@^6.1.0: version "6.1.0" resolved "https://registry.yarnpkg.com/css-what/-/css-what-6.1.0.tgz#fb5effcf76f1ddea2c81bdfaa4de44e79bac70f4" integrity sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw== @@ -1704,7 +1741,16 @@ dom-serializer@^1.0.1, dom-serializer@^1.3.2: domhandler "^4.2.0" entities "^2.0.0" -domelementtype@^2.0.1, domelementtype@^2.2.0: +dom-serializer@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-2.0.0.tgz#e41b802e1eedf9f6cae183ce5e622d789d7d8e53" + integrity sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.2" + entities "^4.2.0" + +domelementtype@^2.0.1, domelementtype@^2.2.0, domelementtype@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.3.0.tgz#5c45e8e869952626331d7aab326d01daf65d589d" integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw== @@ -1716,6 +1762,13 @@ domhandler@^4.0.0, domhandler@^4.2.0, domhandler@^4.3.1: dependencies: domelementtype "^2.2.0" +domhandler@^5.0.1, domhandler@^5.0.2, domhandler@^5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-5.0.3.tgz#cc385f7f751f1d1fc650c21374804254538c7d31" + integrity sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w== + dependencies: + domelementtype "^2.3.0" + domutils@^2.5.2, domutils@^2.8.0: version "2.8.0" resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.8.0.tgz#4437def5db6e2d1f5d6ee859bd95ca7d02048135" @@ -1725,6 +1778,15 @@ domutils@^2.5.2, domutils@^2.8.0: domelementtype "^2.2.0" domhandler "^4.2.0" +domutils@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/domutils/-/domutils-3.0.1.tgz#696b3875238338cb186b6c0612bd4901c89a4f1c" + integrity sha512-z08c1l761iKhDFtfXO04C7kTdPBLi41zwOZl00WS8b5eiaebNpY00HKbztwBq+e3vyqWNwWF3mP9YLUeqIrF+Q== + dependencies: + dom-serializer "^2.0.0" + domelementtype "^2.3.0" + domhandler "^5.0.1" + dot-prop@^7.2.0: version "7.2.0" resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-7.2.0.tgz#468172a3529779814d21a779c1ba2f6d76609809" @@ -1796,6 +1858,11 @@ entities@^2.0.0: resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== +entities@^4.2.0, entities@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-4.3.0.tgz#62915f08d67353bb4eb67e3d62641a4059aec656" + integrity sha512-/iP1rZrSEJ0DTlPiX+jbzlA3eVkY/e8L8SozroF395fIqE3TYF/Nz7YOMAawta+vLmyJ/hkGNNPcSbMADCCXbg== + errno@^0.1.3: version "0.1.8" resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.8.tgz#8bb3e9c7d463be4976ff888f76b4809ebc2e811f" @@ -2405,6 +2472,16 @@ htmlparser2@^6.1.0: domutils "^2.5.2" entities "^2.0.0" +htmlparser2@^8.0.1: + version "8.0.1" + resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-8.0.1.tgz#abaa985474fcefe269bc761a779b544d7196d010" + integrity sha512-4lVbmc1diZC7GUJQtRQ5yBAeUCL1exyMwmForWkRLnwyzWBFxN633SALPMGYaWZvKe9j1pRZJpauvmxENSp/EA== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.2" + domutils "^3.0.1" + entities "^4.3.0" + http-errors@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-2.0.0.tgz#b7774a1486ef73cf7667ac9ae0858c012c57b9d3" @@ -3484,11 +3561,26 @@ parse5-htmlparser2-tree-adapter@^6.0.1: dependencies: parse5 "^6.0.1" +parse5-htmlparser2-tree-adapter@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz#23c2cc233bcf09bb7beba8b8a69d46b08c62c2f1" + integrity sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g== + dependencies: + domhandler "^5.0.2" + parse5 "^7.0.0" + parse5@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/parse5/-/parse5-6.0.1.tgz#e1a1c085c569b3dc08321184f19a39cc27f7c30b" integrity sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw== +parse5@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/parse5/-/parse5-7.0.0.tgz#51f74a5257f5fcc536389e8c2d0b3802e1bfa91a" + integrity sha512-y/t8IXSPWTuRZqXc0ajH/UwDj4mnqLEbSttNbThcFhGrZuOyoyvNBO85PBp2jQa55wY9d07PBNjsK8ZP3K5U6g== + dependencies: + entities "^4.3.0" + parseurl@~1.3.3: version "1.3.3" resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" @@ -4504,7 +4596,7 @@ tr46@~0.0.3: resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o= -tslib@^2.1.0, tslib@^2.2.0: +tslib@^2.1.0, tslib@^2.2.0, tslib@^2.4.0: version "2.4.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3" integrity sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ== From 4b87537a4d122c7f2c098cd689a73fa403f29949 Mon Sep 17 00:00:00 2001 From: zernonia Date: Wed, 1 Jun 2022 12:16:44 +0800 Subject: [PATCH 02/42] update: publish v2 api --- components/Tweet.vue | 2 +- interface.ts | 24 +++++ pages/test.vue | 6 +- server/_lib/{extractor.ts => parserv2.ts} | 102 ++++++++++++++++------ server/api/test.ts | 10 --- server/api/v2/tweet.ts | 18 ++++ 6 files changed, 119 insertions(+), 43 deletions(-) rename server/_lib/{extractor.ts => parserv2.ts} (52%) delete mode 100644 server/api/test.ts create mode 100644 server/api/v2/tweet.ts diff --git a/components/Tweet.vue b/components/Tweet.vue index f1f2710..a8858b1 100644 --- a/components/Tweet.vue +++ b/components/Tweet.vue @@ -12,7 +12,7 @@ const props = defineProps({ const { data, pending } = await useAsyncData( JSON.stringify(props), () => - $fetch("/api/tweet", { + $fetch("/api/v2/tweet", { params: { ...props }, }), { diff --git a/interface.ts b/interface.ts index 929a27b..186409c 100644 --- a/interface.ts +++ b/interface.ts @@ -22,3 +22,27 @@ export interface TweetOptions { export interface ExportOptions { css: string } + +export interface TweetContent { + meta: { + id?: string + url?: string + avatar?: { + [key: string]: string + } + name?: string + username?: string + profile_url?: string + created_at?: number + heart_count?: string + cta_type?: string + cta_count?: string + [key: string]: any + } + html: string + quoted_tweet?: { + id: string + url: string + } + media_html?: string +} diff --git a/pages/test.vue b/pages/test.vue index 65c108c..51a2961 100644 --- a/pages/test.vue +++ b/pages/test.vue @@ -1,7 +1,6 @@ diff --git a/server/_lib/extractor.ts b/server/_lib/parserv2.ts similarity index 52% rename from server/_lib/extractor.ts rename to server/_lib/parserv2.ts index fb64c72..f753d87 100644 --- a/server/_lib/extractor.ts +++ b/server/_lib/parserv2.ts @@ -1,20 +1,64 @@ -import { CheerioAPI, load } from "cheerio" +import { load } from "cheerio" +import { TweetOptions, TweetContent } from "~~/interface" +import { mapClass } from "./reference" + +export const constructHtmlv2 = (data: TweetContent, options: TweetOptions) => { + const { meta, html: content } = data + const mapClassOptions = (key: string) => mapClass(key, options) + + const html = ` +
+
+ ${ + options.layout == "supabase" + ? `
+ + +
+

+ @${ + meta.username + } +
+
` + : `
+ +
+

${meta.name}

+ @${ + meta.username + } +
+
+ ` + } +
+ +
+ ${content} +
+
+ ` + return html +} -export const extractTweetContent = (data: string, id: string) => { - const $ = load(data[id], { +export const getTweetContent = (data: { [key: string]: string }, options: TweetOptions) => { + let d = Object.values(data)[0] + const $ = load(d, { decodeEntities: false, xmlMode: false, }) - return getTweetContent($) -} -export const getTweetContent = ($: CheerioAPI) => { const container = $(".EmbeddedTweet-tweetContainer") if (!container.length) return - const meta: any = {} - const content: any = { meta } + const meta: TweetContent["meta"] = {} + const content: TweetContent = { meta, html: "" } // This is the blockquote with the tweet const subject = container.find('[data-scribe="section:subject"]') @@ -41,20 +85,21 @@ export const getTweetContent = ($: CheerioAPI) => { let quotedTweet: { id: string; url: string } let mediaHtml: string - meta.id = subject.attr("data-tweet-id") + meta.url = subject.attr("cite") meta.avatar = { normal: avatar.attr("data-src-1x"), } meta.name = name.text() meta.username = screenName.text().substring(1) // Omit the initial @ - meta.createdAt = new Date(fullTimestamp.attr("data-datetime")).getTime() - meta.heartCount = heartCount.text() - meta.ctaType = profileText.length ? "profile" : "conversation" + meta.profile_url = "https://twitter.com/" + meta.username + meta.created_at = new Date(fullTimestamp.attr("data-datetime")).getTime() + meta.heart_count = heartCount.text() + meta.cta_type = profileText.length ? "profile" : "conversation" if (conversationText.length) { // Get the formatted count and skip the rest - meta.ctaCount = conversationText.text().match(/^[^\s]+/)[0] + meta.cta_count = conversationText.text().match(/^[^\s]+/)[0] } // If some text ends without a trailing space, it's missing a
@@ -131,10 +176,13 @@ export const getTweetContent = ($: CheerioAPI) => { // Handle emojis inside the text if (props.class?.includes("Emoji--forText")) { this.attribs = { - class: "emoji", "data-type": "emoji-for-text", src: props.src, alt: props.alt, + class: + options.css === "tailwind" + ? "inline-block align-text-bottom w-[1.2em] h-[1.2em] mr-[0.05em] ml-[0.1em]" + : "emoji", } return } @@ -146,26 +194,31 @@ export const getTweetContent = ($: CheerioAPI) => { const props = this.attribs const scribe = props["data-scribe"] const el = $(this) - const asTwitterLink = (type) => { + const asLink = (type: string) => { this.attribs = { "data-type": type, href: props.href, target: "_blank", + class: options.css == "tailwind" ? "text-blue-400" : "tweet-content-link", + } + if (type === "url") { + el.children(".u-hiddenVisually").remove() + return } - // Replace custom tags inside the anchor with text el.text(el.text()) + return } // @mention if (scribe === "element:mention") { - return asTwitterLink("mention") + asLink("mention") } // #hashtag if (scribe === "element:hashtag") { // A hashtag may be a $cashtag too const type = props["data-query-source"] === "cashtag_click" ? "cashtag" : "hashtag" - return asTwitterLink(type) + asLink(type) } if (scribe === "element:url") { @@ -176,21 +229,14 @@ export const getTweetContent = ($: CheerioAPI) => { // el.remove(); // return; // } - - this.attribs = { - "data-type": "url", - href: props.href, - target: "_blank", - } - el.children(".u-hiddenVisually").remove() - return + asLink("url") } }) content.html = tweetContent.html() - if (quotedTweet) content.quotedTweet = quotedTweet - if (mediaHtml) content.mediaHtml = mediaHtml + if (quotedTweet) content.quoted_tweet = quotedTweet + if (mediaHtml) content.media_html = mediaHtml return content } diff --git a/server/api/test.ts b/server/api/test.ts deleted file mode 100644 index b6dc4b1..0000000 --- a/server/api/test.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { extractTweetContent } from "../_lib/extractor" - -export default defineEventHandler(async (event) => { - const { url, layout, css, show_original_link, enable_twemoji } = useQuery(event) - const id = url.toString().split("/")[5] - - const data = await $fetch(`https://syndication.twitter.com/tweets.json?ids=${id}`) - - return { ...extractTweetContent(data, id), data } -}) diff --git a/server/api/v2/tweet.ts b/server/api/v2/tweet.ts new file mode 100644 index 0000000..373bccb --- /dev/null +++ b/server/api/v2/tweet.ts @@ -0,0 +1,18 @@ +import { TweetOptions } from "~~/interface" +import { constructHtmlv2, getTweetContent } from "../../_lib/parserv2" + +export default defineEventHandler(async (event) => { + const { url, layout, css } = useQuery(event) + const id = url.toString().split("/")[5] + + const options: TweetOptions = { + layout: layout?.toString(), + css: css?.toString(), + } + + const data = await $fetch<{ [key: string]: string }>(`https://syndication.twitter.com/tweets.json?ids=${id}`) + + const tweetContent = getTweetContent(data, options) + const html = constructHtmlv2(tweetContent, options) + return { html, meta: tweetContent.meta, data } +}) From 15baa7f57190850c2f819cdb69b987331effe951 Mon Sep 17 00:00:00 2001 From: zernonia Date: Wed, 1 Jun 2022 12:22:33 +0800 Subject: [PATCH 03/42] update: add twemoji toggle to parser --- server/_lib/parserv2.ts | 5 ++++- server/api/v2/tweet.ts | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/server/_lib/parserv2.ts b/server/_lib/parserv2.ts index f753d87..21a6d0a 100644 --- a/server/_lib/parserv2.ts +++ b/server/_lib/parserv2.ts @@ -172,7 +172,10 @@ export const getTweetContent = (data: { [key: string]: string }, options: TweetO tweetContent.children("img").each(function () { const props = this.attribs - + if (!options.enable_twemoji) { + const el = $(this) + el.replaceWith(props.alt) + } // Handle emojis inside the text if (props.class?.includes("Emoji--forText")) { this.attribs = { diff --git a/server/api/v2/tweet.ts b/server/api/v2/tweet.ts index 373bccb..dd904e7 100644 --- a/server/api/v2/tweet.ts +++ b/server/api/v2/tweet.ts @@ -2,12 +2,13 @@ import { TweetOptions } from "~~/interface" import { constructHtmlv2, getTweetContent } from "../../_lib/parserv2" export default defineEventHandler(async (event) => { - const { url, layout, css } = useQuery(event) + const { url, layout, css, enable_twemoji } = useQuery(event) const id = url.toString().split("/")[5] const options: TweetOptions = { layout: layout?.toString(), css: css?.toString(), + enable_twemoji: enable_twemoji ? JSON.parse(enable_twemoji.toString()) : false, } const data = await $fetch<{ [key: string]: string }>(`https://syndication.twitter.com/tweets.json?ids=${id}`) From 106d0b64067a2d0bcaa99cb33b57815565fc2b66 Mon Sep 17 00:00:00 2001 From: zernonia Date: Wed, 1 Jun 2022 12:38:57 +0800 Subject: [PATCH 04/42] feat: show_enable param --- app.vue | 9 +++++++++ components/Tweet.vue | 1 + function.ts | 18 ++++++++++++++++++ interface.ts | 1 + pages/create.vue | 4 +--- server/_lib/parserv2.ts | 8 +++++--- server/api/v2/tweet.ts | 5 +++-- 7 files changed, 38 insertions(+), 8 deletions(-) diff --git a/app.vue b/app.vue index 2329292..4322b59 100644 --- a/app.vue +++ b/app.vue @@ -116,6 +116,15 @@ useHead({ margin: 0 0.05em 0 0.1em; vertical-align: -0.1em; } +.tweet-media { + margin-top: 1rem; + border: 1px solid var(--border); + border-radius: 1rem; + overflow: hidden; +} +.tweet-image { + width: 100%; +} [data-style="supabase"] { width: 400px; diff --git a/components/Tweet.vue b/components/Tweet.vue index a8858b1..dc02575 100644 --- a/components/Tweet.vue +++ b/components/Tweet.vue @@ -5,6 +5,7 @@ const props = defineProps({ css: { type: String, default: "" }, show_original_link: { type: Boolean, default: false }, enable_twemoji: { type: Boolean, default: true }, + show_media: { type: Boolean, default: true }, redirect: { type: Boolean, default: true }, }) diff --git a/function.ts b/function.ts index 32db08e..6f94832 100644 --- a/function.ts +++ b/function.ts @@ -74,6 +74,15 @@ export const obtainCss = (tweetOptions: TweetOptions) => { width: 1.2em; margin: 0 0.05em 0 0.1em; vertical-align: -0.1em; + } + .tweet-media { + margin-top: 1rem; + border: 1px solid var(--border); + border-radius: 1rem; + overflow: hidden; + } + .tweet-image { + width: 100%; }` } else { style += ` @@ -125,6 +134,15 @@ export const obtainCss = (tweetOptions: TweetOptions) => { width: 1.2em; margin: 0 0.05em 0 0.1em; vertical-align: -0.1em; + } + .tweet-media { + margin-top: 1rem; + border: 1px solid var(--border); + border-radius: 1rem; + overflow: hidden; + } + .tweet-image { + width: 100%; }` } diff --git a/interface.ts b/interface.ts index 186409c..5489b10 100644 --- a/interface.ts +++ b/interface.ts @@ -17,6 +17,7 @@ export interface TweetOptions { css?: string show_original_link?: boolean enable_twemoji?: boolean + show_media?: boolean } export interface ExportOptions { diff --git a/pages/create.vue b/pages/create.vue index a9e5414..940a91e 100644 --- a/pages/create.vue +++ b/pages/create.vue @@ -108,10 +108,8 @@ useCustomHead("Tweetic | Create now!", "Create your own static tweets now!") - - Show Original Link - Enable Twemoji + Show Media
diff --git a/server/_lib/parserv2.ts b/server/_lib/parserv2.ts index 21a6d0a..5d22762 100644 --- a/server/_lib/parserv2.ts +++ b/server/_lib/parserv2.ts @@ -3,7 +3,7 @@ import { TweetOptions, TweetContent } from "~~/interface" import { mapClass } from "./reference" export const constructHtmlv2 = (data: TweetContent, options: TweetOptions) => { - const { meta, html: content } = data + const { meta, html: content, media_html } = data const mapClassOptions = (key: string) => mapClass(key, options) const html = ` @@ -40,6 +40,8 @@ export const constructHtmlv2 = (data: TweetContent, options: TweetOptions) => {
${content} + + ${options.show_media && media_html ? media_html : ""}
` @@ -153,7 +155,7 @@ export const getTweetContent = (data: { [key: string]: string }, options: TweetO const width = img.attr("width") this.attribs = { - "data-type": "media-image", + class: "tweet-image", src: `${url}?format=${format}`, height, width, @@ -164,7 +166,7 @@ export const getTweetContent = (data: { [key: string]: string }, options: TweetO // Move the media img to a new container media.append(img) }) - media.attr("data-type", `image-container ${images.length}`) + media.attr("class", "tweet-media") mediaHtml = $("
").append(media).html() } } diff --git a/server/api/v2/tweet.ts b/server/api/v2/tweet.ts index dd904e7..a222d87 100644 --- a/server/api/v2/tweet.ts +++ b/server/api/v2/tweet.ts @@ -2,18 +2,19 @@ import { TweetOptions } from "~~/interface" import { constructHtmlv2, getTweetContent } from "../../_lib/parserv2" export default defineEventHandler(async (event) => { - const { url, layout, css, enable_twemoji } = useQuery(event) + const { url, layout, css, enable_twemoji, show_media } = useQuery(event) const id = url.toString().split("/")[5] const options: TweetOptions = { layout: layout?.toString(), css: css?.toString(), enable_twemoji: enable_twemoji ? JSON.parse(enable_twemoji.toString()) : false, + show_media: show_media ? JSON.parse(show_media.toString()) : false, } const data = await $fetch<{ [key: string]: string }>(`https://syndication.twitter.com/tweets.json?ids=${id}`) const tweetContent = getTweetContent(data, options) const html = constructHtmlv2(tweetContent, options) - return { html, meta: tweetContent.meta, data } + return { html, meta: tweetContent.meta, media: tweetContent?.media_html, data } }) From 421ed5034b6ceeb636b228af8877a6b2460c38b2 Mon Sep 17 00:00:00 2001 From: zernonia Date: Wed, 1 Jun 2022 13:15:32 +0800 Subject: [PATCH 05/42] feat: v2 api docs --- components/Tweet.vue | 3 +- components/TweetOld.vue | 60 ++++++++++++++++++++++++++++ components/docs/DocTweet.vue | 68 ++++++++++++++++++++++++++++++++ components/docs/DocTweetOld.vue | 70 +++++++++++++++++++++++++++++++++ pages/docs.vue | 70 ++------------------------------- 5 files changed, 202 insertions(+), 69 deletions(-) create mode 100644 components/TweetOld.vue create mode 100644 components/docs/DocTweet.vue create mode 100644 components/docs/DocTweetOld.vue diff --git a/components/Tweet.vue b/components/Tweet.vue index dc02575..7964adb 100644 --- a/components/Tweet.vue +++ b/components/Tweet.vue @@ -3,9 +3,8 @@ const props = defineProps({ url: String, layout: { type: String, default: "" }, css: { type: String, default: "" }, - show_original_link: { type: Boolean, default: false }, enable_twemoji: { type: Boolean, default: true }, - show_media: { type: Boolean, default: true }, + show_media: { type: Boolean, default: false }, redirect: { type: Boolean, default: true }, }) diff --git a/components/TweetOld.vue b/components/TweetOld.vue new file mode 100644 index 0000000..f1f2710 --- /dev/null +++ b/components/TweetOld.vue @@ -0,0 +1,60 @@ + + + diff --git a/components/docs/DocTweet.vue b/components/docs/DocTweet.vue new file mode 100644 index 0000000..eb97962 --- /dev/null +++ b/components/docs/DocTweet.vue @@ -0,0 +1,68 @@ + + + diff --git a/components/docs/DocTweetOld.vue b/components/docs/DocTweetOld.vue new file mode 100644 index 0000000..221e5f0 --- /dev/null +++ b/components/docs/DocTweetOld.vue @@ -0,0 +1,70 @@ + + + diff --git a/pages/docs.vue b/pages/docs.vue index 2f7897b..e4a7a78 100644 --- a/pages/docs.vue +++ b/pages/docs.vue @@ -1,70 +1,6 @@ - - From d118c824725f75d9fee378ac18fbe433c9289f62 Mon Sep 17 00:00:00 2001 From: zernonia Date: Wed, 1 Jun 2022 13:42:27 +0800 Subject: [PATCH 06/42] update: wrap component into a base template --- components/docs/Base.vue | 39 ++++++++++++++++++ components/docs/DocTweet.vue | 70 +++++++++++++------------------- components/docs/DocTweetOld.vue | 72 +++++++++++++-------------------- server/api/v2/tweet.ts | 2 +- 4 files changed, 94 insertions(+), 89 deletions(-) create mode 100644 components/docs/Base.vue diff --git a/components/docs/Base.vue b/components/docs/Base.vue new file mode 100644 index 0000000..30a6607 --- /dev/null +++ b/components/docs/Base.vue @@ -0,0 +1,39 @@ + + + diff --git a/components/docs/DocTweet.vue b/components/docs/DocTweet.vue index eb97962..208749e 100644 --- a/components/docs/DocTweet.vue +++ b/components/docs/DocTweet.vue @@ -18,51 +18,35 @@ const highlightResponse = computed(() => diff --git a/components/docs/DocTweetOld.vue b/components/docs/DocTweetOld.vue index 221e5f0..2fb241e 100644 --- a/components/docs/DocTweetOld.vue +++ b/components/docs/DocTweetOld.vue @@ -18,53 +18,35 @@ const highlightResponse = computed(() => diff --git a/server/api/v2/tweet.ts b/server/api/v2/tweet.ts index a222d87..78b6db2 100644 --- a/server/api/v2/tweet.ts +++ b/server/api/v2/tweet.ts @@ -16,5 +16,5 @@ export default defineEventHandler(async (event) => { const tweetContent = getTweetContent(data, options) const html = constructHtmlv2(tweetContent, options) - return { html, meta: tweetContent.meta, media: tweetContent?.media_html, data } + return { html, meta: tweetContent.meta } }) From 9c78d9917ccf652d7fc29a5f16f7a2d28bc7903d Mon Sep 17 00:00:00 2001 From: zernonia Date: Wed, 1 Jun 2022 13:45:01 +0800 Subject: [PATCH 07/42] fix: add try catch to api --- server/api/v2/tweet.ts | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/server/api/v2/tweet.ts b/server/api/v2/tweet.ts index 78b6db2..de12fa0 100644 --- a/server/api/v2/tweet.ts +++ b/server/api/v2/tweet.ts @@ -5,16 +5,21 @@ export default defineEventHandler(async (event) => { const { url, layout, css, enable_twemoji, show_media } = useQuery(event) const id = url.toString().split("/")[5] - const options: TweetOptions = { - layout: layout?.toString(), - css: css?.toString(), - enable_twemoji: enable_twemoji ? JSON.parse(enable_twemoji.toString()) : false, - show_media: show_media ? JSON.parse(show_media.toString()) : false, - } + try { + const options: TweetOptions = { + layout: layout?.toString(), + css: css?.toString(), + enable_twemoji: enable_twemoji ? JSON.parse(enable_twemoji.toString()) : false, + show_media: show_media ? JSON.parse(show_media.toString()) : false, + } - const data = await $fetch<{ [key: string]: string }>(`https://syndication.twitter.com/tweets.json?ids=${id}`) + const data = await $fetch<{ [key: string]: string }>(`https://syndication.twitter.com/tweets.json?ids=${id}`) - const tweetContent = getTweetContent(data, options) - const html = constructHtmlv2(tweetContent, options) - return { html, meta: tweetContent.meta } + const tweetContent = getTweetContent(data, options) + const html = constructHtmlv2(tweetContent, options) + return { html, meta: tweetContent.meta } + } catch (err) { + event.res.statusCode = 400 + return { err } + } }) From 26cd094ffcc6c351594ccc26b5fdfb551234133e Mon Sep 17 00:00:00 2001 From: zernonia Date: Wed, 1 Jun 2022 17:40:23 +0800 Subject: [PATCH 08/42] fix: bundling error cause of dependencies version mismatch --- package.json | 1 - yarn.lock | 15 +-------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/package.json b/package.json index d73b138..2046fff 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,6 @@ "cheerio": "^1.0.0-rc.11", "highlight.js": "^11.5.1", "lodash-es": "^4.17.21", - "node-html-parser": "^5.3.3", "twemoji": "^14.0.2", "twitter-api-v2": "^1.12.0", "vue-toastification": "^2.0.0-rc.5" diff --git a/yarn.lock b/yarn.lock index 26d1ae6..c67ac76 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1511,7 +1511,7 @@ css-declaration-sorter@^6.2.2: resolved "https://registry.yarnpkg.com/css-declaration-sorter/-/css-declaration-sorter-6.2.2.tgz#bfd2f6f50002d6a3ae779a87d3a0c5d5b10e0f02" integrity sha512-Ufadglr88ZLsrvS11gjeu/40Lw74D9Am/Jpr3LlYm5Q4ZP5KdlUhG+6u2EjyXeZcxmZ2h1ebCKngDjolpeLHpg== -css-select@^4.1.3, css-select@^4.2.1, css-select@^4.3.0: +css-select@^4.1.3, css-select@^4.3.0: version "4.3.0" resolved "https://registry.yarnpkg.com/css-select/-/css-select-4.3.0.tgz#db7129b2846662fd8628cfc496abb2b59e41529b" integrity sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ== @@ -2442,11 +2442,6 @@ hash-sum@^2.0.0: resolved "https://registry.yarnpkg.com/hash-sum/-/hash-sum-2.0.0.tgz#81d01bb5de8ea4a214ad5d6ead1b523460b0b45a" integrity sha512-WdZTbAByD+pHfl/g9QSsBIIwy8IT+EsPiKDs0KNX+zSHhdDLFKdZu0BQHljvO+0QI/BasbMSUa8wYNCZTvhslg== -he@1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" - integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== - highlight.js@^11.5.1: version "11.5.1" resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-11.5.1.tgz#027c24e4509e2f4dcd00b4a6dda542ce0a1f7aea" @@ -3241,14 +3236,6 @@ node-gyp-build@^4.2.2, node-gyp-build@^4.3.0: resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.4.0.tgz#42e99687ce87ddeaf3a10b99dc06abc11021f3f4" integrity sha512-amJnQCcgtRVw9SvoebO3BKGESClrfXGCUTX9hSn1OuGQTQBOZmVd0Z0OlecpuRksKvbsUqALE8jls/ErClAPuQ== -node-html-parser@^5.3.3: - version "5.3.3" - resolved "https://registry.yarnpkg.com/node-html-parser/-/node-html-parser-5.3.3.tgz#2845704f3a7331a610e0e551bf5fa02b266341b6" - integrity sha512-ncg1033CaX9UexbyA7e1N0aAoAYRDiV8jkTvzEnfd1GDvzFdrsXLzR4p4ik8mwLgnaKP/jyUFWDy9q3jvRT2Jw== - dependencies: - css-select "^4.2.1" - he "1.2.0" - node-pre-gyp@^0.13.0: version "0.13.0" resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.13.0.tgz#df9ab7b68dd6498137717838e4f92a33fc9daa42" From b226e140cc7c55c7c36a9842ef9b39ef828aa67b Mon Sep 17 00:00:00 2001 From: zernonia Date: Wed, 1 Jun 2022 17:56:36 +0800 Subject: [PATCH 09/42] feat: deprecate old api --- components/Tweet.vue | 2 +- components/TweetOld.vue | 60 ------- components/docs/DocTweet.vue | 2 +- components/docs/DocTweetOld.vue | 52 ------ server/_lib/parser.ts | 289 +++++++++++++++++++++++--------- server/_lib/parserv2.ts | 247 --------------------------- server/api/tweet.ts | 22 +-- server/api/v2/tweet.ts | 25 --- 8 files changed, 226 insertions(+), 473 deletions(-) delete mode 100644 components/TweetOld.vue delete mode 100644 components/docs/DocTweetOld.vue delete mode 100644 server/_lib/parserv2.ts delete mode 100644 server/api/v2/tweet.ts diff --git a/components/Tweet.vue b/components/Tweet.vue index 7964adb..455f683 100644 --- a/components/Tweet.vue +++ b/components/Tweet.vue @@ -12,7 +12,7 @@ const props = defineProps({ const { data, pending } = await useAsyncData( JSON.stringify(props), () => - $fetch("/api/v2/tweet", { + $fetch("/api/tweet", { params: { ...props }, }), { diff --git a/components/TweetOld.vue b/components/TweetOld.vue deleted file mode 100644 index f1f2710..0000000 --- a/components/TweetOld.vue +++ /dev/null @@ -1,60 +0,0 @@ - - - diff --git a/components/docs/DocTweet.vue b/components/docs/DocTweet.vue index 208749e..5db70cb 100644 --- a/components/docs/DocTweet.vue +++ b/components/docs/DocTweet.vue @@ -18,7 +18,7 @@ const highlightResponse = computed(() =>