From ba1b3afa6f9f60b48043fb36eeba57fa47a082d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B3=E5=85=83=E7=9A=93?= Date: Tue, 20 May 2025 14:49:50 +0800 Subject: [PATCH] Make a poc for the api for line today & made some docs for myself and others so others also can use the api --- about/scraping_line_today_home.md | 68 ++++++++++++++++++++++++++ bun.lock | 5 ++ components/app/popup.vue | 59 ++++++++++++++++++++++ components/app/windows/chatbot.vue | 7 +-- components/app/windows/fav.vue | 7 ++- package.json | 1 + pages/desktop.vue | 4 +- pages/home.vue | 2 +- pages/tools/index.vue | 10 +++- public/datainfo/linetodayjsondata.json | 0 server/fetchapi/lt_home.ts | 23 +++++++++ server/fetchapi/run.txt | 35 +++++++++++++ 12 files changed, 212 insertions(+), 9 deletions(-) create mode 100644 about/scraping_line_today_home.md create mode 100644 components/app/popup.vue create mode 100644 public/datainfo/linetodayjsondata.json create mode 100644 server/fetchapi/lt_home.ts create mode 100644 server/fetchapi/run.txt diff --git a/about/scraping_line_today_home.md b/about/scraping_line_today_home.md new file mode 100644 index 0000000..319a1c9 --- /dev/null +++ b/about/scraping_line_today_home.md @@ -0,0 +1,68 @@ +# Scraping line today home + +This took me some time, but they use a fancy system for pulling news data. + +## Main endpoint +For local Taiwan news they use this url: https://today.line.me/_next/data/v1/tw/v3/tab/domestic.json?tabs=domestic + +From the _next? I thought that is static? I mean it maybe is, it is just providing with the URLs that the client will be fetching to the server, which is a bit fun. + +Here is a JSON snippet: +```json +{ + "id": "682b0cef1b1269f8dec93e60", + "type": "HIGHLIGHT", + "containerStyle": "Header", + "name": "國內話題：新北重大車禍", + "source": "LISTING", + "header": { + "title": "新北重大死傷車禍", + "hasCompositeTitle": false, + "subTitle": "一輛小客車19日下午撞上放學人群，造成多名學童、大人送醫，至少3死10多傷，肇事的78歲男子當場昏迷。" + }, + "listings": [ + { + "id": "1feef7d2-3acc-495d-becd-3ef4de6a92ce", + "offset": 0, + "length": 10 + } + ] +}, +``` + +We can ignore everything else, other than the strange UUID in the json. Well, this is the key we need to fetch their api 🤩 + +## Fetch news URLs + +Here is the fancy URL: +`https://today.line.me/api/v6/listings/{the-uuid-you-got-in-the-listings-json-file}/?country=tw&offset=0&length=24` + +This api can be used for fetching the news from them, however, there is an issue, the max length is only just 24 (yes, I tried it only can return 24 when requesting for 1000) + + +And viewing the JSON, oh would you look at that. +```JSON +{ + "id": "262862833", + "title": "派駐芬蘭遭白委扯焦慮症 林昶佐現身喊話", + "publisher": "太報", + "publisherId": "101366", + "publishTimeUnix": 1747670221000, + "contentType": "GENERAL", + "thumbnail": { + "type": "IMAGE", + "hash": "0hvoq7de5NKUBMTjcMHM5WF3QYJTF_KDNJbixudj4ddXJnYm4ecX16Iz0edWwydjsTbH9vdm5IJ3EyKjtBeA" + }, + "url": { + "hash": "8nlkYeV" + }, + "categoryId": 100262, + "categoryName": "國內", + "shortDescription": "前立委林昶佐（右二）將出任駐芬蘭代表，民眾黨立委林憶君卻質疑罹患焦慮症不適合去北歐。翻攝畫面前立委林昶佐將接任駐芬蘭代表，民眾黨立委林憶君今（5/19）質詢指出，林林昶佐曾患焦慮症，北歐國家日常短，病症容易發作，質疑是否適合。林昶佐晚間現身直播節目，向病友喊話，要對自己有信心，「絕對可以回復到正常生活，包括工作」。林憶君指出，1990年芬蘭是全球自殺率最高國家，而且北歐國家的日照很短，病症容易發作..." +}, +``` +The url hash is just what we needed to use my scraper :D + +You can query it by using: https://news.yuanhau.com/api/news/get/lt/8nlkYeV (Also videos are in the list, so avoid that) or just try this https://today.line.me/tw/v2/article/8nlkYeV + +and that's it, I've bypassed Line's attempt to block people like me. :) diff --git a/bun.lock b/bun.lock index 636836b..a42be3f 100644 --- a/bun.lock +++ b/bun.lock @@ -19,6 +19,7 @@ "@vueuse/core": "^13.1.0", "animate.css": "^4.1.1", "argon2": "^0.43.0", + "axios": "^1.9.0", "bootstrap-icons": "^1.12.1", "cheerio": "^1.0.0", "class-variance-authority": "^0.7.1", @@ -850,6 +851,8 @@ "autoprefixer": ["autoprefixer@10.4.21", "", { "dependencies": { "browserslist": "^4.24.4", "caniuse-lite": "^1.0.30001702", "fraction.js": "^4.3.7", "normalize-range": "^0.1.2", "picocolors": "^1.1.1", "postcss-value-parser": "^4.2.0" }, "peerDependencies": { "postcss": "^8.1.0" }, "bin": { "autoprefixer": "bin/autoprefixer" } }, "sha512-O+A6LWV5LDHSJD3LjHYoNi4VLsj/Whi7k6zG12xTYaU4cQ8oxQGckXNX8cRHK5yOZ/ppVHe0ZBXGzSV9jXdVbQ=="], + "axios": ["axios@1.9.0", "", { "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", "proxy-from-env": "^1.1.0" } }, "sha512-re4CqKTJaURpzbLHtIi6XpDv20/CnpXOtjRY5/CU32L8gU8ek9UIivcfvSWvmKEngmVbrUtPpdDwWDWL7DNHvg=="], + "b4a": ["b4a@1.6.7", "", {}, "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg=="], "balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], @@ -1262,6 +1265,8 @@ "fn.name": ["fn.name@1.1.0", "", {}, "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw=="], + "follow-redirects": ["follow-redirects@1.15.9", "", {}, "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ=="], + "foreground-child": ["foreground-child@3.3.1", "", { "dependencies": { "cross-spawn": "^7.0.6", "signal-exit": "^4.0.1" } }, "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw=="], "form-data": ["form-data@4.0.2", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "mime-types": "^2.1.12" } }, "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w=="], diff --git a/components/app/popup.vue b/components/app/popup.vue new file mode 100644 index 0000000..cc56320 --- /dev/null +++ b/components/app/popup.vue @@ -0,0 +1,59 @@ + + diff --git a/components/app/windows/chatbot.vue b/components/app/windows/chatbot.vue index 0ded4f0..cde4da0 100644 --- a/components/app/windows/chatbot.vue +++ b/components/app/windows/chatbot.vue @@ -79,7 +79,7 @@ onMounted(async () => {