From 0c4413f3769d22016eb0a4d442dcb73b4afe55b7 Mon Sep 17 00:00:00 2001 From: huqi Date: Sun, 29 Nov 2020 13:03:19 +0800 Subject: [PATCH] :white_check_mark: and :art: --- mod.ts | 20 ++-- toutiao-search.ts | 16 +-- types.ts | 14 +-- utils.test.ts | 255 +++++++++++++++++++++++++++++++++++++++++++-- utils.ts | 49 ++++++--- weibo-search.ts | 11 +- zhihu-questions.ts | 5 +- zhihu-search.ts | 9 +- zhihu-video.ts | 3 - 9 files changed, 321 insertions(+), 61 deletions(-) diff --git a/mod.ts b/mod.ts index 4522eca04a..0d4e75b0da 100644 --- a/mod.ts +++ b/mod.ts @@ -1,11 +1,11 @@ -import { zhihuVideo } from './zhihu-video.ts' -import { zhihuQuestions } from './zhihu-questions.ts' -import { zhihuSearch } from './zhihu-search.ts' -import { weiboSearch } from './weibo-search.ts' -import { toutiaoSearch } from './toutiao-search.ts' +import { zhihuVideo } from "./zhihu-video.ts"; +import { zhihuQuestions } from "./zhihu-questions.ts"; +import { zhihuSearch } from "./zhihu-search.ts"; +import { weiboSearch } from "./weibo-search.ts"; +import { toutiaoSearch } from "./toutiao-search.ts"; -zhihuVideo() -zhihuQuestions() -zhihuSearch() -weiboSearch() -toutiaoSearch() \ No newline at end of file +zhihuVideo(); +zhihuQuestions(); +zhihuSearch(); +weiboSearch(); +toutiaoSearch(); diff --git a/toutiao-search.ts b/toutiao-search.ts index e2858e9ce9..e6b6429936 100644 --- a/toutiao-search.ts +++ b/toutiao-search.ts @@ -5,9 +5,15 @@ import { join } from "std/path/mod.ts"; import { exists } from "std/fs/mod.ts"; import type { ToutiaoTopSearch, ToutiaoWord } from "./types.ts"; -import { createArchive4Toutiao, createReadme4Toutiao, mergeWords4Toutiao } from "./utils.ts"; +import { + createArchive4Toutiao, + createReadme4Toutiao, + mergeWords4Toutiao, +} from "./utils.ts"; -const response = await fetch("https://is-lq.snssdk.com/api/suggest_words/?business_id=10016"); +const response = await fetch( + "https://is-lq.snssdk.com/api/suggest_words/?business_id=10016", +); if (!response.ok) { console.error(response.statusText); @@ -26,11 +32,10 @@ if (await exists(fullPath)) { wordsAlreadyDownload = JSON.parse(content); } - const wordsAll = mergeWords4Toutiao(words, wordsAlreadyDownload); -export const ToutiaoSearchData = wordsAll.map(x=> { - x.url = `https://so.toutiao.com/search?keyword=${x.word.replace(' ','+')}`; +export const ToutiaoSearchData = wordsAll.map((x) => { + x.url = `https://so.toutiao.com/search?keyword=${x.word.replace(" ", "+")}`; return x; }); @@ -46,5 +51,4 @@ export async function toutiaoSearch() { const archiveText = createArchive4Toutiao(wordsAll, yyyyMMdd); const archivePath = join("archives/toutiao-search", `${yyyyMMdd}.md`); await Deno.writeTextFile(archivePath, archiveText); - } diff --git a/types.ts b/types.ts index 2ce561dd99..38099bb751 100644 --- a/types.ts +++ b/types.ts @@ -32,7 +32,7 @@ export type ZhihuQuestionList = { export type SearchWord = { query: string; display_query: string; - url?: string + url?: string; }; export type TopSearch = { @@ -50,12 +50,12 @@ export type Word = { export type ToutiaoTopSearch = { data: [ { - words: ToutiaoWord[] - } - ] -} + words: ToutiaoWord[]; + }, + ]; +}; export type ToutiaoWord = { word: string; - url?: string -}; \ No newline at end of file + url: string; +}; diff --git a/utils.test.ts b/utils.test.ts index feaa048419..6a4e79f4a5 100644 --- a/utils.test.ts +++ b/utils.test.ts @@ -1,14 +1,99 @@ #!/usr/bin/env -S deno run --unstable --allow-net --allow-read --allow-write --import-map=import_map.json import { assertEquals, assertStringIncludes } from "std/testing/asserts.ts"; -import type { Question } from "./types.ts"; +import type { Question, SearchWord, Word, ToutiaoWord } from "./types.ts"; import { createArchive, - createList, + createTuotiaoList, createReadme, + mergeWords, mergeQuestions, + mergeWords4Toutiao, + mergeWords4Weibo, + createWeiboList, + createQuestionList, + createReadme4Toutiao, + createArchive4Toutiao, + createArchive4Weibo, + createReadme4Weibo, + createReadme4Search, + createArchive4Search, + createSearchList, } from "./utils.ts"; +Deno.test("mergeWords4Toutiao", function (): void { + const words1: ToutiaoWord[] = []; + const words2: ToutiaoWord[] = [{ word: "foo", url: "bar" }]; + const words3: ToutiaoWord[] = [{ word: "foo", url: "hello" }]; + const words4: ToutiaoWord[] = [{ word: "hello", url: "world" }]; + const words5: ToutiaoWord[] = [ + { word: "foo", url: "bar" }, + { word: "hello", url: "world" }, + ]; + + assertEquals(mergeWords4Toutiao(words1, words2), words2); + assertEquals(mergeWords4Toutiao(words1, words5), words5); + assertEquals(mergeWords4Toutiao(words2, words2), words2); + assertEquals( + mergeWords4Toutiao(words2, words3), + [ + { word: "foo", url: "bar" }, + { word: "foo", url: "hello" }, + ], + ); + assertEquals(mergeWords4Toutiao(words4, words5), [ + { word: "hello", url: "world" }, + { word: "foo", url: "bar" }, + ]); + assertEquals( + mergeWords4Toutiao(words3, words5), + [ + { word: "foo", url: "hello" }, + { word: "foo", url: "bar" }, + { word: "hello", url: "world" }, + ], + ); +}); + +Deno.test("createTuotiaoList", function (): void { + const words: ToutiaoWord[] = [ + { word: "foo", url: "bar" }, + { word: "hello", url: "world" }, + ]; + + assertStringIncludes(createTuotiaoList(words), ""); + assertStringIncludes(createTuotiaoList(words), ""); + assertStringIncludes(createTuotiaoList(words), "foo"); + assertStringIncludes(createTuotiaoList(words), "world"); + assertStringIncludes(createTuotiaoList(words), "hello"); +}); + +Deno.test("createArchive4Toutiao", function (): void { + const words: ToutiaoWord[] = [ + { word: "foo", url: "bar" }, + { word: "hello", url: "world" }, + ]; + + assertStringIncludes( + createArchive4Toutiao(words, "2020-02-02"), + "# 2020-02-02", + ); + assertStringIncludes(createArchive4Toutiao(words, "2020-02-02"), "共 2 条"); +}); + +Deno.test("createReadme4Toutiao", async function (): Promise { + const words: ToutiaoWord[] = [ + { word: "foo", url: "bar" }, + { word: "hello", url: "world" }, + ]; + + assertStringIncludes(await createReadme4Toutiao(words), "头条"); + assertStringIncludes( + await createReadme4Toutiao(words), + "zhihu-trending-hot-questions", + ); +}); + Deno.test("mergeQuestions", function (): void { const words1: Question[] = []; const words2: Question[] = [{ title: "foo", url: "bar" }]; @@ -43,17 +128,23 @@ Deno.test("mergeQuestions", function (): void { ); }); -Deno.test("createList", function (): void { +Deno.test("createQuestionList", function (): void { const words: Question[] = [ { title: "foo", url: "bar" }, { title: "hello", url: "world" }, ]; - assertStringIncludes(createList(words), ""); - assertStringIncludes(createList(words), ""); - assertStringIncludes(createList(words), "foo"); - assertStringIncludes(createList(words), "world"); - assertStringIncludes(createList(words), "hello"); + assertStringIncludes( + createQuestionList(words), + "", + ); + assertStringIncludes( + createQuestionList(words), + "", + ); + assertStringIncludes(createQuestionList(words), "foo"); + assertStringIncludes(createQuestionList(words), "world"); + assertStringIncludes(createQuestionList(words), "hello"); }); Deno.test("createArchive", function (): void { @@ -72,9 +163,155 @@ Deno.test("createReadme", async function (): Promise { { title: "hello", url: "world" }, ]; - assertStringIncludes(await createReadme(words), "知乎"); + assertStringIncludes(await createReadme(words), "热门"); assertStringIncludes( await createReadme(words), "zhihu-trending-hot-questions", ); }); + +Deno.test("mergeWords4Weibo", function (): void { + const words1: Word[] = []; + const words2: Word[] = [{ title: "foo", url: "bar" }]; + const words3: Word[] = [{ title: "foo", url: "hello" }]; + const words4: Word[] = [{ title: "hello", url: "world" }]; + const words5: Word[] = [ + { title: "foo", url: "bar" }, + { title: "hello", url: "world" }, + ]; + + assertEquals(mergeWords4Weibo(words1, words2), words2); + assertEquals(mergeWords4Weibo(words1, words5), words5); + assertEquals(mergeWords4Weibo(words2, words2), words2); + assertEquals( + mergeWords4Weibo(words2, words3), + [ + { title: "foo", url: "bar" }, + { title: "foo", url: "hello" }, + ], + ); + assertEquals(mergeWords4Weibo(words4, words5), [ + { title: "hello", url: "world" }, + { title: "foo", url: "bar" }, + ]); + assertEquals( + mergeWords4Weibo(words3, words5), + [ + { title: "foo", url: "hello" }, + { title: "foo", url: "bar" }, + { title: "hello", url: "world" }, + ], + ); +}); + +Deno.test("createWeiboList", function (): void { + const words: Word[] = [ + { title: "foo", url: "bar" }, + { title: "hello", url: "world" }, + ]; + + assertStringIncludes(createWeiboList(words), ""); + assertStringIncludes(createWeiboList(words), ""); + assertStringIncludes(createWeiboList(words), "foo"); + assertStringIncludes(createWeiboList(words), "world"); + assertStringIncludes(createWeiboList(words), "hello"); +}); + +Deno.test("createArchive4Weibo", function (): void { + const words: Word[] = [ + { title: "foo", url: "bar" }, + { title: "hello", url: "world" }, + ]; + + assertStringIncludes( + createArchive4Weibo(words, "2020-02-02"), + "# 2020-02-02", + ); + assertStringIncludes(createArchive4Weibo(words, "2020-02-02"), "共 2 条"); +}); + +Deno.test("createReadme4Weibo", async function (): Promise { + const words: Word[] = [ + { title: "foo", url: "bar" }, + { title: "hello", url: "world" }, + ]; + + assertStringIncludes(await createReadme4Weibo(words), "微博"); + assertStringIncludes( + await createReadme4Weibo(words), + "zhihu-trending-hot-questions", + ); +}); + +Deno.test("mergeWords", function (): void { + const words1: SearchWord[] = []; + const words2: SearchWord[] = [{ query: "foo", display_query: "bar" }]; + const words3: SearchWord[] = [{ query: "foo", display_query: "hello" }]; + const words4: SearchWord[] = [{ query: "hello", display_query: "world" }]; + const words5: SearchWord[] = [ + { query: "foo", display_query: "bar" }, + { query: "hello", display_query: "world" }, + ]; + + assertEquals(mergeWords(words1, words2), words2); + assertEquals(mergeWords(words1, words5), words5); + assertEquals(mergeWords(words2, words2), words2); + assertEquals( + mergeWords(words2, words3), + [ + { query: "foo", display_query: "bar" }, + { query: "foo", display_query: "hello" }, + ], + ); + assertEquals(mergeWords(words4, words5), [ + { query: "hello", display_query: "world" }, + { query: "foo", display_query: "bar" }, + ]); + assertEquals( + mergeWords(words3, words5), + [ + { query: "foo", display_query: "hello" }, + { query: "foo", display_query: "bar" }, + { query: "hello", display_query: "world" }, + ], + ); +}); + +Deno.test("createSearchList", function (): void { + const words: SearchWord[] = [ + { query: "foo", display_query: "bar" }, + { query: "hello", display_query: "world" }, + ]; + + assertStringIncludes(createSearchList(words), ""); + assertStringIncludes(createSearchList(words), ""); + assertStringIncludes(createSearchList(words), "foo"); + assertStringIncludes(createSearchList(words), "world"); + assertStringIncludes(createSearchList(words), "hello"); +}); + +Deno.test("createArchive4Search", function (): void { + const words: SearchWord[] = [ + { query: "foo", display_query: "bar" }, + { query: "hello", display_query: "world" }, + ]; + + assertStringIncludes( + createArchive4Search(words, "2020-02-02"), + "# 2020-02-02", + ); + assertStringIncludes(createArchive4Search(words, "2020-02-02"), "共 2 条"); +}); + +Deno.test("createReadme4Search", async function (): Promise { + const words: SearchWord[] = [ + { query: "foo", display_query: "bar" }, + { query: "hello", display_query: "world" }, + ]; + + assertStringIncludes(await createReadme4Search(words), "热搜"); + assertStringIncludes( + await createReadme4Search(words), + "zhihu-trending-hot-questions", + ); +}); diff --git a/utils.ts b/utils.ts index d886f15e00..c0e7dec497 100644 --- a/utils.ts +++ b/utils.ts @@ -37,7 +37,7 @@ export function mergeWords4Toutiao( ): ToutiaoWord[] { const obj: Record = {}; for (const w of words.concat(another)) { - obj[w.word] = w.word; + obj[w.url] = w.word; } return Object.entries(obj).map(([url, word]) => ({ url, @@ -62,24 +62,42 @@ export function mergeWords4Weibo( export async function createReadme(words: Question[]): Promise { const readme = await Deno.readTextFile("./README.md"); - return readme.replace(/[\W\w]*/, createVideoList(words)) - .replace(/[\W\w]*/, createQuestionList(words)); - + return readme.replace( + /[\W\w]*/, + createVideoList(words), + ) + .replace( + /[\W\w]*/, + createQuestionList(words), + ); } -export async function createReadme4Search(words: SearchWord[]): Promise { +export async function createReadme4Search( + words: SearchWord[], +): Promise { const readme = await Deno.readTextFile("./README.md"); - return readme.replace(/[\W\w]*/, createSearchList(words)); + return readme.replace( + /[\W\w]*/, + createSearchList(words), + ); } export async function createReadme4Weibo(words: Word[]): Promise { const readme = await Deno.readTextFile("./README.md"); - return readme.replace(/[\W\w]*/, createWeiboList(words)); + return readme.replace( + /[\W\w]*/, + createWeiboList(words), + ); } -export async function createReadme4Toutiao(words: ToutiaoWord[]): Promise { +export async function createReadme4Toutiao( + words: ToutiaoWord[], +): Promise { const readme = await Deno.readTextFile("./README.md"); - return readme.replace(/[\W\w]*/, createTuotiaoList(words)); + return readme.replace( + /[\W\w]*/, + createTuotiaoList(words), + ); } export function createVideoList(words: Question[]): string { @@ -133,8 +151,6 @@ ${ `; } - - export function createArchive(words: Question[], date: string): string { return `# ${date}\n 共 ${words.length} 条\n @@ -142,7 +158,10 @@ ${createVideoList(words)} `; } -export function createArchive4Search(words: SearchWord[], date: string): string { +export function createArchive4Search( + words: SearchWord[], + date: string, +): string { return `# ${date}\n 共 ${words.length} 条\n ${createSearchList(words)} @@ -156,10 +175,12 @@ ${createWeiboList(words)} `; } -export function createArchive4Toutiao(words: ToutiaoWord[], date: string): string { +export function createArchive4Toutiao( + words: ToutiaoWord[], + date: string, +): string { return `# ${date}\n 共 ${words.length} 条\n ${createTuotiaoList(words)} `; } - diff --git a/weibo-search.ts b/weibo-search.ts index a50c3726a2..898062c29b 100644 --- a/weibo-search.ts +++ b/weibo-search.ts @@ -5,7 +5,11 @@ import { join } from "std/path/mod.ts"; import { exists } from "std/fs/mod.ts"; import type { Word } from "./types.ts"; -import { createArchive4Weibo, createReadme4Weibo, mergeWords4Weibo } from "./utils.ts"; +import { + createArchive4Weibo, + createReadme4Weibo, + mergeWords4Weibo, +} from "./utils.ts"; const regexp = /(.+)<\/a>/g; @@ -36,12 +40,11 @@ if (await exists(fullPath)) { const queswordsAll = mergeWords4Weibo(words, wordsAlreadyDownload); -export const weiboSearchData = queswordsAll.map(x=> { +export const weiboSearchData = queswordsAll.map((x) => { x.realurl = `https://s.weibo.com/${x.url}`; return x; }); - export async function weiboSearch() { // 保存原始数据 await Deno.writeTextFile(fullPath, JSON.stringify(queswordsAll)); @@ -54,6 +57,4 @@ export async function weiboSearch() { const archiveText = createArchive4Weibo(queswordsAll, yyyyMMdd); const archivePath = join("archives/weibo-search", `${yyyyMMdd}.md`); await Deno.writeTextFile(archivePath, archiveText); - } - diff --git a/zhihu-questions.ts b/zhihu-questions.ts index a2e64d457e..fae226374b 100755 --- a/zhihu-questions.ts +++ b/zhihu-questions.ts @@ -8,7 +8,6 @@ import { exists } from "std/fs/mod.ts"; import type { ZhihuQuestionList, Question } from "./types.ts"; import { createArchive, createReadme, mergeQuestions } from "./utils.ts"; - const response = await fetch( "https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total?limit=100", ); @@ -39,7 +38,6 @@ const questionsAll = mergeQuestions(questions, questionsAlreadyDownload); export const zhihuQuestionData = questionsAll; export async function zhihuQuestions() { - // 保存原始数据 await Deno.writeTextFile(fullPath, JSON.stringify(questionsAll)); @@ -51,5 +49,4 @@ export async function zhihuQuestions() { const archiveText = createArchive(questionsAll, yyyyMMdd); const archivePath = join("archives/zhihu-questions", `${yyyyMMdd}.md`); await Deno.writeTextFile(archivePath, archiveText); - -} \ No newline at end of file +} diff --git a/zhihu-search.ts b/zhihu-search.ts index 41849c1880..2de24f6bd7 100755 --- a/zhihu-search.ts +++ b/zhihu-search.ts @@ -5,7 +5,11 @@ import { join } from "std/path/mod.ts"; import { exists } from "std/fs/mod.ts"; import type { SearchWord, TopSearch } from "./types.ts"; -import { createArchive4Search, createReadme4Search, mergeWords } from "./utils.ts"; +import { + createArchive4Search, + createReadme4Search, + mergeWords, +} from "./utils.ts"; const response = await fetch("https://www.zhihu.com/api/v4/search/top_search"); @@ -28,7 +32,7 @@ if (await exists(fullPath)) { const wordsAll = mergeWords(words, wordsAlreadyDownload); -export const zhihuSearchData = wordsAll.map(x=> { +export const zhihuSearchData = wordsAll.map((x) => { x.url = `https://www.zhihu.com/search?q=${x.query}`; return x; }); @@ -45,5 +49,4 @@ export async function zhihuSearch() { const archiveText = createArchive4Search(wordsAll, yyyyMMdd); const archivePath = join("archives/zhihu-search", `${yyyyMMdd}.md`); await Deno.writeTextFile(archivePath, archiveText); - } diff --git a/zhihu-video.ts b/zhihu-video.ts index c32b6f6d49..ccafa53889 100755 --- a/zhihu-video.ts +++ b/zhihu-video.ts @@ -42,7 +42,6 @@ const questionsAll = mergeQuestions(questions, questionsAlreadyDownload); export const zhihuVideoData = questionsAll; export async function zhihuVideo() { - // 保存原始数据 await Deno.writeTextFile(fullPath, JSON.stringify(questionsAll)); @@ -54,6 +53,4 @@ export async function zhihuVideo() { const archiveText = createArchive(questionsAll, yyyyMMdd); const archivePath = join("archives/zhihu-video", `${yyyyMMdd}.md`); await Deno.writeTextFile(archivePath, archiveText); - } -