Data/trending-in-one
1
0
Fork 0
mirror of https://github.com/huqi-pr/trending-in-one.git synced 2024-11-17 13:58:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

✅ and 🎨

Browse Source
This commit is contained in:
huqi 2020-11-29 13:03:19 +08:00
parent 08b5a73e2b
commit 0c4413f376
9 changed files with 321 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
mod.ts
View File

@ -1,11 +1,11 @@
import { zhihuVideo } from './zhihu-video.ts'
import { zhihuQuestions } from './zhihu-questions.ts'
import { zhihuSearch } from './zhihu-search.ts'
import { weiboSearch } from './weibo-search.ts'
import { toutiaoSearch } from './toutiao-search.ts'
import { zhihuVideo } from "./zhihu-video.ts";
import { zhihuQuestions } from "./zhihu-questions.ts";
import { zhihuSearch } from "./zhihu-search.ts";
import { weiboSearch } from "./weibo-search.ts";
import { toutiaoSearch } from "./toutiao-search.ts";
zhihuVideo()
zhihuQuestions()
zhihuSearch()
weiboSearch()
toutiaoSearch()
zhihuVideo();
zhihuQuestions();
zhihuSearch();
weiboSearch();
toutiaoSearch();

16
toutiao-search.ts
View File

@ -5,9 +5,15 @@ import { join } from "std/path/mod.ts";
import { exists } from "std/fs/mod.ts";
import type { ToutiaoTopSearch, ToutiaoWord } from "./types.ts";
import { createArchive4Toutiao, createReadme4Toutiao, mergeWords4Toutiao } from "./utils.ts";
import {
createArchive4Toutiao,
createReadme4Toutiao,
mergeWords4Toutiao,
} from "./utils.ts";
const response = await fetch("https://is-lq.snssdk.com/api/suggest_words/?business_id=10016");
const response = await fetch(
"https://is-lq.snssdk.com/api/suggest_words/?business_id=10016",
);
if (!response.ok) {
console.error(response.statusText);
@ -26,11 +32,10 @@ if (await exists(fullPath)) {
wordsAlreadyDownload = JSON.parse(content);
}
const wordsAll = mergeWords4Toutiao(words, wordsAlreadyDownload);
export const ToutiaoSearchData = wordsAll.map(x=> {
x.url = `https://so.toutiao.com/search?keyword=${x.word.replace(' ','+')}`;
export const ToutiaoSearchData = wordsAll.map((x) => {
x.url = `https://so.toutiao.com/search?keyword=${x.word.replace(" ", "+")}`;
return x;
});
@ -46,5 +51,4 @@ export async function toutiaoSearch() {
const archiveText = createArchive4Toutiao(wordsAll, yyyyMMdd);
const archivePath = join("archives/toutiao-search", `${yyyyMMdd}.md`);
await Deno.writeTextFile(archivePath, archiveText);
}

12
types.ts
View File

@ -32,7 +32,7 @@ export type ZhihuQuestionList = {
export type SearchWord = {
query: string;
display_query: string;
url?: string
url?: string;
};
export type TopSearch = {
@ -50,12 +50,12 @@ export type Word = {
export type ToutiaoTopSearch = {
data: [
{
words: ToutiaoWord[]
}
]
}
words: ToutiaoWord[];
},
];
};
export type ToutiaoWord = {
word: string;
url?: string
url: string;
};

255
utils.test.ts
View File

@ -1,14 +1,99 @@
#!/usr/bin/env -S deno run --unstable --allow-net --allow-read --allow-write --import-map=import_map.json
import { assertEquals, assertStringIncludes } from "std/testing/asserts.ts";
import type { Question } from "./types.ts";
import type { Question, SearchWord, Word, ToutiaoWord } from "./types.ts";
import {
createArchive,
createList,
createTuotiaoList,
createReadme,
mergeWords,
mergeQuestions,
mergeWords4Toutiao,
mergeWords4Weibo,
createWeiboList,
createQuestionList,
createReadme4Toutiao,
createArchive4Toutiao,
createArchive4Weibo,
createReadme4Weibo,
createReadme4Search,
createArchive4Search,
createSearchList,
} from "./utils.ts";
Deno.test("mergeWords4Toutiao", function (): void {
const words1: ToutiaoWord[] = [];
const words2: ToutiaoWord[] = [{ word: "foo", url: "bar" }];
const words3: ToutiaoWord[] = [{ word: "foo", url: "hello" }];
const words4: ToutiaoWord[] = [{ word: "hello", url: "world" }];
const words5: ToutiaoWord[] = [
{ word: "foo", url: "bar" },
{ word: "hello", url: "world" },
];
assertEquals(mergeWords4Toutiao(words1, words2), words2);
assertEquals(mergeWords4Toutiao(words1, words5), words5);
assertEquals(mergeWords4Toutiao(words2, words2), words2);
assertEquals(
mergeWords4Toutiao(words2, words3),
[
{ word: "foo", url: "bar" },
{ word: "foo", url: "hello" },
],
);
assertEquals(mergeWords4Toutiao(words4, words5), [
{ word: "hello", url: "world" },
{ word: "foo", url: "bar" },
]);
assertEquals(
mergeWords4Toutiao(words3, words5),
[
{ word: "foo", url: "hello" },
{ word: "foo", url: "bar" },
{ word: "hello", url: "world" },
],
);
});
Deno.test("createTuotiaoList", function (): void {
const words: ToutiaoWord[] = [
{ word: "foo", url: "bar" },
{ word: "hello", url: "world" },
];
assertStringIncludes(createTuotiaoList(words), "<!-- BEGIN TOUTIAO -->");
assertStringIncludes(createTuotiaoList(words), "<!-- END TOUTIAO -->");
assertStringIncludes(createTuotiaoList(words), "foo");
assertStringIncludes(createTuotiaoList(words), "world");
assertStringIncludes(createTuotiaoList(words), "hello");
});
Deno.test("createArchive4Toutiao", function (): void {
const words: ToutiaoWord[] = [
{ word: "foo", url: "bar" },
{ word: "hello", url: "world" },
];
assertStringIncludes(
createArchive4Toutiao(words, "2020-02-02"),
"# 2020-02-02",
);
assertStringIncludes(createArchive4Toutiao(words, "2020-02-02"), "共 2 条");
});
Deno.test("createReadme4Toutiao", async function (): Promise<void> {
const words: ToutiaoWord[] = [
{ word: "foo", url: "bar" },
{ word: "hello", url: "world" },
];
assertStringIncludes(await createReadme4Toutiao(words), "头条");
assertStringIncludes(
await createReadme4Toutiao(words),
"zhihu-trending-hot-questions",
);
});
Deno.test("mergeQuestions", function (): void {
const words1: Question[] = [];
const words2: Question[] = [{ title: "foo", url: "bar" }];
@ -43,17 +128,23 @@ Deno.test("mergeQuestions", function (): void {
);
});
Deno.test("createList", function (): void {
Deno.test("createQuestionList", function (): void {
const words: Question[] = [
{ title: "foo", url: "bar" },
{ title: "hello", url: "world" },
];
assertStringIncludes(createList(words), "<!-- BEGIN -->");
assertStringIncludes(createList(words), "<!-- END -->");
assertStringIncludes(createList(words), "foo");
assertStringIncludes(createList(words), "world");
assertStringIncludes(createList(words), "hello");
assertStringIncludes(
createQuestionList(words),
"<!-- BEGIN ZHIHUQUESTIONS -->",
);
assertStringIncludes(
createQuestionList(words),
"<!-- END ZHIHUQUESTIONS -->",
);
assertStringIncludes(createQuestionList(words), "foo");
assertStringIncludes(createQuestionList(words), "world");
assertStringIncludes(createQuestionList(words), "hello");
});
Deno.test("createArchive", function (): void {
@ -72,9 +163,155 @@ Deno.test("createReadme", async function (): Promise<void> {
{ title: "hello", url: "world" },
];
assertStringIncludes(await createReadme(words), "知乎");
assertStringIncludes(await createReadme(words), "热门");
assertStringIncludes(
await createReadme(words),
"zhihu-trending-hot-questions",
);
});
Deno.test("mergeWords4Weibo", function (): void {
const words1: Word[] = [];
const words2: Word[] = [{ title: "foo", url: "bar" }];
const words3: Word[] = [{ title: "foo", url: "hello" }];
const words4: Word[] = [{ title: "hello", url: "world" }];
const words5: Word[] = [
{ title: "foo", url: "bar" },
{ title: "hello", url: "world" },
];
assertEquals(mergeWords4Weibo(words1, words2), words2);
assertEquals(mergeWords4Weibo(words1, words5), words5);
assertEquals(mergeWords4Weibo(words2, words2), words2);
assertEquals(
mergeWords4Weibo(words2, words3),
[
{ title: "foo", url: "bar" },
{ title: "foo", url: "hello" },
],
);
assertEquals(mergeWords4Weibo(words4, words5), [
{ title: "hello", url: "world" },
{ title: "foo", url: "bar" },
]);
assertEquals(
mergeWords4Weibo(words3, words5),
[
{ title: "foo", url: "hello" },
{ title: "foo", url: "bar" },
{ title: "hello", url: "world" },
],
);
});
Deno.test("createWeiboList", function (): void {
const words: Word[] = [
{ title: "foo", url: "bar" },
{ title: "hello", url: "world" },
];
assertStringIncludes(createWeiboList(words), "<!-- BEGIN WEIBO -->");
assertStringIncludes(createWeiboList(words), "<!-- END WEIBO -->");
assertStringIncludes(createWeiboList(words), "foo");
assertStringIncludes(createWeiboList(words), "world");
assertStringIncludes(createWeiboList(words), "hello");
});
Deno.test("createArchive4Weibo", function (): void {
const words: Word[] = [
{ title: "foo", url: "bar" },
{ title: "hello", url: "world" },
];
assertStringIncludes(
createArchive4Weibo(words, "2020-02-02"),
"# 2020-02-02",
);
assertStringIncludes(createArchive4Weibo(words, "2020-02-02"), "共 2 条");
});
Deno.test("createReadme4Weibo", async function (): Promise<void> {
const words: Word[] = [
{ title: "foo", url: "bar" },
{ title: "hello", url: "world" },
];
assertStringIncludes(await createReadme4Weibo(words), "微博");
assertStringIncludes(
await createReadme4Weibo(words),
"zhihu-trending-hot-questions",
);
});
Deno.test("mergeWords", function (): void {
const words1: SearchWord[] = [];
const words2: SearchWord[] = [{ query: "foo", display_query: "bar" }];
const words3: SearchWord[] = [{ query: "foo", display_query: "hello" }];
const words4: SearchWord[] = [{ query: "hello", display_query: "world" }];
const words5: SearchWord[] = [
{ query: "foo", display_query: "bar" },
{ query: "hello", display_query: "world" },
];
assertEquals(mergeWords(words1, words2), words2);
assertEquals(mergeWords(words1, words5), words5);
assertEquals(mergeWords(words2, words2), words2);
assertEquals(
mergeWords(words2, words3),
[
{ query: "foo", display_query: "bar" },
{ query: "foo", display_query: "hello" },
],
);
assertEquals(mergeWords(words4, words5), [
{ query: "hello", display_query: "world" },
{ query: "foo", display_query: "bar" },
]);
assertEquals(
mergeWords(words3, words5),
[
{ query: "foo", display_query: "hello" },
{ query: "foo", display_query: "bar" },
{ query: "hello", display_query: "world" },
],
);
});
Deno.test("createSearchList", function (): void {
const words: SearchWord[] = [
{ query: "foo", display_query: "bar" },
{ query: "hello", display_query: "world" },
];
assertStringIncludes(createSearchList(words), "<!-- BEGIN ZHIHUSEARCH -->");
assertStringIncludes(createSearchList(words), "<!-- END ZHIHUSEARCH -->");
assertStringIncludes(createSearchList(words), "foo");
assertStringIncludes(createSearchList(words), "world");
assertStringIncludes(createSearchList(words), "hello");
});
Deno.test("createArchive4Search", function (): void {
const words: SearchWord[] = [
{ query: "foo", display_query: "bar" },
{ query: "hello", display_query: "world" },
];
assertStringIncludes(
createArchive4Search(words, "2020-02-02"),
"# 2020-02-02",
);
assertStringIncludes(createArchive4Search(words, "2020-02-02"), "共 2 条");
});
Deno.test("createReadme4Search", async function (): Promise<void> {
const words: SearchWord[] = [
{ query: "foo", display_query: "bar" },
{ query: "hello", display_query: "world" },
];
assertStringIncludes(await createReadme4Search(words), "热搜");
assertStringIncludes(
await createReadme4Search(words),
"zhihu-trending-hot-questions",
);
});

49
utils.ts
View File

@ -37,7 +37,7 @@ export function mergeWords4Toutiao(
): ToutiaoWord[] {
const obj: Record<string, string> = {};
for (const w of words.concat(another)) {
obj[w.word] = w.word;
obj[w.url] = w.word;
}
return Object.entries(obj).map(([url, word]) => ({
url,
@ -62,24 +62,42 @@ export function mergeWords4Weibo(
export async function createReadme(words: Question[]): Promise<string> {
const readme = await Deno.readTextFile("./README.md");
return readme.replace(/<!-- BEGIN ZHIHUVIDEO -->[\W\w]*<!-- END ZHIHUVIDEO -->/, createVideoList(words))
.replace(/<!-- BEGIN ZHIHUQUESTIONS -->[\W\w]*<!-- END ZHIHUQUESTIONS -->/, createQuestionList(words));
return readme.replace(
/<!-- BEGIN ZHIHUVIDEO -->[\W\w]*<!-- END ZHIHUVIDEO -->/,
createVideoList(words),
)
.replace(
/<!-- BEGIN ZHIHUQUESTIONS -->[\W\w]*<!-- END ZHIHUQUESTIONS -->/,
createQuestionList(words),
);
}
export async function createReadme4Search(words: SearchWord[]): Promise<string> {
export async function createReadme4Search(
words: SearchWord[],
): Promise<string> {
const readme = await Deno.readTextFile("./README.md");
return readme.replace(/<!-- BEGIN ZHIHUSEARCH -->[\W\w]*<!-- END ZHIHUSEARCH -->/, createSearchList(words));
return readme.replace(
/<!-- BEGIN ZHIHUSEARCH -->[\W\w]*<!-- END ZHIHUSEARCH -->/,
createSearchList(words),
);
}
export async function createReadme4Weibo(words: Word[]): Promise<string> {
const readme = await Deno.readTextFile("./README.md");
return readme.replace(/<!-- BEGIN WEIBO -->[\W\w]*<!-- END WEIBO -->/, createWeiboList(words));
return readme.replace(
/<!-- BEGIN WEIBO -->[\W\w]*<!-- END WEIBO -->/,
createWeiboList(words),
);
}
export async function createReadme4Toutiao(words: ToutiaoWord[]): Promise<string> {
export async function createReadme4Toutiao(
words: ToutiaoWord[],
): Promise<string> {
const readme = await Deno.readTextFile("./README.md");
return readme.replace(/<!-- BEGIN TOUTIAO -->[\W\w]*<!-- END TOUTIAO -->/, createTuotiaoList(words));
return readme.replace(
/<!-- BEGIN TOUTIAO -->[\W\w]*<!-- END TOUTIAO -->/,
createTuotiaoList(words),
);
}
export function createVideoList(words: Question[]): string {
@ -133,8 +151,6 @@ ${
<!-- END TOUTIAO -->`;
}
export function createArchive(words: Question[], date: string): string {
return `# ${date}\n
共 ${words.length} 条\n
@ -142,7 +158,10 @@ ${createVideoList(words)}
`;
}
export function createArchive4Search(words: SearchWord[], date: string): string {
export function createArchive4Search(
words: SearchWord[],
date: string,
): string {
return `# ${date}\n
共 ${words.length} 条\n
${createSearchList(words)}
@ -156,10 +175,12 @@ ${createWeiboList(words)}
`;
}
export function createArchive4Toutiao(words: ToutiaoWord[], date: string): string {
export function createArchive4Toutiao(
words: ToutiaoWord[],
date: string,
): string {
return `# ${date}\n
共 ${words.length} 条\n
${createTuotiaoList(words)}
`;
}

11
weibo-search.ts
View File

@ -5,7 +5,11 @@ import { join } from "std/path/mod.ts";
import { exists } from "std/fs/mod.ts";
import type { Word } from "./types.ts";
import { createArchive4Weibo, createReadme4Weibo, mergeWords4Weibo } from "./utils.ts";
import {
createArchive4Weibo,
createReadme4Weibo,
mergeWords4Weibo,
} from "./utils.ts";
const regexp = /<a href="(\/weibo\?q=[^"]+)".*?>(.+)<\/a>/g;
@ -36,12 +40,11 @@ if (await exists(fullPath)) {
const queswordsAll = mergeWords4Weibo(words, wordsAlreadyDownload);
export const weiboSearchData = queswordsAll.map(x=> {
export const weiboSearchData = queswordsAll.map((x) => {
x.realurl = `https://s.weibo.com/${x.url}`;
return x;
});
export async function weiboSearch() {
// 保存原始数据
await Deno.writeTextFile(fullPath, JSON.stringify(queswordsAll));
@ -54,6 +57,4 @@ export async function weiboSearch() {
const archiveText = createArchive4Weibo(queswordsAll, yyyyMMdd);
const archivePath = join("archives/weibo-search", `${yyyyMMdd}.md`);
await Deno.writeTextFile(archivePath, archiveText);
}

3
zhihu-questions.ts
View File

@ -8,7 +8,6 @@ import { exists } from "std/fs/mod.ts";
import type { ZhihuQuestionList, Question } from "./types.ts";
import { createArchive, createReadme, mergeQuestions } from "./utils.ts";
const response = await fetch(
"https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total?limit=100",
);
@ -39,7 +38,6 @@ const questionsAll = mergeQuestions(questions, questionsAlreadyDownload);
export const zhihuQuestionData = questionsAll;
export async function zhihuQuestions() {
// 保存原始数据
await Deno.writeTextFile(fullPath, JSON.stringify(questionsAll));
@ -51,5 +49,4 @@ export async function zhihuQuestions() {
const archiveText = createArchive(questionsAll, yyyyMMdd);
const archivePath = join("archives/zhihu-questions", `${yyyyMMdd}.md`);
await Deno.writeTextFile(archivePath, archiveText);
}

9
zhihu-search.ts
View File

@ -5,7 +5,11 @@ import { join } from "std/path/mod.ts";
import { exists } from "std/fs/mod.ts";
import type { SearchWord, TopSearch } from "./types.ts";
import { createArchive4Search, createReadme4Search, mergeWords } from "./utils.ts";
import {
createArchive4Search,
createReadme4Search,
mergeWords,
} from "./utils.ts";
const response = await fetch("https://www.zhihu.com/api/v4/search/top_search");
@ -28,7 +32,7 @@ if (await exists(fullPath)) {
const wordsAll = mergeWords(words, wordsAlreadyDownload);
export const zhihuSearchData = wordsAll.map(x=> {
export const zhihuSearchData = wordsAll.map((x) => {
x.url = `https://www.zhihu.com/search?q=${x.query}`;
return x;
});
@ -45,5 +49,4 @@ export async function zhihuSearch() {
const archiveText = createArchive4Search(wordsAll, yyyyMMdd);
const archivePath = join("archives/zhihu-search", `${yyyyMMdd}.md`);
await Deno.writeTextFile(archivePath, archiveText);
}

3
zhihu-video.ts
View File

@ -42,7 +42,6 @@ const questionsAll = mergeQuestions(questions, questionsAlreadyDownload);
export const zhihuVideoData = questionsAll;
export async function zhihuVideo() {
// 保存原始数据
await Deno.writeTextFile(fullPath, JSON.stringify(questionsAll));
@ -54,6 +53,4 @@ export async function zhihuVideo() {
const archiveText = createArchive(questionsAll, yyyyMMdd);
const archivePath = join("archives/zhihu-video", `${yyyyMMdd}.md`);
await Deno.writeTextFile(archivePath, archiveText);
}