导出专栏文章评论

Author: jjeejj

.gitignore

@@ -3,4 +3,5 @@ node_modules
 geektime_*/**
 config.js
 .nyc_output
-coverage
+coverage
+quest.md

columnArticleList.js

@@ -5,6 +5,7 @@ const utils = require('./utils');
 const path = require('path');
 const generaterPdf = require('./generaterPdf.js');
 const downloadAudio = require('./downloadAudio.js');
+const downloadComment = require('./downloadComment.js');
 
 /**
  * 执行方法
@@ -45,6 +46,16 @@ const downloadAudio = require('./downloadAudio.js');
                 audioTitle: columnArticle.audio_title
             };
             columnArticleUrlList.push(articleInfo);
+            articleInfo.commentsTotal = 0;
+            articleInfo.commentsArr = [];
+            // 是否导出评论
+            if (config.isComment) {
+                let {commentsTotal, commentsArr} = await downloadComment(
+                    config.columnBaseUrl + articalId,
+                    articalId);
+                articleInfo.commentsTotal = commentsTotal;
+                articleInfo.commentsArr = commentsArr;
+            };
             // 替换文章名称的 / 线， 解决路径被分割的问题
             let useArticleTtle = columnArticle.article_title.replace(/\//g, '-');
             //生成PDF 

config.js

@@ -3,10 +3,11 @@
  */
 module.exports = {
     url: 'https://time.geekbang.org/serv/v1/article',
+    commentUrl: 'https://time.geekbang.org/serv/v1/comments',
     columnBaseUrl: 'https://time.geekbang.org/column/article/',
-    columnName: '趣谈Linux操作系统', // 专栏名称
-    firstArticalId: 87104, //专栏第一篇文章的ID
-    isdownloadVideo: false, // 是否下载音频
-    isComment: false, // 是否导出评论
+    columnName: '软件工程之美',
+    firstArticalId: 85730, //专栏第一篇文章的ID
+    isdownloadVideo: true, // 是否下载音频
+    isComment: true, // 是否导出评论
     cookie: 'cookie'
 };

downloadComment.js

@@ -0,0 +1,53 @@
+// 获取每篇文章下面所有的评论
+const config = require('./config.js');
+const superagent = require('superagent');
+const utils = require('./utils');
+
+/**
+ * 获取每篇文章下面所有的评论
+ * @param {String} 文章的链接地址
+ * @param {Number} 文章的ID
+ */
+async function downloadComments (url, articleId, prev = 0) {
+    console.log('开始获取 ', url, '评论');
+    let commentsArr = [];
+    let commentsTotal = 0;
+    async function run (prev) {
+        try {
+            let res = await superagent.post(config.commentUrl)
+            .set({
+                'Content-Type': 'application/json',
+                'Cookie': config.cookie,
+                'Referer': url,
+                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
+            }).send({
+                aid: articleId,
+                prev: prev
+            });
+            if (res.body && res.body.error && res.body.error.code){
+                console.log('error msg', res.body.error.msg);
+                throw new Error(res.body.error.msg);
+            };
+            let resData = res.body.data
+            commentsTotal = resData.page.count;
+            let nextPage = resData.page.more;
+            commentsArr.push(...resData.list);
+            if (nextPage) {
+                prev = resData.list[resData.list.length -1].score;
+                await utils.sleep(1);
+                await run(prev);
+            };
+        }catch (err){
+            console.log(`获取 评论 ${url} err`, err.message);
+        };
+    };
+    await run(prev);
+    // console.log('commentsArr', commentsArr);
+    // console.log('commentsTotal', commentsTotal);
+    console.log('结束获取 ', url, '评论 总评论数为', commentsTotal);
+    return {commentsArr, commentsTotal};
+};
+
+// downloadComments('https://time.geekbang.org/column/article/82337',82337);
+
+module.exports = downloadComments;

generaterPdf.js

@@ -3056,6 +3056,33 @@
                         </div>
                         <% } %>
                     </div>
+                    <div class="_1qhD3bdE_0 _2QmGFWqF_0">
+                        <h2><span>精选留言</span><span class="_2FC_cD1O_0"><%- commentsTotal %></span></h2> 
+                        <ul>
+                            <%for(let i=0;i<commentsArr.length;i++){%>
+                            <li class="reJj6Thl_0">
+                                <img src="<%- commentsArr[i].user_header %>" class="_2273kGdT_0"/> 
+                                <div class="_2CG0SquK_0">
+                                    <div class="_304R4gla_0">
+                                        <div>
+                                            <div class="_18Dng5rT_0">
+                                                <span><%- commentsArr[i].user_name %></span>
+                                            </div> 
+                                            <div class="_1H1Z49Dr_0"><%- commentsArr[i].comment_ctime %></div>
+                                        </div>
+                                    </div> 
+                                    <div class="_3M6kV3zb_0 _3D2NkqD6_0"><%- commentsArr[i].comment_content %></div>
+                                    <% if(commentsArr[i].replies) { %>
+                                        <% for(let j=0;j<commentsArr[i].replies.length;j++){ %>
+                                        <div class="_2xNIY4NG_0">
+                                            <p class="_33BLbmw4_0"><%- commentsArr[i].replies[j].user_name %> <%- commentsArr[i].replies[j].content %></p>
+                                        </div>
+                                        <% } %>
+                                    <% } %>
+                                </div>
+                            </li>
+                            <%}%>
+                        </ul>
                 </div>
             </div>
         </div>