c# 基于Titanium爬取微信公眾號歷史文章列表

更新時間：2021年03月11日 09:50:58 作者：leestar54

這篇文章主要介紹了c# 基于Titanium爬取微信公眾號歷史文章列表,幫助大家更好的理解和學習使用c#，感興趣的朋友可以了解下

什么是Titanium

基于C#的跨平臺異步HTTP(S)代理服務器
類似的還有：

https://github.com/http-party/node-http-proxy

原理簡述

對于HTTP

顧名思義，其實代理就是一個「中間人」角色，對于連接到它的客戶端來說，它是服務端；對于要連接的服務端來說，它是客戶端。它就負責在兩端之間來回傳送 HTTP 報文。

對于HTTPS

由于HTTPS加入了CA證書的校驗，服務端可不驗證客戶端的證書，中間人可以偽裝成客戶端與服務端成功完成 TLS 握手；但是中間人沒有服務端證書私鑰，無論如何也無法偽裝成服務端跟客戶端建立 TLS 連接，所以我們需要換個方式代理HTTPS請求。

HTTPS在傳輸層之上建立了安全層，所有的HTTP請求都在安全層上傳輸。既然無法通過像代理一般HTTP請求的方式在應用層代理HTTPS請求，那么我們就退而求其次為在傳輸層為客戶端和服務器建立起TCP連接。一旦 TCP 連接建好，代理無腦轉發(fā)后續(xù)流量即可。所以這種代理，理論上適用于任意基于 TCP 的應用層協(xié)議，HTTPS 網(wǎng)站使用的 TLS 協(xié)議當然也可以。這也是這種代理為什么被稱為隧道的原因。
但是這樣子無腦轉發(fā)我們就無法獲取到他們交互的數(shù)據(jù)了，怎么辦？
此時就需要代理變身為偽HTTPS服務器，然后讓客戶端信任我們自定義的根證書，從而在客戶端和代理、代理和服務端之間都能成功建立 TLS 連接。對于代理來說，兩端的TLS流量都是可以解密的。

最后如果這個代理我們可編程，那么我們就可以對傳送的HTTP報文做控制。
相關的應用場景有訪問控制、防火墻、內(nèi)容過濾、Web緩存、內(nèi)容路由等等。

為什么要爬取歷史文章

微信官方其實已經(jīng)提過了素材列表接口

https://developers.weixin.qq.com/doc/offiaccount/Asset_Management/Get_materials_list.html

但是接口獲取的素材地址并不是真實在公眾號上推送的地址，所以不存在閱讀、評論、好看等功能。
這時候需要我們?nèi)ス娞柕臍v史文章頁取數(shù)據(jù)。

實現(xiàn)步驟

開發(fā)環(huán)境：Visual Studio Community 2019 for Mac Version 8.5.6 (build 11)
Frameworks：.NET Core 3.1.0
NuGet：Newtonsoft.Json 12.0.3、Titanium.Web.Proxy 3.1.1301

大致思路

1、先實現(xiàn)通過代理抓包HTTPS，攔截微信客戶端數(shù)據(jù)交互。
2、過濾其他地址，只監(jiān)測微信文章。
3、訪問任意微信文章頁，獲取header和cookie。
4、模擬微信訪問歷史頁、分析抓取歷史文章列表。

核心代碼

SpiderProxy.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using Newtonsoft.Json.Linq;
using Titanium.Web.Proxy;
using Titanium.Web.Proxy.Http;
using Titanium.Web.Proxy.Models;


namespace WechatArticleSpider
{
 public class SpiderProxy
 {
  private readonly SemaphoreSlim @lock = new SemaphoreSlim(1);
  private readonly ProxyServer proxyServer;
  private readonly ExplicitProxyEndPoint explicitEndPoint;


  public SpiderProxy()
  {
   proxyServer = new ProxyServer();
   //在響應之前事件
   proxyServer.BeforeResponse += ProxyServer_BeforeResponse;


   //綁定監(jiān)聽端口
   explicitEndPoint = new ExplicitProxyEndPoint(IPAddress.Any, 8000, true);
   Console.WriteLine("監(jiān)聽地址127.0.0.1:8000");
   //隧道請求連接前事件，HTTPS用
   explicitEndPoint.BeforeTunnelConnectRequest += ExplicitEndPoint_BeforeTunnelConnectRequest; ;


   //代理服務器注冊監(jiān)聽地址
   proxyServer.AddEndPoint(explicitEndPoint);
  }


  public void Start()
  {
   Console.WriteLine("開始監(jiān)聽");
   //Start方法會檢測證書，若為空會調(diào)用CertificateManager.EnsureRootCertificate();為我們自動生成根證書。
   proxyServer.Start();
  }


  public void Stop()
  {
   // Unsubscribe & Quit
   explicitEndPoint.BeforeTunnelConnectRequest -= ExplicitEndPoint_BeforeTunnelConnectRequest;
   proxyServer.BeforeResponse -= ProxyServer_BeforeResponse;


   Console.WriteLine("結束監(jiān)聽");
   proxyServer.Stop();
  }


  private async Task ExplicitEndPoint_BeforeTunnelConnectRequest(object sender, Titanium.Web.Proxy.EventArguments.TunnelConnectSessionEventArgs e)
  {
   string hostname = e.HttpClient.Request.RequestUri.Host;
   await WriteToConsole("Tunnel to: " + hostname);


   if (!hostname.StartsWith("mp.weixin.qq.com"))
   {
    //是否要解析SSL，不解析就直接轉發(fā)
    e.DecryptSsl = false;
   }
  }


  private async Task ProxyServer_BeforeResponse(object sender, Titanium.Web.Proxy.EventArguments.SessionEventArgs e)
  {
   var request = e.HttpClient.Request;


   //判斷是否是微信文章頁。
   if (request.Host.Contains("mp.weixin.qq.com")&&(request.RequestUriString.StartsWith("/s?") || request.RequestUriString.StartsWith("/s/")))
   {
    byte[] bytes = await e.GetResponseBody();
    string body = System.Text.Encoding.UTF8.GetString(bytes);
    ThreadPool.QueueUserWorkItem((stateInfo) => { CrawlAsync(body, e.HttpClient.Request); }); 
   }
  }


  private async Task CrawlAsync(string body,Request request)
  {
   //采用正則表達式匹配數(shù)據(jù)
   Match match = Regex.Match(body, @"<strong class=""profile_nickname"">(.+)</strong>");
   Match matchGhid = Regex.Match(body, @"var user_name = ""(.+)"";");
   if (!match.Success || !matchGhid.Success)
   {
    return;
   }


   MatchCollection matches = Regex.Matches(body, @"<span class=""profile_meta_value"">(.+)</span>");
   if (match.Groups.Count == 0)
   {
    return;
   }
   await WriteToConsole("檢測到微信文章頁: " + match.Groups[1].Value + " " + matches[0].Groups[1].Value + " " + matches[1].Groups[1].Value + " ");


   var queryString = HttpUtility.ParseQueryString(request.RequestUriString.Substring(3));
   var httpClient = new HttpClient(request.Headers);
   await WriteToConsole("Client實例化，已獲取header，cookie");


   //獲取歷史頁信息
   string result = httpClient.Get(string.Format("https://mp.weixin.qq.com/mp/profile_ext?action=home&__biz={0}&scene=124#wechat_redirect", queryString["__biz"]));
   //封接口檢測
   if (result.Contains("操作頻繁，請稍后再試"))
   {
    await WriteToConsole("操作頻繁，請稍后再試 限制24小時 請更換微信");
    await WriteToConsole("已停止爬蟲任務，請更換之后重啟助手。");
   }


   //是否抓取完
   bool end = false;
   //下標
   int offset = 0;


   do
   {
    //獲取歷史消息
    string jsonResult = httpClient.Get(string.Format("https://mp.weixin.qq.com/mp/profile_ext?action=getmsg&__biz={0}&f=json&offset={1}&count=10&is_ok=1&scene=124&uin={2}&key={3}&pass_ticket={4}&wxtoken=&x5=0&f=json", queryString["__biz"], offset, queryString["uin"], queryString["key"], queryString["pass_ticket"]));
    JObject jobject = JObject.Parse(jsonResult);
    if (Convert.ToInt32(jobject["ret"]) == 0)
    {
     if (Convert.ToInt32(jobject["can_msg_continue"]) == 0)
     {
      end = true;
     }
     offset = Convert.ToInt32(jobject["next_offset"]);
     string strList = jobject["general_msg_list"].ToString();
     JObject temp = JObject.Parse(strList);
     List<JToken> list = temp["list"].ToList();
     foreach (var item in list)
     {
      JToken comm_msg_info = item["comm_msg_info"];
      JToken app_msg_ext_info = item["app_msg_ext_info"];
      if (app_msg_ext_info == null)
      {
       continue;
      }
      //發(fā)布時間
      string publicTime = comm_msg_info["datetime"].ToString();
      //文章標題
      string title = app_msg_ext_info["title"].ToString();
      //文章摘要
      string digest = app_msg_ext_info["digest"].ToString();
      //文章地址
      string content_url = app_msg_ext_info["content_url"].ToString();
      //文章封面
      string cover = app_msg_ext_info["cover"].ToString();
      //作者
      string author = app_msg_ext_info["author"].ToString();


      await WriteToConsole(String.Format("{0},{1},{2},{3},{4},{5}", publicTime, title, digest, content_url, cover, author));


      //今天發(fā)布了多條消息
      if (app_msg_ext_info["is_multi"].ToString() == "1")
      {
       foreach (var multiItem in app_msg_ext_info["multi_app_msg_item_list"].ToList())
       {
        title = multiItem["title"].ToString();
        digest = multiItem["digest"].ToString();
        content_url = multiItem["content_url"].ToString();
        cover = multiItem["cover"].ToString();
        author = multiItem["author"].ToString();
        await WriteToConsole(String.Format("{0},{1},{2},{3},{4},{5}", publicTime, title, digest, content_url, cover, author));
       }
      }
     }
    }
    else
    {
     end = true;
    }
    //每5秒翻頁一次
    await Task.Delay(5000);
   } while (!end);


   await WriteToConsole("歷史文章抓取完成");


  }


  private async Task WriteToConsole(string message, ConsoleColor? consoleColor = null)
  {
   await @lock.WaitAsync();


   if (consoleColor.HasValue)
   {
    ConsoleColor existing = Console.ForegroundColor;
    Console.ForegroundColor = consoleColor.Value;
    Console.WriteLine(message);
    Console.ForegroundColor = existing;
   }
   else
   {
    Console.WriteLine(message);
   }


   @lock.Release();
  }
 }
}

HttpClient.cs

using System;
using System.IO;
using System.Net;
using System.Threading;
using Titanium.Web.Proxy.Http;


namespace WechatArticleSpider
{
 class HttpClient
 {
  private readonly HeaderCollection headerCollection;
  private readonly CookieContainer cookieContainer = new CookieContainer();


  /// <summary>
  /// 微信請求客戶端
  /// </summary>
  /// <param name="headerCollection">攔截微信請求頭集合</param>
  public HttpClient(HeaderCollection headerCollection)
  {
   cookieContainer = new CookieContainer();
   ServicePointManager.DefaultConnectionLimit = 512;
   ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11;
   this.headerCollection = headerCollection;
  }


  /// <summary>
  /// 帶微信參數(shù)的GET請求
  /// </summary>
  /// <param name="url">請求地址</param>
  /// <returns>請求結果</returns>
  public string Get(string url)
  {
   string ret = Retry<string>(() =>
   {
    HttpWebRequest webRequest = WebRequest.CreateHttp(url);
    webRequest = PretendWechat(webRequest);
    HttpWebResponse response = webRequest.GetResponse() as HttpWebResponse;
    string result = new StreamReader(response.GetResponseStream()).ReadToEnd();
    return result;
   });
   return ret;
  }


  /// <summary>
  /// 偽造微信請求
  /// </summary>
  /// <param name="request">需要偽造的request</param>
  /// <returns></returns>
  public HttpWebRequest PretendWechat(HttpWebRequest request)
  {
   try
   {
    request.Host = headerCollection.Headers["Host"].Value;
    request.UserAgent = headerCollection.Headers["User-Agent"].Value;
    request.Headers.Set(headerCollection.Headers["Accept-Language"].Name, headerCollection.Headers["Accept-Language"].Value);
    request.Headers.Set(headerCollection.Headers["Accept-Encoding"].Name, headerCollection.Headers["Accept-Encoding"].Value);
    cookieContainer.SetCookies(new Uri("https://mp.weixin.qq.com"), headerCollection.Headers["Cookie"].Value.Replace(";", ","));
    request.KeepAlive = true;
    request.Accept = headerCollection.Headers["Accept"].Value;
    request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
    request.CookieContainer = cookieContainer;
    request.AllowAutoRedirect = true;
    request.ServicePoint.Expect100Continue = false;
    request.Timeout = 35000;
    return request;
   }
   catch (Exception e)
   {
    Console.WriteLine(e.Message);
    throw;
   }
  }


  /// <summary>
  /// 三次重試機制
  /// </summary>
  /// <typeparam name="T">參數(shù)類型</typeparam>
  /// <param name="func">方法</param>
  /// <returns></returns>
  private static T Retry<T>(Func<T> func)
  {
   int err = 0;
   while (err < 3)
   {
    try
    {
     return func();
    }
    catch (WebException webExp)
    {
     err++;
     Thread.Sleep(5000);
     if (err > 2)
     {
      throw webExp;
     }
    }
   }
   return func();
  }
 }
}