抓取分析网页批量下载评书(2)之分析下载地址

抓取分析网页批量下载评书(2)之分析下载地址【原创】 2016/3/19 0:39:26 阅读:7357 评论:0

本系列目录： 《1、搜索有声小说》 《2、分析详细页地址》 《3、批量下载mp3》

本篇是第二篇，主要介绍如何分析有声读物每集的链接地址，本文希望阅读者根据文章就能自己写出程序，所以写的比较详细，大家见谅，软件的下载地址。

一、获取MP3每集的地址

1、咱们随便找一本评书进入其详细页，我们要获得评书的简介和剧集列表两部分内容，具体页面如下:

分析一下Html代码，简介是包含在<div class="book02 padding5 line5">简介内容</div>中，但需要注意，里包含了br、script等html标签,所以抓取内容后，需要清除一下html标签。

另外有一点需要注意，由于简介中可能含有div，所以正则表达式用贪婪模式或非贪婪模式都可能出现无法正确匹配，所以要保证唯一性，注意下面粗体的部分。

获取简介的正则表达式：<div class=""book02 padding5 line5"">(?<Summary>[\s\S]*?)</div>\r\n<div class=""js[\d]*"">

PS:使用IE开发人员工具（F12）时发现一个问题，它会对代码进行整理，所以你看到内容有可能不是网页的真实内容，如下面两幅图

title,href的顺序不一样，Chrome中显示的是实际代码顺序。

具体的正则表达式：<div class=""b2""><a href=""(?<SubPath>[\d\w]*)/play_[\d]*_(?<Number>[\d]*).htm""(\s*title=""[\s\S]*?.mp3"")*>(?<Title>[\s\S]*?).mp3</a></div>

2、评书下载界面如下 :

3、主要的代码

Sound.cs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace psDownload
{
    /// <summary>
    /// 有声读物类
    /// </summary>
    public class Sound
    {
        /// <summary>
        /// 编号
        /// </summary>
        public int ID { set; get; }

        /// <summary>
        /// 标题
        /// </summary>
        public string Title { set; get; }

        /// <summary>
        /// 演播者
        /// </summary>
        public string Performer { set; get; }

        /// <summary>
        /// 详细页网址
        /// </summary>
        public string Url { set; get; }

        /// <summary>
        /// MP3的下载地址
        /// </summary>
        public string DownUrl { set; get; }

        /// <summary>
        /// 状态  0：等待下载  1：下载成功  -1：下载失败
        /// </summary>
        public int Status { set; get; }

        /// <summary>
        /// 错误详情
        /// </summary>
        public string Error { set; get; }
    }
}

 C# Code 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183

//评书的名称、详细页网址、所属分类、文件输出路径
private string _title = "", _url = "", _category = "", _outpath = "";

//评书集数列表，Sound是一个自己写的类。
private List<Sound> _list = new List<Sound>();

public frmDetail(string Title, string Url, string OutPath)
{
    InitializeComponent();
    this._title = Title;
    this._url = Url;
    this._outpath = OutPath;

    //获取评书所属的分类，后面需要用到，其实从上一个窗体传过来也行。
    Match match = Regex.Match(Url, @"http://www.tingchina.com/(?<Category>[\w]*)/disp_[\d]*.htm", RegexOptions.IgnoreCase | RegexOptions.Multiline);
    if (match.Success)
    {
        this._category = match.Groups["Category"].Value;
    }
}

private void frmDetail_Load(object sender, EventArgs e)
{
    this.Text = this._title;
    this.lbl_Title.Text = this._title;
    this.llbl_Url.Text = this._url;

    //发送异步请求，获取评书的详细信息和剧集列表
    HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(this._url);
    req.Timeout = 15 * 1000;
    req.Method = "GET";
    req.BeginGetResponse(new AsyncCallback(ResponseCallBack), req);
}

/// <summary>
/// http异步请求的回调函数，分析网页，获得评书的简介、剧集列表等信息
/// </summary>
/// <param name="result"></param>
public void ResponseCallBack(IAsyncResult result)
{
    string Html = "";
    HttpWebRequest req = (HttpWebRequest)result.AsyncState;
    try
    {
        using (HttpWebResponse response = (HttpWebResponse)req.EndGetResponse(result))
        {
            Stream resStream = response.GetResponseStream();
            StreamReader sr = new StreamReader(resStream, Encoding.GetEncoding("GB2312"));
            Html = sr.ReadToEnd();
        }
    }
    catch (Exception ex)
    {
        if (IsDisposed || !this.IsHandleCreated) return;
        this.Invoke(new Action(() =>
        {
            MessageBox.Show("抓取网页出现异常,原因:" + ex.Message);
        }));
        return;
    }

    //动态生成Label控件所需的字体
    Font font = new Font("微软雅黑", 10F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(134)));

    //获取评书的简介
    Match ms_Summary = Regex.Match(Html, @"<div class=""book02 padding5 line5"">(?<Summary>[\s\S]*?)</div>\r\n<div class=""js[\d]*"">", RegexOptions.IgnoreCase | RegexOptions.Multiline);
    if (ms_Summary.Success)
    {
        if (IsDisposed || !this.IsHandleCreated) return;
        this.Invoke(new Action(() =>
        {
            this.txt_Summary.Text = ClearHtml(ms_Summary.Groups["Summary"].Value);
        }));
    }

    //正则表达式分析网页，获取评书的剧集。
    MatchCollection ms = Regex.Matches(Html, @"<div class=""b2""><a href=""(?<SubPath>[\d\w]*)/play_[\d]*_(?<Number>[\d]*).htm""(\s*title=""[\s\S]*?.mp3"")*>(?<Title>[\s\S]*?).mp3</a></div>", RegexOptions.IgnoreCase | RegexOptions.Multiline);
    if (ms.Count > 0)
    {
        //最大宽度
        int MaxWidth = 0;
        this._list.Clear();
        foreach (Match m in ms)
        {
            Sound sound = new Sound();
            sound.ID = int.Parse(m.Groups["Number"].Value);
            sound.Title = m.Groups["Title"].Value;
            sound.Url = "http://www.tingchina.com/" + this._category + "/" + m.Groups["SubPath"].Value + "/play_" + m.Groups["SubPath"].Value + "_" + m.Groups["Number"].Value + ".htm";
            this._list.Add(sound);

            //获取字体的大小，找到最宽的那个条记录
            Size size = TextRenderer.MeasureText(m.Groups["Title"].Value, font);
            if (size.Width > MaxWidth)
            {
                MaxWidth = size.Width;
            }
        }

        if (IsDisposed || !this.IsHandleCreated) return;
        this.Invoke(new Action(() =>
        {
            //循环动态生成查询结果.
            foreach (Sound sound in _list)
            {
                Panel pnl = new Panel();

                Label lbl = new Label();
                lbl.Name = "lbl_" + sound.ID.ToString();
                lbl.Text = sound.Title;
                lbl.Tag = sound.Url;
                lbl.Cursor = System.Windows.Forms.Cursors.Hand;
                lbl.Margin = new System.Windows.Forms.Padding(15, 0, 0, 10);
                lbl.ForeColor = System.Drawing.Color.FromArgb(((int)(((byte)(0)))), ((int)(((byte)(192)))), ((int)(((byte)(0)))));
                lbl.Font = font;
                lbl.Width = MaxWidth;
                lbl.Click += new EventHandler(lbl_Click);
                pnl.Controls.Add(lbl);

                PictureBox pic = new PictureBox();
                pic.Name = "pic_" + sound.ID.ToString();
                pic.Left = lbl.Width;
                pic.Top = 3;
                pic.SizeMode = PictureBoxSizeMode.Zoom;
                pic.Height = 16;
                pic.Width = 16;

                pnl.Name = "pnl_" + sound.ID.ToString();
                pnl.Tag = sound;
                pnl.Controls.Add(pic);
                pnl.AutoSize = true;

                this.fpnl_Content.Controls.Add(pnl);
                Application.DoEvents();
            }
            this.btn_Download.Enabled = true;
            this.btn_Download.ForeColor = Color.Black;
            this.btn_Download.Text = "批量下载";
            this.btn_Download.Focus();
        }));
    }
    else
    {
        if (IsDisposed || !this.IsHandleCreated) return;
        this.Invoke(new Action(() =>
        {
            MessageBox.Show("分析网页失败，请检查。");
        }));
    }
}

/// <summary>
/// 清除字符串中HTML控制字符
/// </summary>
/// <param name="s">要清除的字符串</param>
/// <returns></returns>
public static string ClearHtml(string s)
{
    if (s == null) return null;
    return HtmlDecode(Regex.Replace(s, @"(<[^>]+>)|[\r\n]", "", RegexOptions.IgnoreCase | RegexOptions.Singleline));
}

private static string HtmlDecodeMatchEvaluator(Match m)
{
    switch (m.Value)
    {
        case "<": return "<";
        case ">": return ">";
        case "&": return "&";
        case """: return "\"";
        case " ": return "\u0020";
        default: return m.Value;
    }
}

/// <summary>
/// HTML解码
/// </summary>
/// <param name="s"></param>
/// <returns></returns>
private static string HtmlDecode(string s)
{
    return Regex.Replace(s, "(<)|(>)|(&)|(")|( )", new MatchEvaluator(HtmlDecodeMatchEvaluator), RegexOptions.Singleline | RegexOptions.IgnoreCase);
}

暂无评论

发表评论