Skip to content

Commit

Permalink
插件下载，增加哈希校验，充分利用本地缓存文件
Browse files Browse the repository at this point in the history
  • Loading branch information
@nnhy
nnhy committed Aug 10, 2024
1 parent 3022a5f commit 7cf4384
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 16 deletions.
47 changes: 35 additions & 12 deletions NewLife.Core/Web/Link.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,16 @@ public class Link
/// <summary>时间</summary>
public DateTime Time { get; set; }

/// <summary>哈希</summary>
public String? Hash { get; set; }

/// <summary>原始Html</summary>
public String? Html { get; set; }
#endregion

#region 方法
static readonly Regex _regA = new("<a(?<其它1>[^>]*) href=?\"(?<链接>[^>\"]*)?\"(?<其它2>[^>]*)>(?<名称>[^<]*)</a>", RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled);
static readonly Regex _regTitle = new("title=(\"?)(?<标题>[^ \']*?)\\1", RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled);
static readonly Regex _regA = new("""<a[^>]* href=?"(?<链接>[^>"]*)?"[^>]*>(?<名称>[^<]*)</a>\s*</td>[^>]*<td[^>]*>(?<哈希>[^<]*)</td>""", RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled);
static readonly Regex _regTitle = new("""title=("?)(?<标题>[^ ']*?)\1""", RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled);

/// <summary>分析HTML中的链接</summary>
/// <param name="html">Html文本</param>
Expand All @@ -58,8 +61,10 @@ public static Link[] Parse(String html, String? baseUrl = null, Func<Link, Boole
{
Html = match.Value,
FullName = match.Groups["名称"].Value.Trim(),
Url = match.Groups["链接"].Value.Trim()
Url = match.Groups["链接"].Value.Trim(),
Hash = match.Groups["哈希"].Value.Trim(),
};
if (link.Hash.Contains("&lt;")) link.Hash = null;
link.RawUrl = link.Url;
link.Name = link.FullName;

Expand Down Expand Up @@ -100,9 +105,15 @@ public static Link[] Parse(String html, String? baseUrl = null, Func<Link, Boole
// 分割版本，_v1.0.0.0
link.ParseVersion();

// 去掉后缀
var p = link.Name.LastIndexOf('.');
if (p > 0) link.Name = link.Name[..p];
// 去掉后缀，特殊处理.tar.gz双后缀
var name = link.Name;
if (name.EndsWithIgnoreCase(".tar.gz"))
link.Name = name[..^7];
else
{
var p = name.LastIndexOf('.');
if (p > 0) link.Name = name[..p];
}

list.Add(link);
}
Expand Down Expand Up @@ -156,9 +167,15 @@ private static Link[] ParseFTP(String html, String? baseUrl, Func<Link, Boolean>
idx = link.ParseVersion();
if (idx > 0) link.Title = link.Title[..idx];

// 去掉后缀
var p = link.Name.LastIndexOf('.');
if (p > 0) link.Name = link.Name[..p];
// 去掉后缀，特殊处理.tar.gz双后缀
var name = link.Name;
if (name.EndsWithIgnoreCase(".tar.gz"))
link.Name = name[..^7];
else
{
var p = name.LastIndexOf('.');
if (p > 0) link.Name = name[..p];
}

list.Add(link);
}
Expand All @@ -178,9 +195,15 @@ public Link Parse(String file)
ParseTime();
ParseVersion();

// 去掉后缀
var p = Name.LastIndexOf('.');
if (p > 0) Name = Name[..p];
// 去掉后缀，特殊处理.tar.gz双后缀
var name = Name;
if (name.EndsWithIgnoreCase(".tar.gz"))
Name = name[..^7];
else
{
var p = name.LastIndexOf('.');
if (p > 0) Name = name[..p];
}

// 时间
if (Time.Year < 2000)
Expand Down
27 changes: 24 additions & 3 deletions NewLife.Core/Web/WebClientX.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System.Diagnostics;
using System.Net;
using System.Net.Http;
using System.Security.Cryptography;
using NewLife.Http;
using NewLife.Log;

Expand Down Expand Up @@ -216,21 +217,41 @@ public String DownloadLink(String urls, String name, String destdir)
// 已经提前检查过，这里几乎不可能有文件存在
if (File.Exists(file2))
{
// 如果连接名所表示的文件存在，并且带有时间，那么就智能是它啦
// 如果连接名所表示的文件存在，并且带有时间，那么就只能是它啦
var p = linkName.LastIndexOf("_");
if (p > 0 && (p + 8 + 1 == linkName.Length || p + 14 + 1 == linkName.Length))
{
Log.Info("分析得到文件 {0}，目标文件已存在，无需下载 {1}", linkName, link.Url);
Log.Info("分析得到文件：{0}，目标文件已存在，无需下载：{1}", linkName, link.Url);
return file2;
}
// 校验哈希是否一致
if (!link.Hash.IsNullOrEmpty() && link.Hash.Length == 32)
{
var hash = file2.AsFile().MD5().ToHex();
if (link.Hash.EqualIgnoreCase(hash))
{
Log.Info("分析得到文件：{0}，目标文件已存在，且MD5哈希一致", linkName, link.Url);
return file2;
}
}
if (!link.Hash.IsNullOrEmpty() && link.Hash.Length == 128)
{
using var fs = file2.AsFile().OpenRead();
var hash = SHA512.Create().ComputeHash(fs).ToHex();
if (link.Hash.EqualIgnoreCase(hash))
{
Log.Info("分析得到文件：{0}，目标文件已存在，且SHA512哈希一致", linkName, link.Url);
return file2;
}
}
}

Log.Info("分析得到文件 {0}，准备下载 {1}，保存到 {2}", linkName, link.Url, file2);
// 开始下载文件，注意要提前建立目录，否则会报错
file2 = file2.EnsureDirectory();

var sw = Stopwatch.StartNew();
Task.Run(() => DownloadFileAsync(link.Url, file2)).Wait();
Task.Run(() => DownloadFileAsync(link.Url, file2)).Wait(Timeout);
sw.Stop();

if (File.Exists(file2))
Expand Down
4 changes: 3 additions & 1 deletion Test/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,9 @@ private static void Test1()
{
var client = new WebClientX { Log = XTrace.Log };
//var rs = client.DownloadLink("http://sh03.newlifex.com,http://x.newlifex.com", "ip.gz", "tt/");
var rs = client.DownloadLink("http://sh03.newlifex.com,http://x.newlifex.com", "leaf.png", "tt/");
//var rs = client.DownloadLink("http://sh03.newlifex.com,http://x.newlifex.com", "leaf", "tt/");
var rs = client.DownloadLink("http://sh03.newlifex.com,https://x.newlifex.com/dotNet/8.0.7", "dotnet-runtime-8.0.7-linux-x64", "tt/");
XTrace.WriteLine(rs);
}

private static void Test2()
Expand Down

0 comments on commit 7cf4384

Please sign in to comment.