Skip to content

Commit

Permalink
1.修复当只有一页数据的时候，totalPage获取不到的bug。2.增加判定已下载的条件
Browse files Browse the repository at this point in the history
  • Loading branch information
wangzheng committed Jan 2, 2024
1 parent fde463b commit 043b950
Show file tree
Hide file tree
Showing 4 changed files with 69 additions and 6 deletions.
8 changes: 8 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,14 @@



# 软件优势

开源的不开源的微博下载工具很多，但是大部分都有一个致命的缺点，就是下载的图片，并没有修改日期为发博时间。就是当我想按照日期筛选排序的时候，就很尴尬了。他们的日期一般都是默认图片下载日期，这会给使用者造成很大的困扰。

本软件基于这一点，在下载完图片后，自动获取发博时间（年月日时分秒），然后将图片的创建日期、修改日期、访问日期都修改为此时间。虽然就是一个很简单的功能，但是为后期图片分析节约了大量的时间。



# 使用说明

获取微博用户uid以及web版微博Cookie，填入到软件根目录的Settings.json中即可。
Expand Down
10 changes: 9 additions & 1 deletion WeiboAlbumDownloader/MainWindow.xaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,14 @@
</EventTrigger>
</Border.Triggers>
</Border>
<TextBlock
x:Name="TextBlock_UID"
Margin="16,16,16,0"
HorizontalAlignment="Center"
d:Text="uid"
FontSize="22"
FontWeight="Bold"
Foreground="#FFBF00" />
<TextBlock
x:Name="TextBlock_NickName"
Margin="16"
Expand All @@ -165,7 +173,7 @@
ScrollViewer.HorizontalScrollBarVisibility="Disabled">
<ListView.ContextMenu>
<ContextMenu>
<MenuItem Header="复制" Click="ListView_CopyLog"/>
<MenuItem Click="ListView_CopyLog" Header="复制" />
</ContextMenu>
</ListView.ContextMenu>
<ListView.ItemTemplate>
Expand Down
51 changes: 48 additions & 3 deletions WeiboAlbumDownloader/MainWindow.xaml.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ public partial class MainWindow : MicaWindow
private SettingsModel settings;
private string cookie;
private List<string> uids = new List<string>();
//用来跳过到下一个uid的计数。如果当前uid下载的时候已存在文件超过此计数，则判定下载过了。
private int countDownloadedSkipToNextUser;
private CancellationTokenSource cancellationTokenSource;
public ObservableCollection<MessageModel> Messages { get; set; } = new ObservableCollection<MessageModel>();

Expand Down Expand Up @@ -84,6 +86,7 @@ private async Task Start()

await Task.Run(async () =>
{
bool isSkip = false;
foreach (var userId in uids)
{
if (string.IsNullOrEmpty(userId))
Expand All @@ -93,26 +96,36 @@ await Task.Run(async () =>
else
{
string nickName = string.Empty;
countDownloadedSkipToNextUser = 0;
isSkip = false;
AppendLog("开始下载" + userId, MessageEnum.Info);

//源是weibo.com的时候，获取的是获取相册列表，json格式
if (isFromWeiboCom)
{
string albumsUrl = $"https://photo.weibo.com/albums/get_all?uid={userId}&page=1";
TextBlock_UID?.Dispatcher.InvokeAsync(() =>
{
TextBlock_UID.Text = userId;
});

var albums = await HttpHelper.GetAsync<UserAlbumModel>(albumsUrl, isFromWeiboCom, cookie);
Directory.CreateDirectory(downloadFolder + userId);
if (albums != null)
{
foreach (var item in albums?.data?.album_list)
{
//if (item.type != 3)
// continue;
if (isSkip)
break;

Directory.CreateDirectory(downloadFolder + userId + "//" + item.caption);

int page = 1;
while (true)
{
if (isSkip)
break;

string albumUrl = $"https://photo.weibo.com/photos/get_all?uid={userId}&album_id={item.album_id}&count={countPerPage}&page={page}&type={item.type}";
var photos = await HttpHelper.GetAsync<AlbumDetailModel>(albumUrl, isFromWeiboCom, cookie);
if (photos != null && photos.data?.photo_list != null && photos.data?.photo_list.Count > 0)
Expand Down Expand Up @@ -149,6 +162,7 @@ await Task.Run(async () =>
if (File.Exists(fileName))
{
AppendLog("文件已存在，跳过下载" + fileName, MessageEnum.Warning);
countDownloadedSkipToNextUser++;
await Task.Delay(500);
continue;
}
Expand All @@ -175,6 +189,14 @@ await Task.Run(async () =>
break;
}

//已存在的文件超过设置值，判定该用户下载过了
if (settings.CountDownloadedSkipToNextUser > 0 && countDownloadedSkipToNextUser > settings.CountDownloadedSkipToNextUser)
{
AppendLog($"已存在的文件超过设置值{countDownloadedSkipToNextUser}，跳到下一个用户", MessageEnum.Info);
isSkip = true;
break;
}

page++;
//通过加入随机等待避免被限制。爬虫速度过快容易被系统限制(一段时间后限制会自动解除)，加入随机等待模拟人的操作，可降低被系统限制的风险。默认是每爬取1到5页随机等待6到10秒，如果仍然被限，可适当增加sleep时间
Random rd = new Random();
Expand All @@ -195,6 +217,9 @@ await Task.Run(async () =>
Directory.CreateDirectory(downloadFolder + userId + "//" + "微博配图");
while (true)
{
if (isSkip)
break;

//filter，0-全部；1-原创；2-图片
//https://weibo.cn/xxxxxxxxxxxxx?page=2
//https://weibo.cn/xxxxxxxxxxxxx/profile?page=2
Expand All @@ -212,7 +237,8 @@ await Task.Run(async () =>
totalPage = Convert.ToInt32(totalPageHtml[0].Attributes["value"].Value);
}
}
if (totalPage == -1)
//当只有一页数据的时候，totalPage获取不到。需要叠加判定doc.DocumentNode.Descendants("div").Where(x => x.Attributes["class"]?.Value == "c").ToList().Count
if (totalPage == -1 && doc.DocumentNode.Descendants("div").Where(x => x.Attributes["class"]?.Value == "c").ToList().Count == 0)
{
AppendLog("获取总页数失败，请重新登录https://weibo.cn/获取Cookie重试", MessageEnum.Error);
return;
Expand Down Expand Up @@ -251,6 +277,7 @@ await Task.Run(async () =>
Image_Head.ImageSource = bi;

//Image_Head.ImageSource = new BitmapImage(new Uri(fileName));
TextBlock_UID.Text = userId;
TextBlock_NickName.Text = nickName;
TextBlock_WeiboDesc.Text = weiboDesc;
});
Expand All @@ -262,6 +289,9 @@ await Task.Run(async () =>
var nodes = doc.DocumentNode.Descendants("div").Where(x => x.Attributes["class"]?.Value == "c").ToList();
foreach (var node in nodes)
{
if (isSkip)
break;

if (cancellationTokenSource.IsCancellationRequested)
{
AppendLog("用户手动终止。", MessageEnum.Info);
Expand Down Expand Up @@ -296,6 +326,9 @@ await Task.Run(async () =>
var list = doc1.DocumentNode.Descendants("a").ToList();
foreach (var item in list)
{
if (isSkip)
break;

if (item.InnerText.Contains("组图共"))
{
photoListUrl = item.Attributes["href"].Value;
Expand Down Expand Up @@ -372,6 +405,9 @@ await Task.Run(async () =>
int photoCount = 1;
foreach (var item in originalPics)
{
if (isSkip)
break;

if (string.IsNullOrEmpty(item))
continue;

Expand All @@ -386,10 +422,19 @@ await Task.Run(async () =>
else
fileName += ".jpg";
Debug.WriteLine(fileName);

//已存在的文件超过设置值，判定该用户下载过了
if (settings.CountDownloadedSkipToNextUser > 0 && countDownloadedSkipToNextUser > settings.CountDownloadedSkipToNextUser)
{
AppendLog($"已存在的文件超过设置值{countDownloadedSkipToNextUser}，跳到下一个用户", MessageEnum.Info);
isSkip = true;
}

//已经下载过的跳过
if (File.Exists(fileName))
{
AppendLog("文件已存在，跳过下载" + fileName, MessageEnum.Warning);
countDownloadedSkipToNextUser++;
await Task.Delay(500);
continue;
}
Expand Down
6 changes: 4 additions & 2 deletions WeiboAlbumDownloader/Models/SettingsModel.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@ public class SettingsModel
//推送到微信，填了就会发送
public string PushPlusToken { get; set; }
//是否开启Crontab定时任务
public bool EnableCrontab { get; set; }
public bool EnableCrontab { get; set; } = true;
//Crontab定时任务
public string Crontab { get; set; }
public string Crontab { get; set; } = "14 2 * * *";
//用来跳过到下一个uid的计数。如果当前uid下载的时候已存在文件超过此计数，则判定下载过了。-1表示不判定
public int CountDownloadedSkipToNextUser { get; set; } = 20;
}
}

0 comments on commit 043b950

Please sign in to comment.