release 3.0.0

README.md

@@ -21,11 +21,10 @@ DotnetSpider, a .NET Standard web crawling library similar to WebMagic and Scrap
 
 ### OPTIONAL ENVIROMENT
 
-- Run distributed crawler. [Download Redis for windows](https://github.com/MSOpenTech/redis/releases)
+- Distributed crawler. [Download Redis for windows](https://github.com/MSOpenTech/redis/releases)
 - SqlServer.
 - PostgreSQL.
 - MongoDb
-- Cassandra
 
 ### MORE DOCUMENTS
 
@@ -53,7 +52,7 @@ https://github.com/dotnetcore/DotnetSpider/wiki
 
 		private class Spider : EntitySpider
 		{
-			protected override void MyInit(params string[] arguments)
+			protected override void OnInit(params string[] arguments)
 			{
 				var word = "可乐|雪碧";
 				AddStartUrl(string.Format("http://news.baidu.com/ns?word={0}&tn=news&from=news&cl=2&pn=0&rn=20&ct=1", word), new Dictionary<string, dynamic> { { "Keyword", word } });
@@ -130,15 +129,12 @@ NOTE:
 
 ### Storage log and status to database
 
-1. Set SystemConnection in app.config
-2. Update nlog.config like https://github.com/dotnetcore/DotnetSpider/blob/master/src/DotnetSpider.Extension.Test/nlog.config
 
-
-### Web Manager
+### DotnetSpider.Hub
 
 https://github.com/zlzforever/DotnetSpider.Hub
 
-1. Dependences a ci platform forexample i used gitlab-ci right now.
+1. Dependences a ci platform forexample i used teamcity right now.
 2. Dependences Sceduler.NET https://github.com/zlzforever/Scheduler.NET 
 3. More documents continue...
 

src/DotnetSpider.Common.Test/RequestTest.cs

@@ -1,5 +1,6 @@
 using Newtonsoft.Json;
 using System.Collections.Generic;
+using System.Net.Http;
 using Xunit;
 
 namespace DotnetSpider.Common.Test
@@ -24,6 +25,7 @@ public void Request()
 			Assert.Single(request.Properties);
 			Assert.Equal(request.Properties["Test"], "Forever");
 		}
+
 		[Fact(DisplayName = "Request_PutExtra")]
 		public void PutExtra()
 		{

src/DotnetSpider.Common/DotnetSpider.Common.csproj

@@ -3,7 +3,7 @@
     <TargetFrameworks>net40;net45;netstandard2.0</TargetFrameworks>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
     <PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
-    <Version>3.0.0-beta1</Version>
+    <Version>3.0.0</Version>
     <Authors>[email protected];</Authors>
     <AssemblyName>DotnetSpider.Common</AssemblyName>
     <Copyright>Copyright 2018 Lewis Zou</Copyright>
@@ -18,9 +18,11 @@
   </PropertyGroup>
   <ItemGroup Condition=" '$(TargetFramework)' == 'net40' ">
     <Reference Include="Microsoft.CSharp" />
+    <PackageReference Include=" Microsoft.Net.Http" Version="2.2.29" />
   </ItemGroup>
   <ItemGroup Condition=" '$(TargetFramework)' == 'net45' ">
     <Reference Include="Microsoft.CSharp" />
+    <PackageReference Include="System.Net.Http" Version="4.3.3" />
   </ItemGroup>
   <ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' ">
     <PackageReference Include="Microsoft.CSharp" Version="4.5.0" />

src/DotnetSpider.Common/Request.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Collections.Generic;
+using System.Net.Http;
 using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
 
 namespace DotnetSpider.Common
 {
@@ -32,7 +34,7 @@ public class Request : IDisposable
 		/// <summary>
 		/// 请求链接的方法
 		/// </summary>
-		public HttpMethod Method { get; set; }
+		public HttpMethod Method { get; set; } = HttpMethod.Get;
 
 		/// <summary>
 		/// 链接的优先级, 仅用于优先级队列

src/DotnetSpider.Common/TaskName.cs

@@ -5,7 +5,7 @@ namespace DotnetSpider.Common
 	/// <summary>
 	/// 任务名称
 	/// </summary>
-	[AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited = false)]
+	[AttributeUsage(AttributeTargets.Class, Inherited = false)]
 	public class TaskName : Attribute
 	{
 		/// <summary>
@@ -14,7 +14,6 @@ public class TaskName : Attribute
 		public string Name
 		{
 			get;
-			private set;
 		}
 
 		/// <summary>

src/DotnetSpider.Core.Test/PageTest.cs

@@ -1,6 +1,7 @@
 using DotnetSpider.Common;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net.Http;
 using Xunit;
 
 namespace DotnetSpider.Core.Test

src/DotnetSpider.Core.Test/SpiderTest.cs

@@ -153,7 +153,7 @@ public void WhenNoStartUrl()
 
 		internal class TestPipeline : BasePipeline
 		{
-			public override void Process(IEnumerable<ResultItems> resultItems, ILogger logger, dynamic sender = null)
+			public override void Process(IList<ResultItems> resultItems, ILogger logger, dynamic sender = null)
 			{
 				foreach (var resultItem in resultItems)
 				{
@@ -276,7 +276,7 @@ protected override Response DowloadContent(Request request)
 
 		internal class FastExitPipeline : BasePipeline
 		{
-			public override void Process(IEnumerable<ResultItems> resultItems, ILogger logger, dynamic sender = null)
+			public override void Process(IList<ResultItems> resultItems, ILogger logger, dynamic sender = null)
 			{
 				File.AppendAllLines("FastExit_Result.txt", new[] { resultItems.First().Request.Url.ToString() });
 			}

src/DotnetSpider.Core/DotnetSpider.Core.csproj

@@ -1,10 +1,9 @@
 <Project Sdk="Microsoft.NET.Sdk">
-
   <PropertyGroup>
     <TargetFrameworks>net40;net45;netstandard2.0</TargetFrameworks>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
     <PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
-    <Version>3.0.0-beta1</Version>
+    <Version>3.0.0</Version>
     <Authors>[email protected];</Authors>
     <AssemblyName>DotnetSpider.Core</AssemblyName>
     <Copyright>Copyright 2018 Lewis Zou</Copyright>

src/DotnetSpider.Core/Env.cs

@@ -16,11 +16,6 @@ namespace DotnetSpider.Core
 	/// </summary>
 	public static class Env
 	{
-		/// <summary>
-		/// TODO: 原则上此版本号与Nuget包版本号同步, 但是不知道有什么好的自动化更新方法
-		/// </summary>
-		public const string Version = "2.0.21";
-
 		/// <summary>
 		/// 从配置文件中读取默认Redis连接字符串的关键字
 		/// </summary>

src/DotnetSpider.Core/ISpider.cs

@@ -9,7 +9,7 @@ namespace DotnetSpider.Core
 	/// <summary>
 	/// 爬虫接口定义
 	/// </summary>
-	public interface ISpider : IDisposable, IControllable, IAppBase
+	public interface ISpider : IDisposable, IAppBase
 	{
 		/// <summary>
 		/// 采集站点的信息配置

src/DotnetSpider.Core/Infrastructure/Description.cs

@@ -11,26 +11,26 @@ public class Description : Attribute
 		/// <summary>
 		/// 任务所有者
 		/// </summary>
-		public string Owner;
+		public string Owner { get; set; }
 
 		/// <summary>
 		/// 程序的开发者
 		/// </summary>
-		public string Developer;
+		public string Developer{ get; set; }
 
 		/// <summary>
 		/// 程序的开发时间
 		/// </summary>
-		public string Date;
+		public string Date{ get; set; }
 
 		/// <summary>
 		/// 任务主题
 		/// </summary>
-		public string Subject;
+		public string Subject{ get; set; }
 
 		/// <summary>
 		/// 联系邮箱
 		/// </summary>
-		public string Email;
+		public string Email{ get; set; }
 	}
 }