diff --git a/Crawler.Tests/Crawler.Tests.fsproj b/Crawler.Tests/Crawler.Tests.fsproj new file mode 100644 index 0000000..1108118 --- /dev/null +++ b/Crawler.Tests/Crawler.Tests.fsproj @@ -0,0 +1,36 @@ + + + + net7.0 + + false + true + true + + + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive +all + + runtime; build; native; contentfiles; analyzers; buildtransitive +all + + + + + + + + + + + + + diff --git a/Crawler.Tests/CrawlerTest.fs b/Crawler.Tests/CrawlerTest.fs new file mode 100644 index 0000000..c25ff41 --- /dev/null +++ b/Crawler.Tests/CrawlerTest.fs @@ -0,0 +1,83 @@ +module Crawler.Tests + +open NUnit.Framework +open FsUnit +open Moq +open Сrawler.Сrawler +open System.Net.Http + +let informationInTestUrl = + "
\\\\"" + +let fstLink = "" +let sndLink = "" + +[] +let ``Incorrect url test`` () = + let mockClient = new Mock() + + mockClient + .Setup(fun c -> c.GetStringAsync(It.IsAny())) + .Throws() + |> ignore + + let result = + downloadPagesFrom (mockClient.Object, "test_url") |> Async.RunSynchronously + + result.IsNone |> should be True + +[] +let ``Correct input — correct result`` () = + let mockClient = new Mock() + + mockClient + .Setup(fun c -> c.GetStringAsync("test_url")) + .ReturnsAsync(informationInTestUrl) + |> ignore + + mockClient.Setup(fun c -> c.GetStringAsync(fstLink)).ReturnsAsync("text link 1") + |> ignore + + mockClient.Setup(fun c -> c.GetStringAsync(sndLink)).ReturnsAsync("text link 2") + |> ignore + + let result = + downloadPagesFrom (mockClient.Object, "test_url") |> Async.RunSynchronously + + result.IsNone |> should be False + let resultValue = result.Value + + let expected = + seq { + (fstLink, 11) + (sndLink, 11) + } + + resultValue |> should equal expected + +[] +let ``Returns downloaded without errors links information test`` () = + let mockClient = new Mock() + + mockClient + .Setup(fun c -> c.GetStringAsync("test_url")) + .ReturnsAsync(informationInTestUrl) + |> ignore + + mockClient.Setup(fun c -> c.GetStringAsync(fstLink)).ReturnsAsync("text link 1") + |> ignore + + mockClient + .Setup(fun c -> c.GetStringAsync(sndLink)) + .Throws() + |> ignore + + let result = + downloadPagesFrom (mockClient.Object, "test_url") |> Async.RunSynchronously + + result.IsNone |> should be False + let resultValue = result.Value + + let expected = seq { (fstLink, 11) } + + resultValue |> should equal expected diff --git a/Lazy.Tests/Lazy.Tests.fsproj b/Lazy.Tests/Lazy.Tests.fsproj new file mode 100644 index 0000000..179fc3e --- /dev/null +++ b/Lazy.Tests/Lazy.Tests.fsproj @@ -0,0 +1,36 @@ + + + + net7.0 + + false + true + true + + + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive +all + + runtime; build; native; contentfiles; analyzers; buildtransitive +all + + + + + + + + + + + + + diff --git a/Lazy.Tests/LazyTests.fs b/Lazy.Tests/LazyTests.fs new file mode 100644 index 0000000..e109e64 --- /dev/null +++ b/Lazy.Tests/LazyTests.fs @@ -0,0 +1,49 @@ +module Lazy.Tests + +open NUnit.Framework +open FsCheck +open FsUnit +open System.Threading +open Lazy + +[] +let ``Consistent Lazy returns same calculation result`` () = + let consistentLazyChecker (supplier: unit -> int) = + let newLazy = СonsistentLazy(supplier) :> ILazy + let fstResult = newLazy.Get() + let sndResult = newLazy.Get() + let thrdResult = newLazy.Get() + fstResult = sndResult && sndResult = thrdResult + + Check.QuickThrowOnFailure consistentLazyChecker + +let lazies = + [ (fun supplier -> LockLazy(supplier) :> ILazy) + (fun supplier -> LockFreeLazy(supplier) :> ILazy) ] + |> List.map (fun x -> TestCaseData(x)) + +[] +let ``Lock and free lock Lazies execute supplier one time`` (newILazy: (unit -> int) -> ILazy) = + let event = new ManualResetEvent(false) + let mutable count = ref 0 + + let multiThreadLazy = + newILazy (fun () -> + Interlocked.Increment count) + + let task = + async { + event.WaitOne() |> ignore + return multiThreadLazy.Get() + } + + let taskSeq = Seq.init 10 (fun _ -> task) |> Async.Parallel + Thread.Sleep(100) + event.Set() |> ignore + let results = taskSeq |> Async.RunSynchronously + let mutable isEqual = true + + for result in results do + isEqual <- (result = results.[0]) && isEqual + + isEqual |> should be True diff --git a/Lazy/Lazy.fs b/Lazy/Lazy.fs new file mode 100644 index 0000000..be2613b --- /dev/null +++ b/Lazy/Lazy.fs @@ -0,0 +1,51 @@ +namespace Lazy + +open System.Threading + +module Lazy = + /// Interface of Lazy. + type ILazy<'a> = + /// Returns new calculation of supplier value on first call. + /// Result of supplier. + abstract member Get: unit -> 'a + + /// Represents Lazy working with one thread. + type СonsistentLazy<'a>(supplier: unit -> 'a) = + let mutable value: Option<'a> = None + + interface ILazy<'a> with + /// + member this.Get() = + match value with + | None -> + value <- Some(supplier ()) + value.Value + | Some x -> x + + /// Represents Lazy working with multithreads using lock. + type LockLazy<'a>(supplier: unit -> 'a) = + let mutable value: Option<'a> = None + let lockObj = obj () + + interface ILazy<'a> with + /// + member this.Get() = + if value.IsNone then + lock (lockObj) (fun () -> + match value with + | None -> + value <- Some(supplier ()) + value.Value + | Some x -> x) + else value.Value + + /// Represents Lazy working with multithreads using lock free. + type LockFreeLazy<'a>(supplier: unit -> 'a) = + let mutable value: Option<'a> = None + + interface ILazy<'a> with + /// + member this.Get() = + match Interlocked.CompareExchange(&value, Some(supplier ()), None) with + | Some x -> x + | None -> value.Value diff --git a/Lazy/Lazy.fsproj b/Lazy/Lazy.fsproj new file mode 100644 index 0000000..1b8c425 --- /dev/null +++ b/Lazy/Lazy.fsproj @@ -0,0 +1,12 @@ + + + + net7.0 + true + + + + + + + diff --git a/Lazy/Lazy.sln b/Lazy/Lazy.sln new file mode 100644 index 0000000..d89ebec --- /dev/null +++ b/Lazy/Lazy.sln @@ -0,0 +1,31 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 25.0.1706.14 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Lazy", "Lazy.fsproj", "{EDC31770-4B46-45AA-B1C3-E545B53BCD61}" +EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Lazy.Tests", "..\Lazy.Tests\Lazy.Tests.fsproj", "{9E8DBA1A-61A6-4573-A20D-04959A4399C6}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {EDC31770-4B46-45AA-B1C3-E545B53BCD61}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EDC31770-4B46-45AA-B1C3-E545B53BCD61}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EDC31770-4B46-45AA-B1C3-E545B53BCD61}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EDC31770-4B46-45AA-B1C3-E545B53BCD61}.Release|Any CPU.Build.0 = Release|Any CPU + {9E8DBA1A-61A6-4573-A20D-04959A4399C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9E8DBA1A-61A6-4573-A20D-04959A4399C6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9E8DBA1A-61A6-4573-A20D-04959A4399C6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9E8DBA1A-61A6-4573-A20D-04959A4399C6}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {D31E6CAF-FE9C-45C4-9FE3-024C5F8B0609} + EndGlobalSection +EndGlobal diff --git "a/\320\241rawler/Crawler.fs" "b/\320\241rawler/Crawler.fs" new file mode 100644 index 0000000..12b1d81 --- /dev/null +++ "b/\320\241rawler/Crawler.fs" @@ -0,0 +1,55 @@ +namespace Сrawler + +open System.Text.RegularExpressions +open System.Net.Http +open System.Threading.Tasks + +module Сrawler = + type IHttpClient = + abstract member GetStringAsync: string -> Task + + let getHtmlLinks (urlContent: string) = + let regex = new Regex(@"", RegexOptions.IgnoreCase) + let matches = regex.Matches(urlContent) |> Seq.map (fun x -> x.Value) + matches + + let downloadWebPageContent (client: IHttpClient, url: string) : Async> = + async { + try + let! response = client.GetStringAsync(url) |> Async.AwaitTask + return Some response + with :? HttpRequestException -> + printfn "Error downloading URL: %s" url + return None + } + + let printDownloadingResult (results: seq) = + for result in results do + printfn "%s %d" <| "url: " + fst result + "\nAmount of symbols:" <| snd result + + let downloadPagesFrom (client: IHttpClient, url: string) = + async { + let! urlContent = downloadWebPageContent (client, url) + + match urlContent with + | None -> return None + | Some content -> + let downloadingTasks = + getHtmlLinks content + |> Seq.map (fun link -> + async { + let! pageContent = downloadWebPageContent (client, link) + return link, pageContent + }) + |> Async.Parallel + + let! results = downloadingTasks + + let unwrappedResults = + results + |> Seq.filter (fun (x: string * Option) -> (snd x).IsSome) + |> Seq.map (fun (x: string * Option) -> fst x, (snd x).Value.Length) + + printDownloadingResult unwrappedResults + return Some unwrappedResults + } diff --git "a/\320\241rawler/\320\241rawler.fsproj" "b/\320\241rawler/\320\241rawler.fsproj" new file mode 100644 index 0000000..2456c31 --- /dev/null +++ "b/\320\241rawler/\320\241rawler.fsproj" @@ -0,0 +1,17 @@ + + + + net7.0 + true + + + + + + + + + + + + diff --git "a/\320\241rawler/\320\241rawler.sln" "b/\320\241rawler/\320\241rawler.sln" new file mode 100644 index 0000000..645a6cb --- /dev/null +++ "b/\320\241rawler/\320\241rawler.sln" @@ -0,0 +1,31 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 25.0.1706.14 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Сrawler", "Сrawler.fsproj", "{717B2B20-5773-42D6-8147-13E282A5B12E}" +EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Crawler.Tests", "..\Crawler.Tests\Crawler.Tests.fsproj", "{4CC97B9B-17D8-4FB7-803F-084778FE0AA9}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {717B2B20-5773-42D6-8147-13E282A5B12E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {717B2B20-5773-42D6-8147-13E282A5B12E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {717B2B20-5773-42D6-8147-13E282A5B12E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {717B2B20-5773-42D6-8147-13E282A5B12E}.Release|Any CPU.Build.0 = Release|Any CPU + {4CC97B9B-17D8-4FB7-803F-084778FE0AA9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4CC97B9B-17D8-4FB7-803F-084778FE0AA9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4CC97B9B-17D8-4FB7-803F-084778FE0AA9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4CC97B9B-17D8-4FB7-803F-084778FE0AA9}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {DE5DC773-1FC1-4FE5-A564-749994168AEB} + EndGlobalSection +EndGlobal