Merge pull request #233 from AlexCSDev/lib_update

Release 28

LICENSE.md

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019-2022 Aleksey Tsutsey & Contributors
+Copyright (c) 2019-2024 Aleksey Tsutsey & Contributors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

PatreonDownloader.App/PatreonDownloader.App.csproj

@@ -2,14 +2,14 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <PublishSingleFile>true</PublishSingleFile>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="CommandLineParser" Version="2.9.1" />
-    <PackageReference Include="NLog" Version="5.0.0" />
+    <PackageReference Include="NLog" Version="5.3.4" />
   </ItemGroup>
 
   <ItemGroup>

PatreonDownloader.App/Program.cs

@@ -148,7 +148,7 @@ private static async Task<PatreonDownloaderSettings> InitializeSettings(CommandL
             PatreonDownloaderSettings settings = new PatreonDownloaderSettings
             {
                 UrlBlackList = (_configuration["UrlBlackList"] ?? "").ToLowerInvariant().Split("|").ToList(),
-                UserAgent = null,
+                UserAgent = "Patreon/72.2.28 (Android; Android 14; Scale/2.10)",
                 CookieContainer = null,
                 SaveAvatarAndCover = commandLineOptions.SaveAvatarAndCover,
                 SaveDescriptions = commandLineOptions.SaveDescriptions,

PatreonDownloader.App/Properties/AssemblyInfo.cs

@@ -10,7 +10,7 @@
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("")]
 [assembly: AssemblyProduct("Patreon Downloader")]
-[assembly: AssemblyCopyright("Copyright 2019-2023 Aleksey Tsutsey & Contributors")]
+[assembly: AssemblyCopyright("Copyright 2019-2024 Aleksey Tsutsey & Contributors")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
@@ -29,5 +29,5 @@
 //      Build Number
 //      Revision
 //
-[assembly: AssemblyVersion("27.0.0.0")]
-[assembly: AssemblyFileVersion("27.0.0.0")]
+[assembly: AssemblyVersion("28.0.0.0")]
+[assembly: AssemblyFileVersion("28.0.0.0")]

PatreonDownloader.App/settings.json

@@ -1,3 +1,3 @@
 {
-  "UrlBlackList": "patreon.com/posts/|tmblr.co/|t.umblr.com/redirect|mailto:|postybirb.com|picarto.tv|deviantart.com|https://twitter.com|https://steamcommunity.com|http://www.furaffinity.net|https://e621.net/post/show|https://e621.net/posts/|trello.com|https://smutba.se|https://sfmlab.com|http://fav.me|https://inkbunny.net|https://www.pixiv.net/|pixiv.me"
+  "UrlBlackList": "patreon.com/posts/|tmblr.co/|t.umblr.com/redirect|mailto:|postybirb.com|picarto.tv|deviantart.com|https://twitter.com|https://steamcommunity.com|http://www.furaffinity.net|https://e621.net/post/show|https://e621.net/posts/|trello.com|https://smutba.se|https://sfmlab.com|http://fav.me|https://inkbunny.net|https://www.pixiv.net/|pixiv.me|https://x.com|https://www.x.com|http://x.com|http://www.x.com"
 }

PatreonDownloader.Implementation/Models/JSONObjects/Posts.cs

@@ -120,7 +120,7 @@ public class AccessRules
         public List<Data> Data { get; set; }
     }
 
-    public class Attachments
+    public class AttachmentsMedia
     {
         [JsonProperty("data")]
         public List<Data> Data { get; set; }
@@ -192,8 +192,8 @@ public class RootDataRelationships
     {
         [JsonProperty("access_rules")]
         public AccessRules AccessRules { get; set; }
-        [JsonProperty("attachments")]
-        public Attachments Attachments { get; set; }
+        [JsonProperty("attachments_media")]
+        public AttachmentsMedia AttachmentsMedia { get; set; }
         [JsonProperty("audio")]
         public Audio Audio { get; set; }
         [JsonProperty("campaign")]

PatreonDownloader.Implementation/Models/PatreonDownloaderSettings.cs

@@ -50,7 +50,7 @@ public record PatreonDownloaderSettings : UniversalDownloaderPlatformSettings, I
         public bool IsUseLegacyFilenaming { get; init; }
 
         public string LoginPageAddress { get { return "https://www.patreon.com/login"; } }
-        public string LoginCheckAddress { get { return "https://www.patreon.com/api/badges?json-api-version=1.0&include=[]"; } }
+        public string LoginCheckAddress { get { return "https://www.patreon.com/api/badges?json-api-version=1.0&json-api-use-default-includes=false&include=[]"; } }
         public string CaptchaCookieRetrievalAddress { get { return "https://www.patreon.com/home"; } }
         public Uri RemoteBrowserAddress { get; init; }
         public bool IsHeadlessBrowser { get; init; }

PatreonDownloader.Implementation/PatreonCookieValidator.cs

@@ -34,12 +34,6 @@ public async Task ValidateCookies(CookieContainer cookieContainer)
                 throw new CookieValidationException("session_id cookie not found");
             if (cookies["patreon_device_id"] == null)
                 throw new CookieValidationException("patreon_device_id cookie not found");
-            if (cookies["datadome"] == null)
-            {
-                //Some users reported that they don't have datadome cookie and crawling still works fine, so let's ignore that for now.
-                _logger.Warn("Datadome cookie was not found. Usually this is not an issue, but if you are experiencing any issues please make sure to report it via GitHub issues page.");
-                //throw new CookieValidationException("datadome cookie not found");
-            }
 
             string apiResponse = await _webDownloader.DownloadString("https://www.patreon.com/api/current_user");
 

PatreonDownloader.Implementation/PatreonDownloader.Implementation.csproj

@@ -1,16 +1,16 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="HtmlAgilityPack" Version="1.11.43" />
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.65" />
     <PackageReference Include="MimeTypesMap" Version="1.0.8" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="Ninject" Version="3.3.6" />
-    <PackageReference Include="NLog" Version="5.0.0" />
+    <PackageReference Include="NLog" Version="5.3.4" />
   </ItemGroup>
 
   <ItemGroup>

PatreonDownloader.Implementation/PatreonPageCrawler.cs

@@ -31,7 +31,7 @@ internal sealed class PatreonPageCrawler : IPageCrawler
 
         //TODO: Research possibility of not hardcoding this string
         private const string CrawlStartUrl = "https://www.patreon.com/api/posts?" +
-                                             "include=user%2Cattachments%2Ccampaign%2Cpoll.choices%2Cpoll.current_user_responses.user%2Cpoll.current_user_responses.choice%2Cpoll.current_user_responses.poll%2Caccess_rules.tier.null%2Cimages.null%2Caudio.null" +
+                                             "include=user%2Cattachments_media%2Ccampaign%2Cpoll.choices%2Cpoll.current_user_responses.user%2Cpoll.current_user_responses.choice%2Cpoll.current_user_responses.poll%2Caccess_rules.tier.null%2Cimages.null%2Caudio.null" +
                                              "&fields[post]=change_visibility_at%2Ccomment_count%2Ccontent%2Ccurrent_user_can_delete%2Ccurrent_user_can_view%2Ccurrent_user_has_liked%2Cembed%2Cimage%2Cis_paid%2Clike_count%2Cmin_cents_pledged_to_view%2Cpost_file%2Cpost_metadata%2Cpublished_at%2Cpatron_count%2Cpatreon_url%2Cpost_type%2Cpledge_url%2Cthumbnail_url%2Cteaser_text%2Ctitle%2Cupgrade_url%2Curl%2Cwas_posted_by_campaign_owner" +
                                              "&fields[user]=image_url%2Cfull_name%2Curl" +
                                              "&fields[campaign]=show_audio_post_download_links%2Cavatar_photo_url%2Cearnings_visibility%2Cis_nsfw%2Cis_monthly%2Cname%2Curl" +
@@ -126,7 +126,7 @@ private async Task<ParsingResult> ParsePage(string json)
                 {
                     _logger.Warn($"[{jsonEntry.Id}] Current user cannot view this post");
 
-                    string[] skippedAttachments = jsonEntry.Relationships.Attachments?.Data.Select(x => x.Id).ToArray() ?? new string[0];
+                    string[] skippedAttachments = jsonEntry.Relationships.AttachmentsMedia?.Data.Select(x => x.Id).ToArray() ?? new string[0];
                     string[] skippedMedia = jsonEntry.Relationships.Images?.Data.Select(x => x.Id).ToArray() ?? new string[0];
                     _logger.Debug($"[{jsonEntry.Id}] Adding {skippedAttachments.Length} attachments and {skippedMedia.Length} media items to skipped list");
 
@@ -225,20 +225,20 @@ await File.WriteAllTextAsync(
 
                 _logger.Debug($"[{jsonEntry.Id}] Scanning attachment data");
                 //Attachments
-                if(jsonEntry.Relationships.Attachments?.Data != null)
+                if(jsonEntry.Relationships.AttachmentsMedia?.Data != null)
                 {
-                    foreach (var attachment in jsonEntry.Relationships.Attachments.Data)
+                    foreach (var attachment in jsonEntry.Relationships.AttachmentsMedia.Data)
                     {
                         _logger.Debug($"[{jsonEntry.Id} A-{attachment.Id}] Scanning attachment");
-                        if (attachment.Type != "attachment") //sanity check 
+                        if (attachment.Type != "media") //sanity check 
                         {
                             string msg = $"Invalid attachment type for {attachment.Id}!!!";
                             _logger.Fatal($"[{jsonEntry.Id}] {msg}");
                             OnCrawlerMessage(new CrawlerMessageEventArgs(CrawlerMessageType.Error, msg, jsonEntry.Id));
                             continue;
                         }
 
-                        var attachmentData = jsonRoot.Included.FirstOrDefault(x => x.Type == "attachment" && x.Id == attachment.Id);
+                        var attachmentData = jsonRoot.Included.FirstOrDefault(x => x.Type == "media" && x.Id == attachment.Id);
 
                         if (attachmentData == null)
                         {
@@ -249,8 +249,8 @@ await File.WriteAllTextAsync(
                         }
 
                         PatreonCrawledUrl subEntry = (PatreonCrawledUrl)entry.Clone(); ;
-                        subEntry.Url = attachmentData.Attributes.Url;
-                        subEntry.Filename = attachmentData.Attributes.Name;
+                        subEntry.Url = attachmentData.Attributes.DownloadUrl;
+                        subEntry.Filename = attachmentData.Attributes.FileName;
                         subEntry.UrlType = PatreonCrawledUrlType.PostAttachment;
                         subEntry.FileId = attachmentData.Id;
                         crawledUrls.Add(subEntry);

PatreonDownloader.Tests/PatreonDownloader.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>

docs/REMOTEBROWSER.md

@@ -3,7 +3,7 @@
 
 PatreonDownloader has support for using remote browser for situations when using local browser is not possible. (gui-less servers, etc) 
 
-In order to use this feature remote machine should be running compatible version of chromium browser. Required chromium version can be determined by running PatreonDownloader locally and checking .local-chromium subfolder.
+In order to use this feature remote machine should be running compatible version of chromium browser. Required chromium version can be determined by running PatreonDownloader locally and checking `Chrome` subfolder.
 
 Please note that login functionality is disabled while running remote browser mode. Before using remote browser with PatreonDownloader you will need to manually login into your patreon account.