Skip to content

Commit

Permalink
Fix attachments, add user agent, update UDP
Browse files Browse the repository at this point in the history
  • Loading branch information
AlexCSDev committed Oct 22, 2024
1 parent 8806efa commit 1d3412b
Show file tree
Hide file tree
Showing 4 changed files with 13 additions and 13 deletions.
2 changes: 1 addition & 1 deletion PatreonDownloader.App/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ private static async Task<PatreonDownloaderSettings> InitializeSettings(CommandL
PatreonDownloaderSettings settings = new PatreonDownloaderSettings
{
UrlBlackList = (_configuration["UrlBlackList"] ?? "").ToLowerInvariant().Split("|").ToList(),
UserAgent = null,
UserAgent = "Patreon/72.2.28 (Android; Android 14; Scale/2.10)",
CookieContainer = null,
SaveAvatarAndCover = commandLineOptions.SaveAvatarAndCover,
SaveDescriptions = commandLineOptions.SaveDescriptions,
Expand Down
6 changes: 3 additions & 3 deletions PatreonDownloader.Implementation/Models/JSONObjects/Posts.cs
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ public class AccessRules
public List<Data> Data { get; set; }
}

public class Attachments
public class AttachmentsMedia
{
[JsonProperty("data")]
public List<Data> Data { get; set; }
Expand Down Expand Up @@ -192,8 +192,8 @@ public class RootDataRelationships
{
[JsonProperty("access_rules")]
public AccessRules AccessRules { get; set; }
[JsonProperty("attachments")]
public Attachments Attachments { get; set; }
[JsonProperty("attachments_media")]
public AttachmentsMedia AttachmentsMedia { get; set; }
[JsonProperty("audio")]
public Audio Audio { get; set; }
[JsonProperty("campaign")]
Expand Down
16 changes: 8 additions & 8 deletions PatreonDownloader.Implementation/PatreonPageCrawler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ internal sealed class PatreonPageCrawler : IPageCrawler

//TODO: Research possibility of not hardcoding this string
private const string CrawlStartUrl = "https://www.patreon.com/api/posts?" +
"include=user%2Cattachments%2Ccampaign%2Cpoll.choices%2Cpoll.current_user_responses.user%2Cpoll.current_user_responses.choice%2Cpoll.current_user_responses.poll%2Caccess_rules.tier.null%2Cimages.null%2Caudio.null" +
"include=user%2Cattachments_media%2Ccampaign%2Cpoll.choices%2Cpoll.current_user_responses.user%2Cpoll.current_user_responses.choice%2Cpoll.current_user_responses.poll%2Caccess_rules.tier.null%2Cimages.null%2Caudio.null" +
"&fields[post]=change_visibility_at%2Ccomment_count%2Ccontent%2Ccurrent_user_can_delete%2Ccurrent_user_can_view%2Ccurrent_user_has_liked%2Cembed%2Cimage%2Cis_paid%2Clike_count%2Cmin_cents_pledged_to_view%2Cpost_file%2Cpost_metadata%2Cpublished_at%2Cpatron_count%2Cpatreon_url%2Cpost_type%2Cpledge_url%2Cthumbnail_url%2Cteaser_text%2Ctitle%2Cupgrade_url%2Curl%2Cwas_posted_by_campaign_owner" +
"&fields[user]=image_url%2Cfull_name%2Curl" +
"&fields[campaign]=show_audio_post_download_links%2Cavatar_photo_url%2Cearnings_visibility%2Cis_nsfw%2Cis_monthly%2Cname%2Curl" +
Expand Down Expand Up @@ -126,7 +126,7 @@ private async Task<ParsingResult> ParsePage(string json)
{
_logger.Warn($"[{jsonEntry.Id}] Current user cannot view this post");

string[] skippedAttachments = jsonEntry.Relationships.Attachments?.Data.Select(x => x.Id).ToArray() ?? new string[0];
string[] skippedAttachments = jsonEntry.Relationships.AttachmentsMedia?.Data.Select(x => x.Id).ToArray() ?? new string[0];
string[] skippedMedia = jsonEntry.Relationships.Images?.Data.Select(x => x.Id).ToArray() ?? new string[0];
_logger.Debug($"[{jsonEntry.Id}] Adding {skippedAttachments.Length} attachments and {skippedMedia.Length} media items to skipped list");

Expand Down Expand Up @@ -225,20 +225,20 @@ await File.WriteAllTextAsync(

_logger.Debug($"[{jsonEntry.Id}] Scanning attachment data");
//Attachments
if(jsonEntry.Relationships.Attachments?.Data != null)
if(jsonEntry.Relationships.AttachmentsMedia?.Data != null)
{
foreach (var attachment in jsonEntry.Relationships.Attachments.Data)
foreach (var attachment in jsonEntry.Relationships.AttachmentsMedia.Data)
{
_logger.Debug($"[{jsonEntry.Id} A-{attachment.Id}] Scanning attachment");
if (attachment.Type != "attachment") //sanity check
if (attachment.Type != "media") //sanity check
{
string msg = $"Invalid attachment type for {attachment.Id}!!!";
_logger.Fatal($"[{jsonEntry.Id}] {msg}");
OnCrawlerMessage(new CrawlerMessageEventArgs(CrawlerMessageType.Error, msg, jsonEntry.Id));
continue;
}

var attachmentData = jsonRoot.Included.FirstOrDefault(x => x.Type == "attachment" && x.Id == attachment.Id);
var attachmentData = jsonRoot.Included.FirstOrDefault(x => x.Type == "media" && x.Id == attachment.Id);

if (attachmentData == null)
{
Expand All @@ -249,8 +249,8 @@ await File.WriteAllTextAsync(
}

PatreonCrawledUrl subEntry = (PatreonCrawledUrl)entry.Clone(); ;
subEntry.Url = attachmentData.Attributes.Url;
subEntry.Filename = attachmentData.Attributes.Name;
subEntry.Url = attachmentData.Attributes.DownloadUrl;
subEntry.Filename = attachmentData.Attributes.FileName;
subEntry.UrlType = PatreonCrawledUrlType.PostAttachment;
subEntry.FileId = attachmentData.Id;
crawledUrls.Add(subEntry);
Expand Down

0 comments on commit 1d3412b

Please sign in to comment.