Merge branch 'kb_development' into kb_lts

app/controllers/api/v1/accounts_controller.rb

@@ -36,7 +36,7 @@ def create
 
   def follow
     follow  = FollowService.new.call(current_user.account, @account, reblogs: params.key?(:reblogs) ? truthy_param?(:reblogs) : nil, notify: params.key?(:notify) ? truthy_param?(:notify) : nil, languages: params.key?(:languages) ? params[:languages] : nil, with_rate_limit: true)
-    options = @account.locked? || current_user.account.silenced? ? {} : { following_map: { @account.id => { reblogs: follow.show_reblogs?, notify: follow.notify?, languages: follow.languages } }, requested_map: { @account.id => false } }
+    options = @account.locked? || current_user.account.silenced? || (current_user.account.bot? && @account.user&.setting_lock_follow_from_bot) ? {} : { following_map: { @account.id => { reblogs: follow.show_reblogs?, notify: follow.notify?, languages: follow.languages } }, requested_map: { @account.id => false } }
 
     render json: @account, serializer: REST::RelationshipSerializer, relationships: relationships(**options)
   end

app/helpers/context_helper.rb

@@ -29,6 +29,7 @@ module ContextHelper
     limited_scope: { 'kmyblue' => 'http://kmy.blue/ns#', 'limitedScope' => { '@id' => 'kmyblue:limitedScope', '@type' => '@id' } },
     other_setting: { 'fedibird' => 'http://fedibird.com/ns#', 'otherSetting' => 'fedibird:otherSetting' },
     references: { 'fedibird' => 'http://fedibird.com/ns#', 'references' => { '@id' => 'fedibird:references', '@type' => '@id' } },
+    quote_uri: { 'fedibird' => 'http://fedibird.com/ns#', 'quoteUri' => 'fedibird:quoteUri' },
     olm: {
       'toot' => 'http://joinmastodon.org/ns#', 'Device' => 'toot:Device', 'Ed25519Signature' => 'toot:Ed25519Signature', 'Ed25519Key' => 'toot:Ed25519Key', 'Curve25519Key' => 'toot:Curve25519Key', 'EncryptedMessage' => 'toot:EncryptedMessage', 'publicKeyBase64' => 'toot:publicKeyBase64', 'deviceId' => 'toot:deviceId',
       'claim' => { '@type' => '@id', '@id' => 'toot:claim' },

app/lib/activitypub/activity/follow.rb

@@ -30,7 +30,7 @@ def perform
 
     follow_request = FollowRequest.create!(account: @account, target_account: target_account, uri: @json['id'])
 
-    if target_account.locked? || @account.silenced? || block_straight_follow?
+    if target_account.locked? || @account.silenced? || block_straight_follow? || (@account.bot? && target_account.user&.setting_lock_follow_from_bot)
       LocalNotificationWorker.perform_async(target_account.id, follow_request.id, 'FollowRequest', 'follow_request')
     else
       AuthorizeFollowService.new.call(@account, target_account)

app/lib/activitypub/case_transform.rb

@@ -2,6 +2,11 @@
 
 module ActivityPub::CaseTransform
   class << self
+    NO_CONVERT_VALUES = %w(
+      _misskey_content
+      _misskey_quote
+    ).freeze
+
     def camel_lower_cache
       @camel_lower_cache ||= {}
     end
@@ -12,7 +17,9 @@ def camel_lower(value)
       when Hash then value.deep_transform_keys! { |key| camel_lower(key) }
       when Symbol then camel_lower(value.to_s).to_sym
       when String
-        camel_lower_cache[value] ||= if value.start_with?('_:')
+        camel_lower_cache[value] ||= if NO_CONVERT_VALUES.include?(value)
+                                       value
+                                     elsif value.start_with?('_:')
                                        "_:#{value.delete_prefix('_:').underscore.camelize(:lower)}"
                                      else
                                        value.underscore.camelize(:lower)

app/models/concerns/has_user_settings.rb

@@ -127,6 +127,10 @@ def setting_link_preview
     settings['link_preview']
   end
 
+  def setting_single_ref_to_quote
+    settings['single_ref_to_quote']
+  end
+
   def setting_dtl_force_with_tag
     settings['dtl_force_with_tag']&.to_sym || :none
   end
@@ -235,6 +239,10 @@ def setting_disallow_unlisted_public_searchability
     settings['disallow_unlisted_public_searchability']
   end
 
+  def setting_lock_follow_from_bot
+    settings['lock_follow_from_bot']
+  end
+
   def allows_report_emails?
     settings['notification_emails.report']
   end

app/models/status_reference.rb

@@ -17,15 +17,10 @@ class StatusReference < ApplicationRecord
 
   has_one :notification, as: :activity, dependent: :destroy
 
-  validate :validate_status_visibilities
   after_commit :reset_parent_cache
 
   private
 
-  def validate_status_visibilities
-    raise Mastodon::ValidationError, I18n.t('status_references.errors.invalid_status_visibilities') if [:public, :public_unlisted, :unlisted, :login].exclude?(target_status.visibility.to_sym)
-  end
-
   def reset_parent_cache
     Rails.cache.delete("statuses/#{status_id}")
     Rails.cache.delete("statuses/#{target_status_id}")

app/models/user_settings.rb

@@ -40,6 +40,8 @@ class KeyError < Error; end
   setting :unsafe_limited_distribution, default: false
   setting :dtl_force_with_tag, default: :none, in: %w(full searchability none)
   setting :dtl_force_subscribable, default: false
+  setting :lock_follow_from_bot, default: false
+  setting :single_ref_to_quote, default: false
 
   setting_inverse_alias :indexable, :noindex
 

app/serializers/activitypub/note_serializer.rb

@@ -3,24 +3,27 @@
 class ActivityPub::NoteSerializer < ActivityPub::Serializer
   include FormattingHelper
 
-  context_extensions :atom_uri, :conversation, :sensitive, :voters_count, :searchable_by, :references, :limited_scope
+  context_extensions :atom_uri, :conversation, :sensitive, :voters_count, :searchable_by, :references, :limited_scope, :quote_uri
 
   attributes :id, :type, :summary,
              :in_reply_to, :published, :url,
              :attributed_to, :to, :cc, :sensitive,
              :atom_uri, :in_reply_to_atom_uri,
              :conversation, :searchable_by, :limited_scope
 
-  attribute :references, if: :not_private_post?
-
   attribute :content
   attribute :content_map, if: :language?
   attribute :updated, if: :edited?
 
+  attribute :quote_uri, if: :quote?
+  attribute :misskey_quote, key: :_misskey_quote, if: :quote?
+  attribute :misskey_content, key: :_misskey_content, if: :quote?
+
   has_many :virtual_attachments, key: :attachment
   has_many :virtual_tags, key: :tag
 
   has_one :replies, serializer: ActivityPub::CollectionSerializer, if: :local?
+  has_one :references, serializer: ActivityPub::CollectionSerializer
 
   has_many :poll_options, key: :one_of, if: :poll_and_not_multiple?
   has_many :poll_options, key: :any_of, if: :poll_and_multiple?
@@ -67,7 +70,19 @@ def replies
   end
 
   def references
-    ActivityPub::TagManager.instance.references_uri_for(object)
+    refs = object.references.reorder(id: :asc).take(5).pluck(:id, :uri)
+    last_id = refs.last&.first
+
+    ActivityPub::CollectionPresenter.new(
+      type: :unordered,
+      id: ActivityPub::TagManager.instance.references_uri_for(object),
+      first: ActivityPub::CollectionPresenter.new(
+        type: :unordered,
+        part_of: ActivityPub::TagManager.instance.references_uri_for(object),
+        items: refs.map(&:second),
+        next: last_id ? ActivityPub::TagManager.instance.references_uri_for(object, page: true, min_id: last_id) : ActivityPub::TagManager.instance.references_uri_for(object, page: true, only_other_accounts: true)
+      )
+    )
   end
 
   def language?
@@ -156,8 +171,20 @@ def local?
     object.account.local?
   end
 
-  def not_private_post?
-    !object.private_visibility? && !object.direct_visibility? && !object.limited_visibility?
+  def quote?
+    object.references.count == 1 && object.account.user&.settings&.[]('single_ref_to_quote')
+  end
+
+  def quote_uri
+    ActivityPub::TagManager.instance.uri_for(object.references.first)
+  end
+
+  def misskey_quote
+    quote_uri
+  end
+
+  def misskey_content
+    object.text
   end
 
   def poll_options

app/services/follow_service.rb

@@ -36,7 +36,7 @@ def call(source_account, target_account, options = {})
     # and the feeds are being merged
     mark_home_feed_as_partial! if @source_account.not_following_anyone?
 
-    if (@target_account.locked? && !@options[:bypass_locked]) || @source_account.silenced? || @target_account.activitypub?
+    if (@target_account.locked? && !@options[:bypass_locked]) || @source_account.silenced? || @target_account.activitypub? || (@source_account.bot? && @target_account.user&.setting_lock_follow_from_bot)
       request_follow!
     elsif @target_account.local?
       direct_follow!

app/services/notify_service.rb

@@ -9,6 +9,7 @@ class NotifyService < BaseService
     update
     poll
     emoji_reaction
+    status_reference
     warning
   ).freeze
 

app/services/post_status_service.rb

@@ -188,7 +188,7 @@ def postprocess_status!
 
     process_hashtags_service.call(@status)
     Trends.tags.register(@status)
-    ProcessReferencesService.perform_worker_async(@status, @reference_ids, [])
+    ProcessReferencesService.call_service(@status, @reference_ids, [])
     LinkCrawlWorker.perform_async(@status.id)
     DistributionWorker.perform_async(@status.id)
     ActivityPub::DistributionWorker.perform_async(@status.id)

app/services/process_references_service.rb

@@ -3,30 +3,39 @@
 class ProcessReferencesService < BaseService
   include Payloadable
   include FormattingHelper
+  include Redisable
+  include Lockable
 
   DOMAIN = ENV['WEB_DOMAIN'] || ENV.fetch('LOCAL_DOMAIN', nil)
   REFURL_EXP = /(RT|QT|BT|RN|RE)((:|;)?\s+|:|;)(#{URI::DEFAULT_PARSER.make_regexp(%w(http https))})/
   MAX_REFERENCES = 5
 
-  def call(status, reference_parameters, urls: nil)
+  def call(status, reference_parameters, urls: nil, fetch_remote: true, no_fetch_urls: nil)
     @status = status
     @reference_parameters = reference_parameters || []
     @urls = urls || []
+    @no_fetch_urls = no_fetch_urls || []
+    @fetch_remote = fetch_remote
+    @again = false
 
-    @references_count = old_references.size
+    with_redis_lock("process_status_refs:#{@status.id}") do
+      @references_count = old_references.size
 
-    return unless added_references.size.positive? || removed_references.size.positive?
+      if added_references.size.positive? || removed_references.size.positive?
+        StatusReference.transaction do
+          remove_old_references
+          add_references
 
-    StatusReference.transaction do
-      remove_old_references
-      add_references
+          @status.save!
+        end
 
-      @status.save!
-    end
+        create_notifications!
+      end
 
-    Rails.cache.delete("status_reference:#{@status.id}")
+      Rails.cache.delete("status_reference:#{@status.id}")
+    end
 
-    create_notifications!
+    launch_worker if @again
   end
 
   def self.need_process?(status, reference_parameters, urls)
@@ -37,7 +46,13 @@ def self.perform_worker_async(status, reference_parameters, urls)
     return unless need_process?(status, reference_parameters, urls)
 
     Rails.cache.write("status_reference:#{status.id}", true, expires_in: 10.minutes)
-    ProcessReferencesWorker.perform_async(status.id, reference_parameters, urls)
+    ProcessReferencesWorker.perform_async(status.id, reference_parameters, urls, [])
+  end
+
+  def self.call_service(status, reference_parameters, urls)
+    return unless need_process?(status, reference_parameters, urls)
+
+    ProcessReferencesService.new.call(status, reference_parameters || [], urls: urls || [], fetch_remote: false)
   end
 
   private
@@ -59,14 +74,22 @@ def removed_references
   end
 
   def scan_text!
-    text = @status.account.local? ? @status.text : @status.text.gsub(%r{</?[^>]*>}, '')
-    @scan_text = fetch_statuses!(text.scan(REFURL_EXP).pluck(3).uniq).map(&:id).uniq.filter { |status_id| !status_id.zero? }
+    text = extract_status_plain_text(@status)
+    statuses = fetch_statuses!(text.scan(REFURL_EXP).pluck(3).uniq)
+
+    @again = true if !@fetch_remote && statuses.any?(&:nil?)
+
+    @scan_text = statuses.compact.map(&:id).uniq.filter { |status_id| !status_id.zero? }
   end
 
   def fetch_statuses!(urls)
-    (urls + @urls)
-      .map { |url| ResolveURLService.new.call(url, on_behalf_of: @status.account) }
-      .filter { |status| status }
+    target_urls = urls + @urls
+
+    target_urls.map do |url|
+      status = ResolveURLService.new.call(url, on_behalf_of: @status.account, fetch_remote: @fetch_remote && @no_fetch_urls.exclude?(url))
+      @no_fetch_urls << url if !@fetch_remote && status.present?
+      status
+    end
   end
 
   def add_references
@@ -106,4 +129,8 @@ def remove_old_references
       @references_count -= 1
     end
   end
+
+  def launch_worker
+    ProcessReferencesWorker.perform_async(@status.id, @reference_parameters, @urls, @no_fetch_urls)
+  end
 end

app/services/resolve_url_service.rb

@@ -6,13 +6,13 @@ class ResolveURLService < BaseService
 
   USERNAME_STATUS_RE = %r{/@(?<username>#{Account::USERNAME_RE})/(?<status_id>[0-9]+)\Z}
 
-  def call(url, on_behalf_of: nil)
+  def call(url, on_behalf_of: nil, fetch_remote: true)
     @url          = url
     @on_behalf_of = on_behalf_of
 
     if local_url?
       process_local_url
-    elsif !fetched_resource.nil?
+    elsif fetch_remote && !fetched_resource.nil?
       process_url
     else
       process_url_from_db

app/services/update_status_service.rb

@@ -162,7 +162,7 @@ def reset_preview_card!
   def update_references!
     reference_ids = (@options[:status_reference_ids] || []).map(&:to_i).filter(&:positive?)
 
-    ProcessReferencesService.perform_worker_async(@status, reference_ids, [])
+    ProcessReferencesService.call_service(@status, reference_ids, [])
   end
 
   def update_metadata!

app/views/settings/preferences/other/show.html.haml

@@ -11,6 +11,12 @@
     .fields-group
       = ff.input :aggregate_reblogs, wrapper: :with_label, recommended: true, label: I18n.t('simple_form.labels.defaults.setting_aggregate_reblogs'), hint: I18n.t('simple_form.hints.defaults.setting_aggregate_reblogs')
 
+    .fields-group
+      = ff.input :lock_follow_from_bot, wrapper: :with_label, kmyblue: true, label: I18n.t('simple_form.labels.defaults.setting_lock_follow_from_bot')
+
+    .fields-group
+      = ff.input :single_ref_to_quote, wrapper: :with_label, kmyblue: true, label: I18n.t('simple_form.labels.defaults.setting_single_ref_to_quote'), hint: I18n.t('simple_form.hints.defaults.setting_single_ref_to_quote')
+
     %h4= t 'preferences.posting_defaults'
 
     .fields-row

app/workers/process_references_worker.rb

@@ -3,8 +3,8 @@
 class ProcessReferencesWorker
   include Sidekiq::Worker
 
-  def perform(status_id, ids, urls)
-    ProcessReferencesService.new.call(Status.find(status_id), ids || [], urls: urls || [])
+  def perform(status_id, ids, urls, no_fetch_urls)
+    ProcessReferencesService.new.call(Status.find(status_id), ids || [], urls: urls || [], no_fetch_urls: no_fetch_urls)
   rescue ActiveRecord::RecordNotFound
     true
   end

config/locales/simple_form.en.yml

@@ -71,6 +71,7 @@ en:
         setting_dtl_menu: Show DTL menu on web
         setting_emoji_reaction_policy: Even with this setting, users on other servers are free to put their stamp on the post and share it within the same server. If you simply want to remove the stamp from your own screen, you can disable it from the appearance settings
         setting_enable_emoji_reaction: If turn off, other users still can react your posts
+        setting_single_ref_to_quote: If this server does not have target post, target server maybe cannot read your quote
         setting_use_blurhash: Gradients are based on the colors of the hidden visuals but obfuscate any details
         setting_use_pending_items: Hide timeline updates behind a click instead of automatically scrolling the feed
         username: You can use letters, numbers, and underscores
@@ -262,6 +263,7 @@ en:
         setting_hide_recent_emojis: Hide recent emojis
         setting_hide_statuses_count: Hide statuses count
         setting_link_preview: Generate post link preview card
+        setting_lock_follow_from_bot: Request approval about bot follow
         setting_noai: Set noai meta tags
         setting_public_post_to_unlisted: Convert public post to public unlisted if not using Web app
         setting_reduce_motion: Reduce motion in animations
@@ -271,6 +273,7 @@ en:
         setting_show_application: Disclose application used to send posts
         setting_show_emoji_reaction_on_timeline: Show all stamps on timeline
         setting_simple_timeline_menu: Reduce post menu on timeline
+        setting_single_ref_to_quote: Deliver single reference to other server as quote
         setting_stay_privacy: Not change privacy after post
         setting_stop_emoji_reaction_streaming: Disable stamp streamings
         setting_system_font_ui: Use system's default font

config/locales/simple_form.ja.yml

@@ -82,6 +82,7 @@ ja:
         setting_public_post_to_unlisted: 未対応のサードパーティアプリからもローカル公開で投稿できますが、公開投稿はWeb以外できなくなります
         setting_reject_unlisted_subscription: Misskeyやそのフォーク(Calckeyなど)は、フォローしていないアカウントの「未収載」投稿を **購読・検索** することができます。これはkmyblueの挙動と異なります。そのようなサーバーに、指定した公開範囲の投稿を「フォロワーのみ」として配送します。ただし構造上、完璧な対応は困難でたまに未収載として配信されること、ご理解ください
         setting_show_application: 投稿するのに使用したアプリが投稿の詳細ビューに表示されるようになります
+        setting_single_ref_to_quote: 当サーバーがまだ対象投稿を取り込んでいない場合、引用が相手に正常に認識されない場合があります
         setting_stop_emoji_reaction_streaming: 通信容量の節約に役立ちます
         setting_unsafe_limited_distribution: Mastodon 3.5、4.0、4.1のサーバーにも限定投稿（相互のみ）が届くようになりますが、安全でない方法で送信します
         setting_use_blurhash: ぼかしはメディアの色を元に生成されますが、細部は見えにくくなっています
@@ -276,6 +277,7 @@ ja:
         setting_hide_recent_emojis: 絵文字ピッカーで最近使用した絵文字を隠す（リアクションデッキのみを表示する）
         setting_hide_statuses_count: 投稿数を隠す
         setting_link_preview: リンクのプレビューを生成する
+        setting_lock_follow_from_bot: botからのフォローを承認制にする
         setting_stay_privacy: 投稿時に公開範囲を保存する
         setting_noai: 自分のコンテンツのAI学習利用に対して不快感を表明する
         setting_public_post_to_unlisted: サードパーティから公開範囲「公開」で投稿した場合、「ローカル公開」に変更する
@@ -286,6 +288,7 @@ ja:
         setting_show_application: 送信したアプリを開示する
         setting_show_emoji_reaction_on_timeline: タイムライン上に他の人のつけたスタンプを表示する
         setting_simple_timeline_menu: タイムライン上でメニューの項目を減らす
+        setting_single_ref_to_quote: 参照が１つしかない投稿は、他のサーバーには引用として配信する
         setting_stay_privacy: 投稿時に公開範囲を保存する
         setting_stop_emoji_reaction_streaming: スタンプのストリーミングを停止する
         setting_system_font_ui: システムのデフォルトフォントを使う