Skip to content

Commit

Permalink
Merge branch 'master' into add_gpt_api_key_per_user
Browse files Browse the repository at this point in the history
  • Loading branch information
Melhaya committed Jul 30, 2024
2 parents 8d381a8 + df6988e commit bbe208b
Show file tree
Hide file tree
Showing 33 changed files with 1,624 additions and 510 deletions.
3 changes: 3 additions & 0 deletions Gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ gem 'json_schemer'
gem 'js-routes'
gem 'kramdown'
gem 'kramdown-parser-gfm'
gem 'loofah'
gem 'nested_form_fields'
gem 'net-http'
gem 'net-imap', require: false
Expand All @@ -30,6 +31,7 @@ gem 'proformaxml', '~> 1.4.0'
gem 'puma'
gem 'rails', '~> 7.1.3'
gem 'rails_admin'
gem 'rails-html-sanitizer'
gem 'rails-i18n'
gem 'ransack'
gem 'rqrcode'
Expand All @@ -39,6 +41,7 @@ gem 'sassc-rails'
gem 'shakapacker', '8.0.1'
gem 'simple_form'
gem 'slim-rails'
gem 'solid_queue'
gem 'sprockets-rails'
gem 'terser'
gem 'turbolinks'
Expand Down
19 changes: 17 additions & 2 deletions Gemfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,8 @@ GEM
docile (1.4.0)
drb (2.2.1)
erubi (1.13.0)
et-orbi (1.2.11)
tzinfo
event_stream_parser (1.0.0)
execjs (2.9.1)
factory_bot (6.4.6)
Expand All @@ -169,6 +171,9 @@ GEM
faraday-net_http (3.1.0)
net-http
ffi (1.17.0)
fugit (1.11.0)
et-orbi (~> 1, >= 1.2.11)
raabro (~> 1.4)
glob (0.4.1)
globalid (1.2.1)
activesupport (>= 6.1)
Expand Down Expand Up @@ -280,7 +285,7 @@ GEM
net-smtp (0.5.0)
net-protocol
nio4r (2.7.3)
nokogiri (1.16.6)
nokogiri (1.16.7)
mini_portile2 (~> 2.8.2)
racc (~> 1.4)
omniauth (2.1.2)
Expand All @@ -299,7 +304,7 @@ GEM
parser (3.3.4.0)
ast (~> 2.4.1)
racc
pg (1.5.6)
pg (1.5.7)
proformaxml (1.4.0)
activemodel (>= 5.2.3, < 8.0.0)
activesupport (>= 5.2.3, < 8.0.0)
Expand All @@ -326,6 +331,7 @@ GEM
rspec-expectations (~> 3.12)
rspec-mocks (~> 3.12)
rspec-support (~> 3.12)
raabro (1.4.0)
racc (1.8.0)
rack (3.1.7)
rack-mini-profiler (3.3.1)
Expand Down Expand Up @@ -522,6 +528,12 @@ GEM
slim_lint (0.27.0)
rubocop (>= 1.0, < 2.0)
slim (>= 3.0, < 6.0)
solid_queue (0.3.3)
activejob (>= 7.1)
activerecord (>= 7.1)
concurrent-ruby (>= 1.3.1)
fugit (~> 1.11.0)
railties (>= 7.1)
sorted_set (1.0.3)
rbtree
set (~> 1.0)
Expand Down Expand Up @@ -608,6 +620,7 @@ DEPENDENCIES
kramdown-parser-gfm
letter_opener
listen
loofah
mnemosyne-ruby
nested_form_fields
net-http
Expand All @@ -628,6 +641,7 @@ DEPENDENCIES
rack-mini-profiler
rails (~> 7.1.3)
rails-controller-testing
rails-html-sanitizer
rails-i18n
rails_admin
ransack
Expand All @@ -654,6 +668,7 @@ DEPENDENCIES
simplecov
slim-rails
slim_lint
solid_queue
sprockets-rails
stackprof
terser
Expand Down
11 changes: 11 additions & 0 deletions app/jobs/application_job.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# frozen_string_literal: true

class ApplicationJob < ActiveJob::Base
include ActiveRecordLogging

# Automatically retry jobs that encountered a deadlock
retry_on ActiveRecord::Deadlocked

# Most jobs are safe to ignore if the underlying records are no longer available
discard_on ActiveJob::DeserializationError
end
20 changes: 20 additions & 0 deletions app/jobs/concerns/active_record_logging.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# frozen_string_literal: true

# This module is used to log ActiveRecord queries performed in jobs.
module ActiveRecordLogging
extend ActiveSupport::Concern

included do
around_perform do |_job, block|
# With our current Solid Queue setup, there is a difference between both logger:
# - *ActiveRecord::Base.logger*: This logger is used for SQL queries and, normally, writes to the log file only.
# - *Rails.logger*: The regular logger, which writes to the log file and the console.
# For the duration of the job, we want to write the SQL queries to the Rails logger, so they show up in the console.
# See config/solid_queue_logging.rb for more information.
previous_logger = ActiveRecord::Base.logger
ActiveRecord::Base.logger = Rails.logger
block.call
ActiveRecord::Base.logger = previous_logger
end
end
end
22 changes: 22 additions & 0 deletions app/jobs/nbp_sync_all_job.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# frozen_string_literal: true

class NbpSyncAllJob < ApplicationJob
def perform
uuids = Set[]

# First, add all uploaded UUIDs.
# This allows us to delete the ones that are still present remote but no longer in the local database.
Nbp::PushConnector.instance.process_uploaded_task_uuids do |uuid|
uuids.add(uuid)
end

# Then, add all local UUIDs.
# This allows us to upload tasks missing remote (and remove private tasks not yet removed).
Task.select(:id, :uuid).find_each {|task| uuids.add(task.uuid) }

# Finally, schedule a full sync for each UUID identified.
uuids.each do |uuid|
NbpSyncJob.perform_later uuid
end
end
end
18 changes: 18 additions & 0 deletions app/jobs/nbp_sync_job.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# frozen_string_literal: true

class NbpSyncJob < ApplicationJob
retry_on Faraday::Error, Nbp::PushConnector::ServerError, wait: :polynomially_longer, attempts: 5

def perform(uuid)
task = Task.find_by(uuid:)

if task.present? && task.access_level_public?
builder = Nokogiri::XML::Builder.new(encoding: 'UTF-8') {|xml| LomService::ExportLom.call(task:, xml:) }
Nbp::PushConnector.instance.push_lom!(builder.to_xml)
Rails.logger.debug { "Task ##{task.id} \"#{task}\" pushed to NBP" }
else
Nbp::PushConnector.instance.delete_task!(uuid)
Rails.logger.debug { "Task with UUID #{uuid} deleted from NBP" }
end
end
end
11 changes: 10 additions & 1 deletion app/models/task.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@
class Task < ApplicationRecord
acts_as_taggable_on :state

before_validation :lowercase_language
after_commit :sync_metadata_with_nbp, if: -> { Nbp::PushConnector.enabled? }

validates :title, presence: true

validates :uuid, uniqueness: true

before_validation :lowercase_language
validates :language, format: {with: /\A[a-zA-Z]{1,8}(-[a-zA-Z0-9]{1,8})*\z/, message: :not_de_or_us}
validate :primary_language_tag_in_iso639?

Expand Down Expand Up @@ -102,6 +104,13 @@ def self.ransackable_associations(_auth_object = nil)
%w[labels]
end

def sync_metadata_with_nbp
if access_level_public? || saved_change_to_access_level?
NbpSyncJob.perform_later uuid
NbpSyncJob.perform_later uuid_previously_was if saved_change_to_uuid? && access_level_previously_was == 'public'
end
end

# This method creates a duplicate while leaving permissions and ownership unchanged
def duplicate
dup.tap do |task|
Expand Down
4 changes: 3 additions & 1 deletion app/services/lom_service/export_lom.rb
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,9 @@ def oml_general(xml)
end
xml.language @task.iso639_lang
xml.description do
xml.string ApplicationController.helpers.render_markdown(@task.description), language: @task.iso639_lang
html_fragment = Loofah.fragment(ApplicationController.helpers.render_markdown(@task.description))
html_fragment.scrub!(NbpScrubber.new)
xml.string html_fragment.to_s, language: @task.iso639_lang
end
if @task.programming_language&.language.present?
xml.keyword do
Expand Down
13 changes: 13 additions & 0 deletions app/services/lom_service/nbp_scrubber.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# frozen_string_literal: true

module LomService
class NbpScrubber < Rails::HTML::PermitScrubber
ALLOW_LIST = YAML.safe_load_file(Rails.root.join('app/services/lom_service/nbp_scrubber_allow_list.yml'))

def initialize
super
self.tags = ALLOW_LIST['tags']
self.attributes = ALLOW_LIST['attributes']
end
end
end
Loading

0 comments on commit bbe208b

Please sign in to comment.