From 7772233b8a312fdaba6194c88523c7ae8c185482 Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Thu, 19 Sep 2024 23:44:48 -0700 Subject: [PATCH] Remove unused backfill rake tasks (#5051) Signed-off-by: Samuel Giddins --- .rubocop.yml | 3 - lib/tasks/extraneous_dependencies.rake | 111 -------------------- lib/tasks/gemcutter.rake | 80 -------------- lib/tasks/helpers/gemcutter_tasks_helper.rb | 42 -------- 4 files changed, 236 deletions(-) delete mode 100644 lib/tasks/extraneous_dependencies.rake delete mode 100644 lib/tasks/helpers/gemcutter_tasks_helper.rb diff --git a/.rubocop.yml b/.rubocop.yml index b9f77f71b2e..f7a301544aa 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -148,9 +148,6 @@ Style/StringLiterals: Style/FrozenStringLiteralComment: Enabled: false -Security/MarshalLoad: - Enabled: false - Style/EmptyMethod: EnforcedStyle: expanded diff --git a/lib/tasks/extraneous_dependencies.rake b/lib/tasks/extraneous_dependencies.rake deleted file mode 100644 index 7ce9f834373..00000000000 --- a/lib/tasks/extraneous_dependencies.rake +++ /dev/null @@ -1,111 +0,0 @@ -require "tasks/helpers/compact_index_tasks_helper" - -namespace :extraneous_dependencies do - def fetch_spec_deps(full_name) - spec_uri = URI("https://rubygems.org/quick/Marshal.4.8/#{full_name}.gemspec.rz") - http = Net::HTTP.new(spec_uri.host, spec_uri.port) - http.use_ssl = true - - request = Net::HTTP::Get.new(spec_uri.request_uri) - res = http.request(request) - - raise StandardError, "fetch deps request for #{full_name} failed #{res.inspect}" unless res.code == "200" - - spec_obj = Marshal.load(Gem::Util.inflate(res.body)) - - spec_run_deps = spec_obj.dependencies.filter_map do |s| - s.name.to_s.downcase if s.type == :runtime && Rubygem.where(name: s.name.to_s).present? - end.sort - - spec_dev_deps = spec_obj.dependencies.filter_map do |s| - s.name.to_s.downcase if s.type == :development && Rubygem.where(name: s.name.to_s).present? - end.sort - - [spec_run_deps, spec_dev_deps] - end - - desc "Remove dependencies from DB where it doesn't match the gemspec" - task clean: :environment do |task| - ActiveRecord::Base.logger.level = 1 if Rails.env.development? - - versions = Version.joins("inner join dependencies on versions.id = dependencies.version_id") - .where("date_trunc('day', dependencies.created_at) = '2009-09-02 00:00:00'::timestamp") - .where("versions.indexed = 'true'") - .distinct("versions.id") - - total = versions.count - processed = 0 - errored = 0 - dev_mis_match = 0 - run_mis_match = 0 - mis_match_versions = 0 - total_deleted_deps = 0 - - Rails.logger.info "[extraneous_dependencies:clean] found #{total} versions for clean up" - versions.each do |version| - print format("\r%.2f%% (%d/%d) complete", processed.to_f / total * 100.0, processed, total) - - spec_run_deps, spec_dev_deps = fetch_spec_deps(version.full_name) - - db_run_deps = {} - db_dev_deps = {} - db_deps = version.dependencies.to_a - db_deps.each do |d| - db_run_deps[d.id.to_s] = d.rubygem.name.downcase if d.scope == "runtime" && d.rubygem.present? - db_dev_deps[d.id.to_s] = d.rubygem.name.downcase if d.scope == "development" && d.rubygem.present? - end - - deps_to_delete = [] - if spec_run_deps != db_run_deps.values.sort - unique_run_devs = [] - db_run_deps.each do |id, name| - deps_to_delete << id unless spec_run_deps.include?(name) - - if unique_run_devs.include?(name) - deps_to_delete << id - else - unique_run_devs << name - end - end - - run_mis_match += 1 - Rails.logger.info("[extraneous_dependencies:clean] spec and db run deps don't match " \ - "for: #{version.full_name} spec: #{spec_run_deps} db: #{db_run_deps}") - end - - if spec_dev_deps != db_dev_deps.values.sort - unique_dev_deps = [] - db_dev_deps.sort.to_h.each do |id, name| - if unique_dev_deps.include?(name) - deps_to_delete << id - else - unique_dev_deps << name - end - end - - dev_mis_match += 1 - Rails.logger.info("[extraneous_dependencies:clean] spec and db dev deps don't match " \ - "for: #{version.full_name} spec: #{spec_dev_deps} db: #{db_dev_deps}") - end - - if deps_to_delete.present? - mis_match_versions += 1 - total_deleted_deps += deps_to_delete.count - Rails.logger.info("[extraneous_dependencies:clean] deleting dependencies with ids: #{deps_to_delete}") - Dependency.destroy(deps_to_delete) - - CompactIndexTasksHelper.update_last_checksum(version.rubygem, task) - end - rescue StandardError => e - errored += 1 - Rails.logger.error("[extraneous_dependencies:clean] skipping #{version.inspect} - #{e.message}") - ensure - processed += 1 - end - - Rails.logger.info("[extraneous_dependencies:clean] #{total_deleted_deps} dependencies deleted") - Rails.logger.info("[extraneous_dependencies:clean] #{errored}/#{processed} errors") - Rails.logger.info("[extraneous_dependencies:clean] #{mis_match_versions}/#{processed} version mismatches " \ - "(run_deps: #{run_mis_match}, dev_deps: #{dev_mis_match})") - end -end diff --git a/lib/tasks/gemcutter.rake b/lib/tasks/gemcutter.rake index 51987a3623c..c2e0c61935c 100644 --- a/lib/tasks/gemcutter.rake +++ b/lib/tasks/gemcutter.rake @@ -1,5 +1,3 @@ -require "tasks/helpers/gemcutter_tasks_helper" - namespace :gemcutter do namespace :index do desc "Update the index" @@ -26,84 +24,6 @@ namespace :gemcutter do end end - namespace :checksums do - desc "Initialize missing checksums." - task init: :environment do - without_sha256 = Version.where(sha256: nil) - mod = ENV["shard"] - without_sha256.where("id % 4 = ?", mod.to_i) if mod - - total = without_sha256.count - i = 0 - without_sha256.find_each do |version| - GemcutterTaskshelper.recalculate_sha256!(version) - i += 1 - print format("\r%.2f%% (%d/%d) complete", i.to_f / total * 100.0, i, total) - end - puts - puts "Done." - end - - desc "Check existing checksums." - task check: :environment do - failed = false - i = 0 - total = Version.count - Version.find_each do |version| - actual_sha256 = GemcutterTaskshelper.recalculate_sha256(version.full_name) - if actual_sha256 && version.sha256 != actual_sha256 - puts "#{version.full_name}.gem has sha256 '#{actual_sha256}', " \ - "but '#{version.sha256}' was expected." - failed = true - end - i += 1 - print format("\r%.2f%% (%d/%d) complete", i.to_f / total * 100.0, i, total) - end - end - end - - namespace :metadata do - desc "Backfill old gem versions with metadata." - task backfill: :environment do - without_metadata = Version.where("metadata = ''") - mod = ENV["shard"] - without_metadata = without_metadata.where("id % 4 = ?", mod.to_i) if mod - - total = without_metadata.count - i = 0 - puts "Total: #{total}" - without_metadata.find_each do |version| - GemcutterTaskshelper.recalculate_metadata!(version) - i += 1 - print format("\r%.2f%% (%d/%d) complete", i.to_f / total * 100.0, i, total) - end - puts - puts "Done." - end - end - - namespace :required_ruby_version do - desc "Backfill gem versions with rubygems_version." - task backfill: :environment do - ActiveRecord::Base.logger.level = 1 if Rails.env.development? - - without_required_ruby_version = Version.where("created_at < '2014-03-21' and required_ruby_version is null and indexed = true") - mod = ENV["shard"] - without_required_ruby_version = without_required_ruby_version.where("id % 4 = ?", mod.to_i) if mod - - total = without_required_ruby_version.count - i = 0 - puts "Total: #{total}" - without_required_ruby_version.find_each do |version| - GemcutterTaskshelper.assign_required_ruby_version!(version) - i += 1 - print format("\r%.2f%% (%d/%d) complete", i.to_f / total * 100.0, i, total) - end - puts - puts "Done." - end - end - namespace :typo do desc "Add names to gem typo exception list\nUsage: rake gemcutter:typo:exception[,]" task :exception, %i[name info] => %i[environment] do |_task, args| diff --git a/lib/tasks/helpers/gemcutter_tasks_helper.rb b/lib/tasks/helpers/gemcutter_tasks_helper.rb deleted file mode 100644 index 77477f5dc69..00000000000 --- a/lib/tasks/helpers/gemcutter_tasks_helper.rb +++ /dev/null @@ -1,42 +0,0 @@ -module GemcutterTaskshelper - module_function - - def recalculate_sha256(version_full_name) - key = "gems/#{version_full_name}.gem" - file = RubygemFs.instance.get(key) - Digest::SHA2.base64digest(file) if file - end - - def recalculate_sha256!(version) - sha256 = recalculate_sha256(version.full_name) - version.update(sha256: sha256) - end - - def recalculate_metadata!(version) - metadata = get_spec_attribute(version.full_name, "metadata") - version.update(metadata: metadata || {}) - end - - def assign_required_ruby_version!(version) - required_ruby_version = get_spec_attribute(version.full_name, "required_ruby_version") - - return if required_ruby_version.nil? || required_ruby_version.to_s == ">= 0" - Rails.logger.info("[gemcutter:required_ruby_version:backfill] updating version: #{version.full_name} " \ - "with required_ruby_version: #{required_ruby_version}") - - version.update_column(:required_ruby_version, required_ruby_version.to_s) - CompactIndexTasksHelper.update_last_checksum(version.rubygem, "gemcutter:required_ruby_version:backfill") - end - - def get_spec_attribute(version_full_name, attribute_name) - key = "quick/Marshal.4.8/#{version_full_name}.gemspec.rz" - file = RubygemFs.instance.get(key) - return nil unless file - spec = Marshal.load(Gem::Util.inflate(file)) - spec.send(attribute_name) - rescue StandardError => e - Rails.logger.info("[gemcutter:required_ruby_version:backfill] could not get required_ruby_version for version: #{version_full_name} " \ - "error: #{e.inspect}") - nil - end -end