diff options
85 files changed, 2124 insertions, 654 deletions
@@ -10,8 +10,16 @@ ENV USE="-bindist" RUN emerge -C openssh RUN emerge net-libs/nodejs +RUN emerge sys-process/cronie # Bundler is how we install the ruby stuff. -RUN emerge dev-ruby/bundler +RUN mkdir -p /etc/portage/package.accept_keywords/ +RUN echo "=dev-ruby/rdoc-6.2.0 ~amd64" >> /etc/portage/package.accept_keywords/ruby +RUN echo "=dev-lang/ruby-2.5.6 ~amd64" >> /etc/portage/package.accept_keywords/ruby + +RUN emerge =dev-lang/ruby-2.5.6 +RUN gem install bundler + +RUN emerge dev-vcs/git # Needed for changelogs. RUN git clone https://anongit.gentoo.org/git/repo/gentoo.git /mnt/packages-tree/gentoo/ @@ -24,6 +32,7 @@ RUN bundler install # Git clones here. RUN cp /var/www/packages.gentoo.org/htdocs/config/secrets.yml.dist /var/www/packages.gentoo.org/htdocs/config/secrets.yml RUN sed -i 's/set_me/ENV["SECRET_KEY_BASE"]/'g /var/www/packages.gentoo.org/htdocs/config/secrets.yml +RUN cp /var/www/packages.gentoo.org/htdocs/config/initializers/kkuleomi_config.rb.dist /var/www/packages.gentoo.org/htdocs/config/initializers/kkuleomi_config.rb # Precompile our assets. RUN bundle exec rake assets:precompile @@ -1,7 +1,7 @@ source 'https://rubygems.org' # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' -gem 'rails', '~> 5.2.3' +gem 'rails', '~> 6.0.0' # Use mysql as the database for Active Record # gem 'mysql2' # Use SCSS for stylesheets @@ -23,8 +23,8 @@ gem 'jbuilder', '~> 2.0' gem 'sdoc', '~> 1.0', group: :doc # packages stuff -gem 'elasticsearch-rails', '~> 5.0' -gem 'elasticsearch-persistence', '~> 5.0' +gem 'elasticsearch-rails', '~> 7.0.0' +gem 'elasticsearch-persistence', '~> 7.0.0' gem 'nokogiri' gem 'thin' @@ -33,9 +33,13 @@ gem 'sidekiq', require: false gem 'rdiscount' +gem 'parslet' + # UI gem 'octicons_helper' +gem 'rails-controller-testing' + # Use ActiveModel has_secure_password # gem 'bcrypt', '~> 3.1.7' @@ -63,8 +67,4 @@ group :development do # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring gem 'spring' -end - -group :test do - gem 'rails-controller-testing' -end +end
\ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index 78774c0..f959a48 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,89 +1,92 @@ GEM remote: https://rubygems.org/ specs: - actioncable (5.2.3) - actionpack (= 5.2.3) + actioncable (6.0.0) + actionpack (= 6.0.0) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailer (5.2.3) - actionpack (= 5.2.3) - actionview (= 5.2.3) - activejob (= 5.2.3) + actionmailbox (6.0.0) + actionpack (= 6.0.0) + activejob (= 6.0.0) + activerecord (= 6.0.0) + activestorage (= 6.0.0) + activesupport (= 6.0.0) + mail (>= 2.7.1) + actionmailer (6.0.0) + actionpack (= 6.0.0) + actionview (= 6.0.0) + activejob (= 6.0.0) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.2.3) - actionview (= 5.2.3) - activesupport (= 5.2.3) + actionpack (6.0.0) + actionview (= 6.0.0) + activesupport (= 6.0.0) rack (~> 2.0) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.2.3) - activesupport (= 5.2.3) + rails-html-sanitizer (~> 1.0, >= 1.2.0) + actiontext (6.0.0) + actionpack (= 6.0.0) + activerecord (= 6.0.0) + activestorage (= 6.0.0) + activesupport (= 6.0.0) + nokogiri (>= 1.8.5) + actionview (6.0.0) + activesupport (= 6.0.0) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.0, >= 1.0.3) - activejob (5.2.3) - activesupport (= 5.2.3) + rails-html-sanitizer (~> 1.1, >= 1.2.0) + activejob (6.0.0) + activesupport (= 6.0.0) globalid (>= 0.3.6) - activemodel (5.2.3) - activesupport (= 5.2.3) - activerecord (5.2.3) - activemodel (= 5.2.3) - activesupport (= 5.2.3) - arel (>= 9.0) - activestorage (5.2.3) - actionpack (= 5.2.3) - activerecord (= 5.2.3) + activemodel (6.0.0) + activesupport (= 6.0.0) + activerecord (6.0.0) + activemodel (= 6.0.0) + activesupport (= 6.0.0) + activestorage (6.0.0) + actionpack (= 6.0.0) + activejob (= 6.0.0) + activerecord (= 6.0.0) marcel (~> 0.3.1) - activesupport (5.2.3) + activesupport (6.0.0) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) - arel (9.0.0) + zeitwerk (~> 2.1, >= 2.1.8) ast (2.4.0) - axiom-types (0.1.1) - descendants_tracker (~> 0.0.4) - ice_nine (~> 0.11.0) - thread_safe (~> 0.3, >= 0.3.1) bindex (0.8.1) builder (3.2.3) byebug (11.0.1) - coercible (1.0.0) - descendants_tracker (~> 0.0.1) concurrent-ruby (1.1.5) connection_pool (2.2.2) crass (1.0.4) daemons (1.3.1) - descendants_tracker (0.0.4) - thread_safe (~> 0.3, >= 0.3.1) - elasticsearch (5.0.5) - elasticsearch-api (= 5.0.5) - elasticsearch-transport (= 5.0.5) - elasticsearch-api (5.0.5) + elasticsearch (7.3.0) + elasticsearch-api (= 7.3.0) + elasticsearch-transport (= 7.3.0) + elasticsearch-api (7.3.0) multi_json - elasticsearch-model (5.1.0) + elasticsearch-model (7.0.0) activesupport (> 3) - elasticsearch (~> 5) + elasticsearch (> 1) hashie - elasticsearch-persistence (5.1.0) + elasticsearch-persistence (7.0.0) activemodel (> 4) activesupport (> 4) - elasticsearch (~> 5) - elasticsearch-model (~> 5) + elasticsearch (~> 7) + elasticsearch-model (= 7.0.0) hashie - virtus - elasticsearch-rails (5.1.0) - elasticsearch-transport (5.0.5) + elasticsearch-rails (7.0.0) + elasticsearch-transport (7.3.0) faraday multi_json - equalizer (0.0.11) - erubi (1.8.0) + erubi (1.9.0) eventmachine (1.2.7) execjs (2.7.0) - faraday (0.15.4) + faraday (0.16.1) multipart-post (>= 1.2, < 3) ffi (1.11.1) globalid (0.4.2) @@ -91,7 +94,6 @@ GEM hashie (3.6.0) i18n (1.6.0) concurrent-ruby (~> 1.0) - ice_nine (0.11.2) jaro_winkler (1.5.3) jbuilder (2.9.1) activesupport (>= 4.2.0) @@ -114,11 +116,11 @@ GEM mimemagic (0.3.3) mini_mime (1.0.2) mini_portile2 (2.4.0) - minitest (5.11.3) + minitest (5.12.1) multi_json (1.13.1) multipart-post (2.1.1) - nio4r (2.4.0) - nokogiri (1.10.3) + nio4r (2.5.2) + nokogiri (1.10.4) mini_portile2 (~> 2.4.0) octicons (9.1.1) nokogiri (>= 1.6.3.1) @@ -126,25 +128,28 @@ GEM octicons (= 9.1.1) rails parallel (1.17.0) - parser (2.6.3.0) + parser (2.6.4.1) ast (~> 2.4.0) + parslet (1.8.2) rack (2.0.7) - rack-protection (2.0.5) + rack-protection (2.0.7) rack rack-test (1.1.0) rack (>= 1.0, < 3) - rails (5.2.3) - actioncable (= 5.2.3) - actionmailer (= 5.2.3) - actionpack (= 5.2.3) - actionview (= 5.2.3) - activejob (= 5.2.3) - activemodel (= 5.2.3) - activerecord (= 5.2.3) - activestorage (= 5.2.3) - activesupport (= 5.2.3) + rails (6.0.0) + actioncable (= 6.0.0) + actionmailbox (= 6.0.0) + actionmailer (= 6.0.0) + actionpack (= 6.0.0) + actiontext (= 6.0.0) + actionview (= 6.0.0) + activejob (= 6.0.0) + activemodel (= 6.0.0) + activerecord (= 6.0.0) + activestorage (= 6.0.0) + activesupport (= 6.0.0) bundler (>= 1.3.0) - railties (= 5.2.3) + railties (= 6.0.0) sprockets-rails (>= 2.0.0) rails-controller-testing (1.0.4) actionpack (>= 5.0.1.x) @@ -153,22 +158,22 @@ GEM rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) - rails-html-sanitizer (1.0.4) + rails-html-sanitizer (1.2.0) loofah (~> 2.2, >= 2.2.2) - railties (5.2.3) - actionpack (= 5.2.3) - activesupport (= 5.2.3) + railties (6.0.0) + actionpack (= 6.0.0) + activesupport (= 6.0.0) method_source rake (>= 0.8.7) - thor (>= 0.19.0, < 2.0) + thor (>= 0.20.3, < 2.0) rainbow (3.0.0) - rake (12.3.3) + rake (13.0.0) rb-fsevent (0.10.3) rb-inotify (0.10.0) ffi (~> 1.0) rdiscount (2.2.0.1) - rdoc (6.1.1) - redis (4.1.2) + rdoc (6.2.0) + redis (4.1.3) rubocop (0.73.0) jaro_winkler (~> 1.5.1) parallel (~> 1.10) @@ -183,9 +188,8 @@ GEM rubocop (>= 0.72.0) ruby-progressbar (1.10.1) ruby_dep (1.5.0) - sassc (2.0.1) + sassc (2.2.1) ffi (~> 1.9) - rake sassc-rails (2.1.2) railties (>= 4.0.0) sassc (>= 2.0) @@ -194,11 +198,11 @@ GEM tilt sdoc (1.0.0) rdoc (>= 5.0) - sidekiq (5.2.7) - connection_pool (~> 2.2, >= 2.2.2) - rack (>= 1.5.0) - rack-protection (>= 1.5.0) - redis (>= 3.3.5, < 5) + sidekiq (6.0.0) + connection_pool (>= 2.2.2) + rack (>= 2.0.0) + rack-protection (>= 2.0.0) + redis (>= 4.1.0) spring (2.1.0) sprockets (3.7.2) concurrent-ruby (~> 1.0) @@ -213,20 +217,15 @@ GEM rack (>= 1, < 3) thor (0.20.3) thread_safe (0.3.6) - tilt (2.0.9) - turbolinks (5.2.0) + tilt (2.0.10) + turbolinks (5.2.1) turbolinks-source (~> 5.2) turbolinks-source (5.2.0) tzinfo (1.2.5) thread_safe (~> 0.1) - uglifier (4.1.20) + uglifier (4.2.0) execjs (>= 0.3.0, < 3) unicode-display_width (1.6.0) - virtus (1.0.5) - axiom-types (~> 0.1) - coercible (~> 1.0) - descendants_tracker (~> 0.0, >= 0.0.3) - equalizer (~> 0.0, >= 0.0.9) web-console (3.7.0) actionview (>= 5.0) activemodel (>= 5.0) @@ -235,20 +234,22 @@ GEM websocket-driver (0.7.1) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.4) + zeitwerk (2.1.10) PLATFORMS ruby DEPENDENCIES byebug - elasticsearch-persistence (~> 5.0) - elasticsearch-rails (~> 5.0) + elasticsearch-persistence (~> 7.0.0) + elasticsearch-rails (~> 7.0.0) jbuilder (~> 2.0) jquery-rails (~> 4.3.5) listen nokogiri octicons_helper - rails (~> 5.2.3) + parslet + rails (~> 6.0.0) rails-controller-testing rdiscount rubocop (= 0.73.0) @@ -264,4 +265,4 @@ DEPENDENCIES web-console (~> 3.0) BUNDLED WITH - 1.17.3 + 2.0.2 diff --git a/app/assets/javascripts/index/query_generator.js b/app/assets/javascripts/index/query_generator.js new file mode 100644 index 0000000..17cb798 --- /dev/null +++ b/app/assets/javascripts/index/query_generator.js @@ -0,0 +1,93 @@ +function updateDropdown(self) { + getThirdParent(self).querySelector('button > span:first-child').innerHTML = self.innerHTML; +} + +function buildAdvancedQuery(){ + var query = "" + document.querySelectorAll('#search-container > .row').forEach(function(element) { + var term = element.querySelector('.form-control').value; + + if(!term.replace(/\s/g, '').length){ + return; + }else{ + term = parseSearchTerm(term); + } + + var operator = parseOperator(element.querySelector('.pgo-query-operator > span:first-child').innerHTML); + var field = element.querySelector('.pgo-query-field > span:first-child').innerHTML; + + query += operator + field + ":" + term + " "; + }); + document.getElementById('q').value = query; +} + +function parseOperator(operator){ + switch(operator) { + case "should match": + return ""; + case "must match": + return "+"; + case "must not match": + return "-"; + default: + return ""; + } +} + +function parseSearchTerm(term){ + if (/\s/.test(term) && !/^\".*\"$/.test(term)) { + return "\"" + term + "\"" + }else{ + return term + } +} + +function addInput(self){ + var new_input = document.querySelector('#search-container > .row').cloneNode(true); + resetInput(new_input); + document.querySelector('#search-container').append(new_input); + checkDeleteButtons(); + checkAddButtons(); +} + +function resetInput(input) { + input.querySelector('.form-control').value = ''; + input.querySelector('.pgo-query-operator > span:first-child').innerHTML = 'should match'; + input.querySelector('.pgo-query-field > span:first-child').innerHTML = 'name'; +} + +function deleteInput(self){ + getThirdParent(self).removeChild(getSecondParent(self)); + checkDeleteButtons(); + checkAddButtons(); +} + +function checkDeleteButtons(){ + if(document.querySelectorAll('#search-container > .row').length == 1){ + document.querySelectorAll('.pgo-query-delete-btn').forEach(function(element) { + element.style.display = 'none'; + }); + }else{ + document.querySelectorAll('.pgo-query-delete-btn').forEach(function(element) { + element.style.display = 'block'; + }); + } +} + +function checkAddButtons(){ + document.querySelectorAll('.pgo-query-add-btn').forEach(function(element) { + element.style.display = 'none'; + }); + + document.querySelectorAll('.pgo-query-add-btn')[document.querySelectorAll('.pgo-query-add-btn').length - 1].style.display = 'block'; +} + +function getThirdParent(self) { + return self.parentElement.parentElement.parentElement; +} + +function getSecondParent(self) { + return self.parentElement.parentElement; +} + +checkDeleteButtons();
\ No newline at end of file diff --git a/app/assets/stylesheets/application.css b/app/assets/stylesheets/application.css index 6b38ea1..ee26125 100644 --- a/app/assets/stylesheets/application.css +++ b/app/assets/stylesheets/application.css @@ -14,3 +14,17 @@ *= require jquery.typeahead.min *= require_self */ + +/* Keep the footer at the bottom */ +body { + min-height: 100vh; + position: relative; + margin: 0; + padding-bottom: 215px; +} +footer { + position: absolute; + margin-top: 40px; + bottom: 0; + width: 100%; +}
\ No newline at end of file diff --git a/app/controllers/arches_controller.rb b/app/controllers/arches_controller.rb index cbbcb65..c72e378 100644 --- a/app/controllers/arches_controller.rb +++ b/app/controllers/arches_controller.rb @@ -43,9 +43,9 @@ class ArchesController < ApplicationController def keyworded_packages(arch) Rails.cache.fetch("keyworded_packages/#{arch}", expires_in: 10.minutes) do - Change.filter_all({ change_type: 'keyword', arches: arch }, - size: 50, - sort: { created_at: { order: 'desc' } }).map do |change| + ChangeRepository.filter_all({ change_type: 'keyword', arches: arch }, + size: 50, + sort: { created_at: { order: 'desc' } }).map do |change| change.to_os(:change_type, :package, :category, :version, :arches, :created_at) end end @@ -53,9 +53,9 @@ class ArchesController < ApplicationController def stabled_packages(arch) Rails.cache.fetch("stabled_packages/#{arch}", expires_in: 10.minutes) do - Change.filter_all({ change_type: 'stable', arches: arch }, - size: 50, - sort: { created_at: { order: 'desc' } }).map do |change| + ChangeRepository.filter_all({ change_type: 'stable', arches: arch }, + size: 50, + sort: { created_at: { order: 'desc' } }).map do |change| change.to_os(:change_type, :package, :category, :version, :arches, :created_at) end end diff --git a/app/controllers/categories_controller.rb b/app/controllers/categories_controller.rb index 33817aa..a9c9b06 100644 --- a/app/controllers/categories_controller.rb +++ b/app/controllers/categories_controller.rb @@ -3,15 +3,15 @@ class CategoriesController < ApplicationController before_action :set_nav def index - @categories = Category.all_sorted_by(:name, :asc) + @categories = CategoryRepository.all_sorted_by(:id, :asc) end def show @packages = Rails.cache.fetch("category/#{@category.name}/packages", expires_in: 10.minutes) do - Package.find_all_by(:category, - @category.name, - sort: { name_sort: { order: 'asc' } }).map do |pkg| + PackageRepository.find_all_by(:category, + @category.name, + sort: { name_sort: { order: 'asc' } }).map do |pkg| pkg.to_os(:name, :atom, :description) end end @@ -24,7 +24,7 @@ class CategoriesController < ApplicationController private def set_category - @category = Category.find_by(:name, params[:id]) + @category = CategoryRepository.find_by(:name, params[:id]) fail ActionController::RoutingError, 'No such category' unless @category @title = @category.name diff --git a/app/controllers/concerns/package_update_feeds.rb b/app/controllers/concerns/package_update_feeds.rb index 2d20672..28a951b 100644 --- a/app/controllers/concerns/package_update_feeds.rb +++ b/app/controllers/concerns/package_update_feeds.rb @@ -3,7 +3,7 @@ module PackageUpdateFeeds def new_packages Rails.cache.fetch('new_packages', expires_in: 10.minutes) do - Change.find_all_by(:change_type, 'new_package', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| + ChangeRepository.find_all_by(:change_type, 'new_package', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| change.to_os(:change_type, :package, :category, :created_at) end end @@ -11,7 +11,7 @@ module PackageUpdateFeeds def version_bumps Rails.cache.fetch('version_bumps', expires_in: 10.minutes) do - Change.find_all_by(:change_type, 'version_bump', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| + ChangeRepository.find_all_by(:change_type, 'version_bump', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| change.to_os(:change_type, :package, :category, :version, :created_at) end end @@ -19,7 +19,7 @@ module PackageUpdateFeeds def keyworded_packages Rails.cache.fetch('keyworded_packages', expires_in: 10.minutes) do - Change.find_all_by(:change_type, 'keyword', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| + ChangeRepository.find_all_by(:change_type, 'keyword', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| change.to_os(:change_type, :package, :category, :version, :arches, :created_at) end end @@ -27,7 +27,7 @@ module PackageUpdateFeeds def stabled_packages Rails.cache.fetch('stabled_packages', expires_in: 10.minutes) do - Change.find_all_by(:change_type, 'stable', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| + ChangeRepository.find_all_by(:change_type, 'stable', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| change.to_os(:change_type, :package, :category, :version, :arches, :created_at) end end diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index 64cb289..e735d6c 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -8,21 +8,24 @@ class PackagesController < ApplicationController def search @offset = params[:o].to_i || 0 - @packages = Package.default_search(params[:q], @offset) + @packages = PackageRepository.default_search(params[:q], @offset) + @query = params[:q] + + render_packages_feed :packageinfo, t(:feed_search_results, query: params[:q] ) redirect_to package_path(@packages.first).gsub('%2F', '/') if @packages.size == 1 end def suggest - @packages = Package.suggest(params[:q]) + @packages = PackageRepository.suggest(params[:q]) end def resolve - @packages = Package.resolve(params[:atom]) + @packages = PackageRepository.resolve(params[:atom]) end def show - @package = Package.find_by(:atom, params[:id]) + @package = PackageRepository.find_by(:atom, params[:id]) fail ActionController::RoutingError, 'No such package' unless @package fresh_when etag: @package.updated_at, last_modified: @package.updated_at, public: true @@ -34,12 +37,12 @@ class PackagesController < ApplicationController end def changelog - @package = Package.find_by(:atom, params[:id]) + @package = PackageRepository.find_by(:atom, params[:id]) fail ActionController::RoutingError, 'No such package' unless @package if stale?(etag: @package.updated_at, last_modified: @package.updated_at, public: true) @changelog = Rails.cache.fetch("changelog/#{@package.atom}") do - Portage::Util::History.for(@package.category, @package.name, 5) + CommitRepository.find_sorted_by('packages', @package.category + '/'+ @package.name, "date", "desc", 5) end respond_to do |wants| @@ -82,6 +85,17 @@ class PackagesController < ApplicationController end end + def render_packages_feed(type, title) + respond_to do |wants| + wants.html {} + wants.atom do + @feed_type = type + @feed_title = title + render template: 'feeds/packages' + end + end + end + def set_nav @nav = :packages end diff --git a/app/controllers/useflags_controller.rb b/app/controllers/useflags_controller.rb index 0fa74f4..9802b78 100644 --- a/app/controllers/useflags_controller.rb +++ b/app/controllers/useflags_controller.rb @@ -6,18 +6,18 @@ class UseflagsController < ApplicationController end def show - @useflags = Useflag.get_flags(params[:id]) + @useflags = UseflagRepository.get_flags(params[:id]) if @useflags.empty? || (@useflags[:use_expand].empty? && @useflags[:local].empty? && @useflags[:global].empty?) fail ActionController::RoutingError, 'No such useflag' end - @packages = Package.find_atoms_by_useflag(params[:id]) + @packages = PackageRepository.find_atoms_by_useflag(params[:id]) @title = '%s – %s' % [params[:id], t(:use_flags)] unless @useflags[:use_expand].empty? @useflag = @useflags[:use_expand].first - @use_expand_flags = Useflag.find_all_by(:use_expand_prefix, @useflag.use_expand_prefix) + @use_expand_flags = UseflagRepository.find_all_by(:use_expand_prefix, @useflag.use_expand_prefix) @use_expand_flag_name = @useflag.use_expand_prefix.upcase render template: 'useflags/show_use_expand' @@ -29,16 +29,16 @@ class UseflagsController < ApplicationController def search # TODO: Different search? - @flags = Useflag.suggest(params[:q]) + @flags = UseflagRepository.suggest(params[:q]) end def suggest - @flags = Useflag.suggest(params[:q]) + @flags = UseflagRepository.suggest(params[:q]) end def popular @popular_useflags = Rails.cache.fetch('popular_useflags', expires_in: 24.hours) do - Version.get_popular_useflags(100) + VersionRepository.get_popular_useflags(100) end end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 619582c..8405e59 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -37,6 +37,9 @@ module ApplicationHelper end def i18n_date(date, format = '%a, %e %b %Y %H:%M') + + date = Time.parse(date).utc if date.is_a? String + content_tag :span, l(date, format: format), class: 'kk-i18n-date', diff --git a/app/helpers/packages_helper.rb b/app/helpers/packages_helper.rb index ee83a2e..10aee94 100644 --- a/app/helpers/packages_helper.rb +++ b/app/helpers/packages_helper.rb @@ -1,3 +1,5 @@ +require 'open-uri' + # Helpers for displaying package models module PackagesHelper def restrict_label(version) @@ -42,7 +44,7 @@ module PackagesHelper #end def annotate_bugs(str) - annotated_str = str.gsub(/([bB]ug\s+|[bB]ug\s+#|#)(\d+)/) do + annotated_str = (h str).gsub(/([bB]ug\s+|[bB]ug\s+#|#)(\d+)/) do link_to_bug("#{$1}#{$2}", $2) end @@ -79,15 +81,21 @@ module PackagesHelper # Tries to find a matching changelog entry for a change object def matching_changelog_entry(change) changelog = Rails.cache.fetch("changelog/#{cp_to_atom(change.category, change.package)}", expires_in: 10.minutes) do - Portage::Util::History.for(change.category, change.package, 5) + CommitRepository.find_sorted_by('packages', change.category + '/' + change.package, "date", "desc", 5) end changelog.each do |changelog_entry| - if changelog_entry[:files][:added].include?('%s-%s.ebuild' % [change.package, change.version]) + if changelog_entry.files["added"].include?('%s/%s/%s-%s.ebuild' % [change.category, change.package, change.package, change.version]) return changelog_entry end end nil end + + def documentation_label(package) + doc = Nokogiri::XML(open("https://wiki.gentoo.org/api.php?action=query&titles=" + package + "&format=xml")) + doc.xpath("//api/query/pages/page")[0].attr('missing').nil? ? (t :res_docs) : (t :res_search_docs) + end + end diff --git a/app/jobs/category_update_job.rb b/app/jobs/category_update_job.rb index 7443099..e764ad8 100644 --- a/app/jobs/category_update_job.rb +++ b/app/jobs/category_update_job.rb @@ -5,8 +5,8 @@ class CategoryUpdateJob < ApplicationJob category_path, options = args model = Portage::Repository::Category.new(category_path) - category = Category.find_by(:name, model.name) || Category.new - idx_packages = Package.find_all_by(:category, model.name) || [] + category = CategoryRepository.find_by(:name, model.name) || Category.new + idx_packages = PackageRepository.find_all_by(:category, model.name) || [] if category.needs_import? model category.import! model diff --git a/app/jobs/commits_update_job.rb b/app/jobs/commits_update_job.rb new file mode 100644 index 0000000..f4c170b --- /dev/null +++ b/app/jobs/commits_update_job.rb @@ -0,0 +1,8 @@ +class CommitsUpdateJob < ApplicationJob + queue_as :default + + def perform(*args) + Portage::Util::History.update() + end + +end diff --git a/app/jobs/package_removal_job.rb b/app/jobs/package_removal_job.rb index 877ed07..e625b96 100644 --- a/app/jobs/package_removal_job.rb +++ b/app/jobs/package_removal_job.rb @@ -4,11 +4,11 @@ class PackageRemovalJob < ApplicationJob def perform(*args) atom, _options = args - package_doc = Package.find_by(:atom, atom) + package_doc = PackageRepository.find_by(:atom, atom) return if package_doc.nil? - package_doc.versions.each(&:delete) - package_doc.delete + package_doc.versions.each { |v| VersionRepository.delete(v) } + PackageRepository.delete(package_doc) Rails.logger.warn { "Package deleted: #{atom}" } # USE flags are cleaned up by the UseflagsUpdateJob diff --git a/app/jobs/package_update_job.rb b/app/jobs/package_update_job.rb index 55e278f..53a352c 100644 --- a/app/jobs/package_update_job.rb +++ b/app/jobs/package_update_job.rb @@ -4,7 +4,7 @@ class PackageUpdateJob < ApplicationJob def perform(*args) path, options = args package_model = Portage::Repository::Package.new(path) - package_doc = Package.find_by(:atom, package_model.to_cp) || Package.new + package_doc = PackageRepository.find_by(:atom, package_model.to_cp) || Package.new if package_doc.needs_import? package_model package_doc.import!(package_model, options) diff --git a/app/jobs/record_change_job.rb b/app/jobs/record_change_job.rb index 0e6a011..ed5dd5e 100644 --- a/app/jobs/record_change_job.rb +++ b/app/jobs/record_change_job.rb @@ -25,6 +25,6 @@ class RecordChangeJob < ApplicationJob c.change_type = 'removal' end - c.save + ChangeRepository.save(c) end end diff --git a/app/jobs/useflags_update_job.rb b/app/jobs/useflags_update_job.rb index 21145c3..5558d47 100644 --- a/app/jobs/useflags_update_job.rb +++ b/app/jobs/useflags_update_job.rb @@ -10,7 +10,7 @@ class UseflagsUpdateJob < ApplicationJob def update_global(repo) model_flags = repo.global_useflags - index_flags = Useflag.global + index_flags = UseflagRepository.global new_flags = model_flags.keys - index_flags.keys del_flags = index_flags.keys - model_flags.keys @@ -21,24 +21,24 @@ class UseflagsUpdateJob < ApplicationJob flag_doc.name = flag flag_doc.description = model_flags[flag] flag_doc.scope = 'global' - flag_doc.save + UseflagRepository.save(flag_doc) end eql_flags.each do |flag| unless index_flags[flag].description == model_flags[flag] index_flags[flag].description = model_flags[flag] - index_flags[flag].save + UseflagRepository.save(index_flags[flag]) end end del_flags.each do |flag| - index_flags[flag].delete + UseflagRepository.delete(index_flags[flag]) end end def update_use_expand(repo) model_flags = repo.use_expand_flags - index_flags = Useflag.use_expand + index_flags = UseflagRepository.use_expand # Calculate keys only once index_flag_keys = index_flags.keys @@ -55,7 +55,7 @@ class UseflagsUpdateJob < ApplicationJob if index_flag_keys.include? _flag unless index_flags[_flag].description == desc index_flags[_flag].description = desc - index_flags[_flag].save + UseflagRepository.save(index_flags[_flag]) end else # New flag @@ -64,14 +64,14 @@ class UseflagsUpdateJob < ApplicationJob flag_doc.description = desc flag_doc.scope = 'use_expand' flag_doc.use_expand_prefix = variable - flag_doc.save + UseflagRepository.save(flag_doc) end end end # Find and process removed flags flag_status.each_pair do |flag, status| - index_flags[flag].delete unless status + UseflagRepository.delete(index_flags[flag]) unless status end end diff --git a/app/models/category.rb b/app/models/category.rb index f629bde..4e361c1 100644 --- a/app/models/category.rb +++ b/app/models/category.rb @@ -1,12 +1,39 @@ class Category - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model + include ActiveModel::Model + include ActiveModel::Validations - index_name "categories-#{Rails.env}" + ATTRIBUTES = [:id, + :created_at, + :updated_at, + :name, + :description, + :metadata_hash] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :name, presence: true + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end + + def attributes + @id = @name + @created_at ||= DateTime.now + @updated_at = DateTime.now + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes - attribute :name, String, mapping: { type: 'keyword' } - attribute :description, String, mapping: { type: 'text' } - attribute :metadata_hash, String, mapping: { type: 'text' } # Determines if the document model needs an update from the repository model # @@ -29,7 +56,7 @@ class Category # @param [Portage::Repository::Category] category_model Input category model def import!(category_model) import(category_model) - save + CategoryRepository.save(self) end # Returns the URL parameter for referencing this package (Rails internal stuff) diff --git a/app/models/change.rb b/app/models/change.rb index 6eaf00c..1793da4 100644 --- a/app/models/change.rb +++ b/app/models/change.rb @@ -1,13 +1,48 @@ class Change - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model + include ActiveModel::Model + include ActiveModel::Validations - index_name "change-#{Rails.env}" + ATTRIBUTES = [:_id, + :created_at, + :updated_at, + :package, + :category, + :change_type, + :version, + :arches, + :commit] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :package, presence: true + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end + + def attributes + @created_at ||= DateTime.now + @updated_at = DateTime.now + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes + + # Converts the model to an OpenStruct instance + # + # @param [Array<Symbol>] fields Fields to export into the OpenStruct, or all fields if nil + # @return [OpenStruct] OpenStruct containing the selected fields + def to_os(*fields) + fields = all_fields if fields.empty? + OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }]) + end - attribute :package, String, mapping: { type: 'keyword' } - attribute :category, String, mapping: { type: 'keyword' } - attribute :change_type, String, mapping: { type: 'keyword' } - attribute :version, String, mapping: { type: 'keyword' } - attribute :arches, String, mapping: { type: 'keyword' } - attribute :commit, Hash, default: {}, mapping: { type: 'object' } end diff --git a/app/models/commit.rb b/app/models/commit.rb new file mode 100644 index 0000000..2512ced --- /dev/null +++ b/app/models/commit.rb @@ -0,0 +1,37 @@ +class Commit + include ActiveModel::Model + include ActiveModel::Validations + + ATTRIBUTES = [:id, + :author, + :email, + :date, + :message, + :files, + :packages, + :created_at, + :updated_at] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end + + def attributes + @created_at ||= DateTime.now + @updated_at = DateTime.now + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes + +end diff --git a/app/models/package.rb b/app/models/package.rb index 7ad3cbe..11ef135 100644 --- a/app/models/package.rb +++ b/app/models/package.rb @@ -1,31 +1,52 @@ class Package - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model + include ActiveModel::Model + include ActiveModel::Validations include Kkuleomi::Store::Models::PackageImport - include Kkuleomi::Store::Models::PackageSearch - - index_name "packages-#{Rails.env}" - - raw_fields = { - type: 'keyword' - } - - attribute :category, String, mapping: raw_fields - attribute :name, String, mapping: raw_fields - attribute :name_sort, String, mapping: raw_fields - attribute :atom, String, mapping: raw_fields - attribute :description, String, mapping: { type: 'text' } - attribute :longdescription, String, mapping: { type: 'text' } - attribute :homepage, String, default: [], mapping: raw_fields - attribute :license, String, mapping: raw_fields - attribute :licenses, String, default: [], mapping: raw_fields - attribute :herds, String, default: [], mapping: raw_fields - attribute :maintainers, Array, default: [], mapping: { type: 'object' } - attribute :useflags, Hash, default: {}, mapping: { type: 'object' } - attribute :metadata_hash, String, mapping: raw_fields + + ATTRIBUTES = [:id, + :created_at, + :updated_at, + :category, + :name, + :name_sort, + :atom, + :description, + :longdescription, + :homepage, + :license, + :licenses, + :herds, + :maintainers, + :useflags, + :metadata_hash] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :name, presence: true + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end + + def attributes + @id = @atom + @created_at ||= DateTime.now + @updated_at = DateTime.now + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes def category_model - @category_model ||= Category.find_by(:name, category) + @category_model ||= CategoryRepository.find_by(:name, category) end def to_param @@ -44,7 +65,7 @@ class Package end def versions - @versions ||= Version.find_all_by(:package, atom, sort: { sort_key: { order: 'asc' } }) + @versions ||= VersionRepository.find_all_by(:package, atom, sort: { sort_key: { order: 'asc' } }) end def latest_version @@ -65,6 +86,15 @@ class Package maintainers.empty? && herds.empty? end + # Converts the model to an OpenStruct instance + # + # @param [Array<Symbol>] fields Fields to export into the OpenStruct, or all fields if nil + # @return [OpenStruct] OpenStruct containing the selected fields + def to_os(*fields) + fields = all_fields if fields.empty? + OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }]) + end + private # Splits a license string into single licenses, stripping the permitted logic constructs diff --git a/app/models/useflag.rb b/app/models/useflag.rb index 131a89c..12758cb 100644 --- a/app/models/useflag.rb +++ b/app/models/useflag.rb @@ -1,14 +1,41 @@ class Useflag - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model - - index_name "useflags-#{Rails.env}" + include ActiveModel::Model + include ActiveModel::Validations + + ATTRIBUTES = [:id, + :created_at, + :updated_at, + :name, + :description, + :atom, + :scope, + :use_expand_prefix] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :name, presence: true + + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end - attribute :name, String, mapping: { type: 'keyword' } - attribute :description, String, mapping: { type: 'text' } - attribute :atom, String, mapping: { type: 'keyword' } - attribute :scope, String, mapping: { type: 'keyword' } - attribute :use_expand_prefix, String, mapping: { type: 'keyword' } + def attributes + @id = @name + '-' + (@atom || 'global' ) + '-' + @scope + @created_at ||= DateTime.now + @updated_at = DateTime.now + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes def all_fields [:name, :description, :atom, :scope, :use_expand_prefix] @@ -22,78 +49,14 @@ class Useflag name.gsub(use_expand_prefix + '_', '') end - class << self - # Retrieves all flags sorted by their state - def get_flags(name) - result = { local: {}, global: [], use_expand: [] } - - find_all_by(:name, name).each do |flag| - case flag.scope - when 'local' - result[:local][flag.atom] = flag - when 'global' - result[:global] << flag - when 'use_expand' - result[:use_expand] << flag - end - end - - result - end - - def suggest(q) - results = Useflag.search( - size: 20, - query: { match_phrase_prefix: { name: q } } - ) - - processed_results = {} - results.each do |result| - if processed_results.key? result.name - processed_results[result.name] = { - name: result.name, - description: '(multiple definitions)', - scope: 'multi' - } - else - processed_results[result.name] = result - end - end - - processed_results.values.sort { |a, b| a[:name].length <=> b[:name].length } - end - - # Loads the local USE flags for a given package in a name -> model hash - # - # @param [String] atom Package to find flags for - # @return [Hash] - def local_for(atom) - map_by_name find_all_by(:atom, atom) - end - - # Maps the global USE flags in the index by their name - # This is expensive! - # - def global - map_by_name find_all_by(:scope, 'global') - end - - # Maps the USE_EXPAND variables in the index by their name - # - def use_expand - map_by_name find_all_by(:scope, 'use_expand') - end - - private + # Converts the model to a Hash + # + # @param [Array<Symbol>] fields Fields to export into the Hash, or all fields if nil + # @return [Hash] Hash containing the selected fields + def to_hsh(*fields) + fields = all_fields if fields.empty? + Hash[fields.map { |field| [field, send(field)] }] + end - def map_by_name(collection) - map = {} - collection.each do |item| - map[item.name] = item - end - - map - end - end end diff --git a/app/models/version.rb b/app/models/version.rb index 62c72f8..429f4d1 100644 --- a/app/models/version.rb +++ b/app/models/version.rb @@ -1,23 +1,52 @@ +require 'date' + class Version - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model + include ActiveModel::Model + include ActiveModel::Validations include Kkuleomi::Store::Models::VersionImport - index_name "versions-#{Rails.env}" - - attribute :version, String, mapping: { type: 'keyword' } - attribute :package, String, mapping: { type: 'keyword' } - attribute :atom, String, mapping: { type: 'keyword' } - attribute :sort_key, Integer, mapping: { type: 'integer' } - attribute :slot, String, mapping: { type: 'keyword' } - attribute :subslot, String, mapping: { type: 'keyword' } - attribute :eapi, String, mapping: { type: 'keyword' } - attribute :keywords, String, mapping: { type: 'keyword' } - attribute :masks, Array, default: [], mapping: { type: 'object' } - attribute :use, String, default: [], mapping: { type: 'keyword' } - attribute :restrict, String, default: [], mapping: { type: 'keyword' } - attribute :properties, String, default: [], mapping: { type: 'keyword' } - attribute :metadata_hash, String, mapping: { type: 'keyword' } + ATTRIBUTES = [:id, + :created_at, + :updated_at, + :version, + :package, + :atom, + :sort_key, + :slot, + :subslot, + :eapi, + :keywords, + :masks, + :use, + :restrict, + :properties, + :metadata_hash] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :version, presence: true + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end + + def attributes + @id = @atom + @created_at ||= DateTime.now + @updated_at = DateTime.now + + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes # Returns the keywording state on a given architecture # @@ -136,26 +165,24 @@ class Version private def calc_useflags - result = { local: {}, global: {}, use_expand: {} } + result = { local: [], global: [], use_expand: [] } - local_flag_map = Useflag.local_for(atom.gsub("-#{version}", '')) + local_flag_map = UseflagRepository.local_for(atom.gsub("-#{version}", '')) local_flags = local_flag_map.keys use.sort.each do |flag| if local_flags.include? flag - result[:local][flag] = local_flag_map[flag].to_hsh + result[:local] << local_flag_map[flag].to_hsh else - useflag = Useflag.find_by(:name, flag) + useflag = UseflagRepository.find_by(:name, flag) # This should not happen, but let's be sure next unless useflag if useflag.scope == 'global' - result[:global][useflag.name] = useflag.to_hsh + result[:global] << useflag.to_hsh elsif useflag.scope == 'use_expand' - prefix = useflag.use_expand_prefix.upcase - result[:use_expand][prefix] ||= {} - result[:use_expand][prefix][useflag.name.gsub(useflag.use_expand_prefix + '_', '')] = useflag.to_hsh + result[:use_expand] << useflag.to_hsh end end end diff --git a/app/repositories/base_repository.rb b/app/repositories/base_repository.rb new file mode 100644 index 0000000..397b275 --- /dev/null +++ b/app/repositories/base_repository.rb @@ -0,0 +1,108 @@ +require 'forwardable' +require 'singleton' + +class BaseRepository + include Elasticsearch::Persistence::Repository + include Elasticsearch::Persistence::Repository::DSL + include Singleton + + client ElasticsearchClient.default + + class << self + extend Forwardable + def_delegators :instance, :find_all_by, :filter_all, :find_by, :find_all_by_parent, :all_sorted_by + def_delegators :instance, :find_sorted_by, :n_sorted_by + def_delegators :instance, :count, :search, :delete, :save, :refresh_index!, :create_index + end + + # Finds instances by exact IDs using the 'term' filter + def find_all_by(field, value, opts = {}) + search({ + size: 10_000, + query: { match: { field => value } } + }.merge(opts)) + end + + # Filter all instances by the given parameters + def filter_all(filters, opts = {}) + filter_args = [] + filters.each_pair { |field, value| filter_args << { term: { field => value } } } + + search({ + query: { + bool: { filter: { bool: { must: filter_args } } } + }, + size: 10_000 + }.merge(opts)) + end + + def find_by(field, value, opts = {}) + find_all_by(field, value, opts).first + end + + def find_all_by_parent(parent, opts = {}) + search(opts.merge( + size: 10_000, + query: { + bool: { + filter: { + has_parent: { + parent_type: parent.class.document_type, + query: { term: { _id: parent.id } } + } + }, + must: { + match_all: {} + } + } + }) + ) + end + + # Returns the given number of records of this class sorted by a field. + def find_sorted_by(field, value, sort_field, order, num_return, options = {}) + search({ + size: num_return, + query: { term: { field => value } }, + sort: { sort_field => { order: order } } + }.merge(options)) + end + + + # Returns n records of this class sorted by a field. + def n_sorted_by(n, field, order, options = {}) + search({ + size: n, + query: { match_all: {} }, + sort: { field => { order: order } } + }.merge(options)) + end + + # Returns all (by default 10k) records of this class sorted by a field. + def all_sorted_by(field, order, options = {}) + search({ + size: 10_000, + query: { match_all: {} }, + sort: { field => { order: order } } + }.merge(options)) + end + + # Converts the model to an OpenStruct instance + # + # @param [Array<Symbol>] fields Fields to export into the OpenStruct, or all fields if nil + # @return [OpenStruct] OpenStruct containing the selected fields + def to_os(*fields) + fields = all_fields if fields.empty? + OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }]) + end + + # Converts the model to a Hash + # + # @param [Array<Symbol>] fields Fields to export into the Hash, or all fields if nil + # @return [Hash] Hash containing the selected fields + def to_hsh(*fields) + fields = all_fields if fields.empty? + Hash[fields.map { |field| [field, send(field)] }] + end + +end
\ No newline at end of file diff --git a/app/repositories/category_repository.rb b/app/repositories/category_repository.rb new file mode 100644 index 0000000..5757633 --- /dev/null +++ b/app/repositories/category_repository.rb @@ -0,0 +1,30 @@ +require 'singleton' + +class CategoryRepository < BaseRepository + include Singleton + + client ElasticsearchClient.default + + index_name "categories-#{Rails.env}" + + klass Category + + mapping dynamic: 'strict' do + indexes :id, type: 'keyword' + indexes :name, type: 'text' + indexes :description, type: 'text' + indexes :metadata_hash, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + + # Parse the "created_at" and "updated_at" fields in the document + # + def deserialize(document) + hash = document['_source'] + hash['created_at'] = Time.parse(hash['created_at']).utc if hash['created_at'] + hash['updated_at'] = Time.parse(hash['updated_at']).utc if hash['updated_at'] + Category.new hash + end + +end diff --git a/app/repositories/change_repository.rb b/app/repositories/change_repository.rb new file mode 100644 index 0000000..84dca92 --- /dev/null +++ b/app/repositories/change_repository.rb @@ -0,0 +1,33 @@ +require 'singleton' + +class ChangeRepository < BaseRepository + include Singleton + + client ElasticsearchClient.default + + index_name "change-#{Rails.env}" + + klass Change + + mapping dynamic: 'strict' do + indexes :id, type: 'keyword' + indexes :package, type: 'keyword' + indexes :category, type: 'keyword' + indexes :change_type, type: 'keyword' + indexes :version, type: 'keyword' + indexes :arches, type: 'keyword' + indexes :commit, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + + # Parse the "created_at" and "updated_at" fields in the document + # + def deserialize(document) + hash = document['_source'] + hash['created_at'] = Time.parse(hash['created_at']).utc if hash['created_at'] + hash['updated_at'] = Time.parse(hash['updated_at']).utc if hash['updated_at'] + Change.new hash + end + +end diff --git a/app/repositories/commit_repository.rb b/app/repositories/commit_repository.rb new file mode 100644 index 0000000..b2086be --- /dev/null +++ b/app/repositories/commit_repository.rb @@ -0,0 +1,37 @@ +require 'singleton' + +class CommitRepository < BaseRepository + include Singleton + + client ElasticsearchClient.default + + index_name "commit-#{Rails.env}" + + klass Commit + + mapping dynamic: 'strict' do + indexes :id, type: 'keyword' + indexes :author, type: 'keyword' + indexes :email, type: 'keyword' + indexes :date, type: 'date' + indexes :message, type: 'text' + indexes :files do + indexes :modified, type: 'keyword' + indexes :deleted, type: 'keyword' + indexes :added, type: 'keyword' + end + indexes :packages, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + + # Parse the "created_at" and "updated_at" fields in the document + # + def deserialize(document) + hash = document['_source'] + hash['created_at'] = Time.parse(hash['created_at']).utc if hash['created_at'] + hash['updated_at'] = Time.parse(hash['updated_at']).utc if hash['updated_at'] + Commit.new hash + end + +end diff --git a/app/repositories/elasticsearch_client.rb b/app/repositories/elasticsearch_client.rb new file mode 100644 index 0000000..88de0c8 --- /dev/null +++ b/app/repositories/elasticsearch_client.rb @@ -0,0 +1,13 @@ +class ElasticsearchClient + + def self.default + @default ||= Elasticsearch::Client.new host: ENV['ELASTICSEARCH_URL'] || 'localhost:9200' + end + + private + + def initialize(*) + raise "Should not be initialiazed" + end + +end
\ No newline at end of file diff --git a/app/repositories/package_repository.rb b/app/repositories/package_repository.rb new file mode 100644 index 0000000..5caaf01 --- /dev/null +++ b/app/repositories/package_repository.rb @@ -0,0 +1,216 @@ +require 'forwardable' +require 'singleton' +require_relative './query_parser/search_query_parser' + +class PackageRepository < BaseRepository + include Singleton + + class << self + extend Forwardable + def_delegators :instance, :suggest, :resolve, :find_atoms_by_useflag, :default_search_size, :default_search, + :build_query, :match_wildcard, :match_phrase, :match_description, :match_category, :scoring_functions + end + + index_name "packages-#{Rails.env}" + + klass Package + + mapping dynamic: 'strict' do + indexes :id, type: 'keyword' + indexes :category, type: 'keyword' + indexes :name, type: 'keyword' + indexes :name_sort, type: 'keyword' + indexes :atom, type: 'keyword' + indexes :description, type: 'text' + indexes :longdescription, type: 'text' + indexes :homepage, type: 'keyword' + indexes :license, type: 'keyword' + indexes :licenses, type: 'keyword' + indexes :herds, type: 'keyword' + indexes :maintainers do + indexes :name, type: 'keyword' + indexes :description, type: 'text' + indexes :type, type: 'keyword' + indexes :restrict, type: 'keyword' + indexes :email, type: 'keyword' + end + indexes :useflags do + indexes :local do + indexes :scope, type: 'keyword' + indexes :name, type: 'keyword' + indexes :description, type: 'text' + indexes :atom, type: 'keyword' + indexes :use_expand_prefix, type: 'keyword' + end + indexes :global do + indexes :scope, type: 'keyword' + indexes :name, type: 'keyword' + indexes :description, type: 'text' + indexes :atom, type: 'keyword' + indexes :use_expand_prefix, type: 'keyword' + end + indexes :use_expand do + indexes :scope, type: 'keyword' + indexes :name, type: 'keyword' + indexes :description, type: 'text' + indexes :atom, type: 'keyword' + indexes :use_expand_prefix, type: 'keyword' + end + end + indexes :metadata_hash, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + + def suggest(q) + search(build_query(q, 20, 0)) + end + + # Tries to resolve a query atom to one or more packages + def resolve(atom) + [] if atom.nil? || atom.empty? + + PackageRepository.find_all_by(:atom, atom) + PackageRepository.find_all_by(:name, atom) + end + + # Searches the versions index for versions using a certain USE flag. + # Results are aggregated by package atoms. + def find_atoms_by_useflag(useflag) + VersionRepository.search( + size: 0, # collect all packages. + query: { + bool: { + must: { match_all: {} }, + filter: { term: { use: useflag } } + } + }, + aggs: { + group_by_package: { + terms: { + field: 'package', + order: { '_key' => 'asc' }, + # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html + # ES actually dislikes large sizes like this (it defines 10k buckets basically) and it will be *very* expensive but lets try it and see. + # Other limits in this app are also 10k mostly to 'make things fit kinda'. + size: 10000, + } + } + }, + ).response.aggregations['group_by_package'].buckets + end + + def default_search_size + 25 + end + + def default_search(q, offset, search_size=default_search_size) + return [] if q.nil? || q.empty? + + search(build_query(q, search_size , offset)) + + end + + def build_query(q, size, offset) + parser = Object.const_get("SearchQueryParser::QueryParser").new + transformer = Object.const_get("SearchQueryParser::QueryTransformer").new + + { + size: size, + from: offset, + query: { + function_score: { + query: { bool: transformer.apply(parser.parse(q)).to_elasticsearch }, + functions: scoring_functions + } + } + } + end + + def bool_query_parts(q, category = nil) + q_dwncsd = q.downcase + + query = { + must: [ + match_wildcard(q_dwncsd) + ], + should: [ + match_phrase(q_dwncsd), + match_description(q) + ] + } + + query[:must] << [match_category(category)] if category + + query + end + + def match_wildcard(q) + q = ('*' + q + '*') unless q.include? '*' + q.tr!(' ', '*') + + { + wildcard: { + name_sort: { + wildcard: q, + boost: 4 + } + } + } + end + + def match_phrase(q) + { + match_phrase: { + name: { + query: q, + boost: 5 + } + } + } + end + + def match_description(q) + { + match: { + description: { + query: q, + boost: 0.1 + } + } + } + end + + def match_category(cat) + { + match: { + category: { + query: cat, + boost: 2 + } + } + } + end + + def scoring_functions + [ + { + filter: { + term: { + category: 'virtual' + } + }, + weight: 0.6 + } + ] + end + + # Parse the "created_at" and "updated_at" fields in the document + # + def deserialize(document) + hash = document['_source'] + hash['created_at'] = Time.parse(hash['created_at']).utc if hash['created_at'] + hash['updated_at'] = Time.parse(hash['updated_at']).utc if hash['updated_at'] + Package.new hash + end + +end diff --git a/app/repositories/query_parser/search_query_parser.rb b/app/repositories/query_parser/search_query_parser.rb new file mode 100644 index 0000000..f3e67c6 --- /dev/null +++ b/app/repositories/query_parser/search_query_parser.rb @@ -0,0 +1,145 @@ +require 'parslet' + +module SearchQueryParser + + class QueryParser < Parslet::Parser + rule(:term) { match('[^\s"]').repeat(1).as(:term) } + rule(:quote) { str('"') } + rule(:operator) { (str('+') | str('-')).as(:operator) } + + rule(:fieldname) { match('[^\s:"]').repeat(1).as(:fieldname) } + rule(:field) { (fieldname >> str(':')).as(:field) } + + rule(:phrase) do + (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) + end + rule(:clause) { (operator.maybe >> field.maybe >> (phrase | term)).as(:clause) } + rule(:space) { match('\s').repeat(1) } + rule(:query) { (clause >> space.maybe).repeat.as(:query) } + root(:query) + end + + class QueryTransformer < Parslet::Transform + rule(:clause => subtree(:clause)) do + if clause[:term] + TermClause.new(clause[:operator]&.to_s, clause[:field], clause[:term].to_s) + elsif clause[:phrase] + phrase = clause[:phrase].map { |p| p[:term].to_s }.join(" ") + PhraseClause.new(clause[:operator]&.to_s, clause[:field], phrase) + else + raise "Unexpected clause type: '#{clause}'" + end + end + rule(:query => sequence(:clauses)) { Query.new(clauses) } + end + + class Operator + def self.symbol(str) + case str + when '+' + :must + when '-' + :must_not + when nil + :should + else + raise "Unknown operator: #{str}" + end + end + end + + class TermClause + attr_accessor :operator, :field, :term + + def initialize(operator, field, term) + self.operator = Operator.symbol(operator) + self.field = field + self.term = term + end + end + + class PhraseClause + attr_accessor :operator, :field, :phrase + + def initialize(operator, field, phrase) + self.operator = Operator.symbol(operator) + self.field = field + self.phrase = phrase + end + end + + class Query + attr_accessor :should_clauses, :must_not_clauses, :must_clauses + + def initialize(clauses) + grouped = clauses.chunk { |c| c.operator }.to_h + self.should_clauses = grouped.fetch(:should, []) + self.must_not_clauses = grouped.fetch(:must_not, []) + self.must_clauses = grouped.fetch(:must, []) + end + + def to_elasticsearch + query = { } + + if should_clauses.any? + query[:should] = should_clauses.map do |clause| + clause_to_query(clause) + end + end + + if must_clauses.any? + query[:must] = must_clauses.map do |clause| + clause_to_query(clause) + end + end + + if must_not_clauses.any? + query[:must_not] = must_not_clauses.map do |clause| + clause_to_query(clause) + end + end + + query + end + + def clause_to_query(clause) + case clause + when TermClause + match(clause.field, clause.term) + when PhraseClause + match_phrase(clause.field, clause.phrase) + else + raise "Unknown clause type: #{clause}" + end + end + + def match(field, term) + if field + { + :match => { + field[:fieldname].to_s.to_sym => { + :query => term + } + } + } + else + { + :multi_match => { + :query => term, + :fields => ["atom^3", "name^2"] + } + } + end + end + + def match_phrase(field, phrase) + { + :match_phrase => { + field ? field[:fieldname].to_s.to_sym : :name => { + :query => phrase + } + } + } + end + end +end diff --git a/app/repositories/useflag_repository.rb b/app/repositories/useflag_repository.rb new file mode 100644 index 0000000..5bc1e00 --- /dev/null +++ b/app/repositories/useflag_repository.rb @@ -0,0 +1,105 @@ +require 'singleton' + +class UseflagRepository < BaseRepository + include Singleton + + class << self + extend Forwardable + def_delegators :instance, :get_flags, :suggest, :local_for, :global, :use_expand + end + + index_name "useflags-#{Rails.env}" + + klass Useflag + + mapping dynamic: 'strict' do + indexes :id, type: 'keyword' + indexes :name, type: 'text' + indexes :description, type: 'text' + indexes :atom, type: 'keyword' + indexes :scope, type: 'keyword' + indexes :use_expand_prefix, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + + + # Retrieves all flags sorted by their state + def get_flags(name) + result = { local: {}, global: [], use_expand: [] } + + find_all_by(:name, name).each do |flag| + case flag.scope + when 'local' + result[:local][flag.atom] = flag + when 'global' + result[:global] << flag + when 'use_expand' + result[:use_expand] << flag + end + end + + result + end + + def suggest(q) + results = search( + size: 20, + query: { match_phrase_prefix: { name: q } } + ) + + processed_results = {} + results.each do |result| + if processed_results.key? result.name + processed_results[result.name] = Useflag.new ({ "name"=> result.name, "description" => '(multiple definitions)', "scope" => 'multi' }) + else + processed_results[result.name] = result + end + end + + processed_results.values.sort { |a, b| a.name.length <=> b.name.length } + end + + # Loads the local USE flags for a given package in a name -> model hash + # + # @param [String] atom Package to find flags for + # @return [Hash] + def local_for(atom) + map_by_name find_all_by(:atom, atom) + end + + # Maps the global USE flags in the index by their name + # This is expensive! + # + def global + map_by_name find_all_by(:scope, 'global') + end + + # Maps the USE_EXPAND variables in the index by their name + # + def use_expand + map_by_name find_all_by(:scope, 'use_expand') + end + + # Parse the "created_at" and "updated_at" fields in the document + # + def deserialize(document) + hash = document['_source'] + hash['created_at'] = Time.parse(hash['created_at']).utc if hash['created_at'] + hash['updated_at'] = Time.parse(hash['updated_at']).utc if hash['updated_at'] + Useflag.new hash + end + + private + + def map_by_name(collection) + map = {} + + collection.each do |item| + map[item.name] = item + end + + map + end + +end diff --git a/app/repositories/version_repository.rb b/app/repositories/version_repository.rb new file mode 100644 index 0000000..337ce38 --- /dev/null +++ b/app/repositories/version_repository.rb @@ -0,0 +1,66 @@ +require 'singleton' + +class VersionRepository < BaseRepository + include Singleton + + class << self + extend Forwardable + def_delegators :instance, :get_popular_useflags + end + + index_name "versions-#{Rails.env}" + + klass Version + + mapping dynamic: 'strict' do + indexes :id, type: 'keyword' + indexes :version, type: 'keyword' + indexes :package, type: 'keyword' + indexes :atom, type: 'keyword' + indexes :sort_key, type: 'integer' + indexes :slot, type: 'keyword' + indexes :subslot, type: 'keyword' + indexes :eapi, type: 'keyword' + indexes :keywords, type: 'keyword' + indexes :masks do + indexes :arches, type: 'keyword' + indexes :atoms, type: 'keyword' + indexes :author, type: 'keyword' + indexes :date, type: 'keyword' + indexes :reason, type: 'text' + end + indexes :use, type: 'keyword' + indexes :restrict, type: 'keyword' + indexes :properties, type: 'keyword' + indexes :metadata_hash, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + + # Retrieves the most widely used USE flags by all versions + # Note that packages with many versions are over-represented + def get_popular_useflags(n = 50) + search( + query: { match_all: {} }, + aggs: { + group_by_flag: { + terms: { + field: 'use', + size: n + } + } + }, + size: 0 + ).response.aggregations['group_by_flag'].buckets + end + + # Parse the "created_at" and "updated_at" fields in the document + # + def deserialize(document) + hash = document['_source'] + hash['created_at'] = Time.parse(hash['created_at']).utc if hash['created_at'] + hash['updated_at'] = Time.parse(hash['updated_at']).utc if hash['updated_at'] + Version.new hash + end + +end diff --git a/app/views/about/index.html.erb b/app/views/about/index.html.erb index 5d9ed0f..efa98a2 100644 --- a/app/views/about/index.html.erb +++ b/app/views/about/index.html.erb @@ -15,6 +15,11 @@ <h2>FAQ</h2> <dl> + <dt>How do I use advanced search queries?</dt> + <dd> + Please view the <a href="/about/queries">advanced search queries page</a> for further information about advanced search queries. + </dd> + <br> <dt>How often is the site updated?</dt> <dd> Updates are scheduled <strong>every 10 minutes</strong> and are processed using delayed jobs. diff --git a/app/views/about/queries.html.erb b/app/views/about/queries.html.erb new file mode 100644 index 0000000..bdba20a --- /dev/null +++ b/app/views/about/queries.html.erb @@ -0,0 +1,171 @@ +<ol class="breadcrumb"> + <li><a href="/"><%= t :home %></a></li> + <li><a href="/about"><%= t :about %></a></li> + <li class="active"><%= t :queries %></li> +</ol> + +<h1><%= t :queries %></h1> + +This website provides a search functionality to find Gentoo packages. You can use field/value pairs combined with operators to run advanced search queries. +The possible fields and operators are summarized in the following tables: +<ul style="margin-top:5px;"> + <li><a href="#fields">Possible Fields</a></li> + <li><a href="#operators">Possible Operators</a></li> + <li><a href="#examples">Examples</a></li> +</ul> + +<hr> + +<h2 id="fields">Possible Fields</h2> + +<table class="table"> + <thead> + <tr> + <th scope="col">Field</th> + <th scope="col">Description</th> + </tr> + </thead> + <tbody> + <tr> + <th scope="row">atom</th> + <td>The unique identifier of a package <br> <i style="padding-left:2em">e.g. sys-kernel/gentoo-sources</i></td> + </tr> + <tr> + <th scope="row">category</th> + <td>The category of a package <br> <i style="padding-left:2em">e.g. sys-kernel</i></td> + </tr> + <tr> + <th scope="row">name</th> + <td>The name of a package <br> <i style="padding-left:2em">e.g. gentoo-sources</i></td> + </tr> + <tr> + <th scope="row">description</th> + <td>The description of a package <br> <i style="padding-left:2em">e.g. A tiling window manager</i> </td> + </tr> + <tr> + <th scope="row">longdescription</th> + <td>The full descripiton of a package <br> <i style="padding-left:2em">e.g. xmonad is a tiling window manager for [...]</i></td> + </tr> + <tr> + <th scope="row">homepage</th> + <td>The homepage of a package <br> <i style="padding-left:2em">e.g. http://xmonad.org</i></td> + </tr> + <tr> + <th scope="row">license</th> + <td>The license of a package <br> <i style="padding-left:2em">e.g. BSD</i></td> + </tr> + <tr> + <th scope="row">Maintainers</th> + <td></td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">maintainers.name</th> + <td>The name of the maintainer <br> <i style="padding-left:2em">e.g. Gentoo Haskell</i></td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">maintainers.description</th> + <td>The description of the maintainers</td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">maintainers.type</th> + <td>The type of maintainter <br> <i style="padding-left:2em">e.g. project</i></td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">maintainers.restrict</th> + <td></td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">maintainers.email</th> + <td>The email of the maintainer <br> <i style="padding-left:2em">e.g. haskell@gentoo.org</i></td> + </tr> + <tr> + <th scope="row">Useflag</th> + <td></td> + </tr> + <tr> + <th scope="row" style="padding-left:1em">global</th> + <td></td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">useflags.global.name</th> + <td>The name of the global useflag <br> <i style="padding-left:2em">e.g. hscolour</i></td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">useflags.global.description</th> + <td>The description of the global useflag <br> <i style="padding-left:2em">e.g. Include coloured haskell sources to [...]</i></td> + </tr> + <tr> + <th scope="row" style="padding-left:1em">local</th> + <td></td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">useflags.local.name</th> + <td>The name of the local useflag</td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">useflags.local.description</th> + <td>The description of the local useflag</td> + </tr> + <tr> + <th scope="row" style="padding-left:1em">use_expand</th> + <td></td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">useflags.use_expand.name</th> + <td>The name of the local use_expand</td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">useflags.use_expand.description</th> + <td>The description of the use_expand</td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">useflags.use_expand.use_expand_prefix</th> + <td>The use_expand prefix <br> <i style="padding-left:2em">e.g. python_targets</i></td> + </tr> + <tr> + <th scope="row">metadata_hash</th> + <td>The hash of the metadata <br> <i style="padding-left:2em">e.g. 5cd76e098f966b4edcd1848866dd9099</i></td> + </tr> + </tbody> +</table> +<h2 id="operators">Possible Operators</h2> +The following operators can be used to combine multiple field/value pairs: +<table class="table"> + <thead> + <tr> + <th scope="col">Operator</th> + <th scope="col">Description</th> + </tr> + </thead> + <tbody> + <tr> + <th scope="row"></th> + <td>The term <b>should</b> appear (default)</td> + </tr> + <tr> + <th scope="row">+</th> + <td>The term <b>must</b> appear</td> + </tr> + <tr> + <th scope="row">-</th> + <td>The term <b>must not</b> appear</td> + </tr> + <tr> + <th scope="row">"..."</th> + <td>Can be used to <b>group</b> phrases <br> <i style="padding-left:2em">e.g. +description:"window manager"</i></td> + </tr> + </tbody> +</table> + +<h2 id="examples">Examples</h2> + +<ul> + <li>Find all packages named git: <br><code style="margin-left:2em">+name:git</code></li> + <li>Find all packages in the category sys-kernel: <br><code style="margin-left:2em">+category:sys-kernel</code></li> + <li>Find all packages with a BSD license: <br><code style="margin-left:2em">+license:BSD</code></li> + <li>Find all packages that neither have a BSD license nor a MIT license: <br><code style="margin-left:2em">-license:BSD -license:MIT</code></li> + <li>Find all packages maintained by the Haskell Team: <br><code style="margin-left:2em">+maintainer.email:haskell@gentoo.org</code></li> + <li>Find all packages maintained by the Haskell Team but that aren't in the 'dev-haskell' category: <br><code style="margin-left:2em">+maintainer.email:haskell@gentoo.org -category:dev-haskell</code></li> + <li>Find all packages those description contains 'window manager': <br><code style="margin-left:2em">+description:"window manager"</code></li> + <li>Find all packages that contain the use_expand 'python_targets': <br><code style="margin-left:2em">+useflags.use_expand.use_expand_prefix:python_targets</code></li> +</ul>
\ No newline at end of file diff --git a/app/views/arches/keyworded.html.erb b/app/views/arches/keyworded.html.erb index b7ae03d..ae1df29 100644 --- a/app/views/arches/keyworded.html.erb +++ b/app/views/arches/keyworded.html.erb @@ -12,7 +12,7 @@ <% cache("keyworded-full-#{@arch}-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'packages/changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> diff --git a/app/views/arches/stable.html.erb b/app/views/arches/stable.html.erb index b1a4548..eb66245 100644 --- a/app/views/arches/stable.html.erb +++ b/app/views/arches/stable.html.erb @@ -12,7 +12,7 @@ <% cache("stable-full-#{@arch}-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'packages/changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> diff --git a/app/views/feeds/changes.atom.builder b/app/views/feeds/changes.atom.builder index 5991f45..a8af3df 100644 --- a/app/views/feeds/changes.atom.builder +++ b/app/views/feeds/changes.atom.builder @@ -10,7 +10,7 @@ atom_feed(id: atom_id(@feed_type, @feed_id, 'feed')) do |feed| @changes.each do |change| atom = cp_to_atom change.category, change.package - package = Package.find_by :atom, atom + package = PackageRepository.find_by :atom, atom if package.nil? logger.warn "Package for change (#{change}) nil!" next diff --git a/app/views/feeds/packages.atom.builder b/app/views/feeds/packages.atom.builder new file mode 100644 index 0000000..bac7686 --- /dev/null +++ b/app/views/feeds/packages.atom.builder @@ -0,0 +1,41 @@ +@feed_id ||= nil + +atom_feed(id: atom_id(@feed_type, @feed_id, 'feed')) do |feed| + + all_packages = PackageRepository.default_search(@query, 0, 10_000) + + feed.title @feed_title + feed.updated !all_packages.empty? ? all_packages.first.created_at : Time.now + + feed.author do |author| + author.name 'Gentoo Packages Database' + end + + all_packages.each do |package| + atom = package.atom + + commit = CommitRepository.find_sorted_by :packages, atom, :date, "desc", 1 + commit = commit.first + + if package.nil? + logger.warn "Package nil!" + next + end + + id = atom + + feed.entry( + package, + id: atom_id(@feed_type, @feed_id, id), + url: absolute_link_to_package(atom)) do |entry| + entry.updated commit ? commit.date.to_datetime.rfc3339 : Time.now.to_datetime.rfc3339 + + entry.title(t :feed_keyworded_title, + atom: atom, + description: package.description) + entry.content(t :feed_commit_content, + hash: commit ? commit.id[0..6] : "", + message: commit ? commit.message : "No commit available") + end + end +end diff --git a/app/views/index/_package.html.erb b/app/views/index/_package.html.erb index eeb3109..a364209 100644 --- a/app/views/index/_package.html.erb +++ b/app/views/index/_package.html.erb @@ -1,4 +1,4 @@ -<%- package = Package.find_by(:atom, cp_to_atom(change.category, change.package)); unless package.nil? -%> +<%- package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)); unless package.nil? -%> <tr> <td> <a href="<%= slf(package_path(cp_to_atom(change.category, change.package))) %>"> diff --git a/app/views/index/index.html.erb b/app/views/index/index.html.erb index 890a5f3..ee2bd7f 100644 --- a/app/views/index/index.html.erb +++ b/app/views/index/index.html.erb @@ -1,5 +1,5 @@ <div class="jumbotron"> - <h2 class="site-welcome stick-top">Welcome to the Home of <span class="text-primary"><%= number_with_delimiter Package.count %></span> Gentoo Packages</h2> + <h2 class="site-welcome stick-top">Welcome to the Home of <span class="text-primary"><%= number_with_delimiter PackageRepository.count %></span> Gentoo Packages</h2> <form action="<%= search_packages_path %>" method="get"> <div class="typeahead-container"> @@ -8,6 +8,11 @@ <input id="q" name="q" type="search" autocomplete="off" placeholder="<%= t :find_packages %>" aria-label="<%= t :find_packages %>" autofocus> </span> <span class="typeahead-button"> + <button type="button" onclick="$('#searchHelp').modal('show')" title="Help" aria-label="<%= "Help" %>"> + <span class="fa fa-question" style="font-size: 15px;"></span><span class="sr-only"><%= "Help" %></span> + </button> + </span> + <span class="typeahead-button"> <button type="submit" title="<%= t :find %>" aria-label="<%= t :find %>"> <span class="typeahead-search-icon"></span><span class="sr-only"><%= t :find %></span> </button> @@ -43,11 +48,91 @@ </div> <ul class="list-group"> <% @version_bumps.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'packages/changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> </div> <% end %> +<div class="modal fade" id="searchHelp" tabindex="-1" role="dialog" aria-labelledby="searchHelpTitle"> + <div class="modal-dialog modal-lg" role="document"> + <div class="modal-content"> + <div class="modal-header"> + <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">×</span></button> + <h3 class="modal-title" id="searchHelpTitle"><span class="fa fa-info" style="font-size: 15px;"></span> Advanced Search Queries</h3> + </div> + <div class="modal-body"> + Using the following forms you can compose advanced search queries instead of writing them manually. + + <div id="search-container" style="margin-bottom:25px;"> + <div class="row" style="margin-top:25px;"> + <div class="col-lg-2"></div> + <div class="col-lg-8"> + <div class="input-group"> + <div class="input-group-btn"> + <div class="btn-group"> + <button type="button" class="pgo-query-field btn btn-default dropdown-toggle" data-toggle="dropdown"><span>name</span> + <span class="caret"></span></button> + <ul class="dropdown-menu" role="menu"> + <li><a onclick="updateDropdown(this);">name</a></li> + <li><a onclick="updateDropdown(this);">category</a></li> + <li><a onclick="updateDropdown(this);">atom</a></li> + <li><a onclick="updateDropdown(this);">description</a></li> + <li><a onclick="updateDropdown(this);">longdescription</a></li> + <li><a onclick="updateDropdown(this);">license</a></li> + <li class="divider"></li> + <li><a onclick="updateDropdown(this);">maintainers.name</a></li> + <li><a onclick="updateDropdown(this);">maintainers.description</a></li> + <li><a onclick="updateDropdown(this);">maintainers.type</a></li> + <li><a onclick="updateDropdown(this);">maintainers.restrict</a></li> + <li><a onclick="updateDropdown(this);">maintainers.email</a></li> + <li class="divider"></li> + <li><a onclick="updateDropdown(this);">useflags.global.name</a></li> + <li><a onclick="updateDropdown(this);">useflags.global.description</a></li> + <li><a onclick="updateDropdown(this);">useflags.local.name</a></li> + <li><a onclick="updateDropdown(this);">useflags.local.description</a></li> + <li><a onclick="updateDropdown(this);">useflags.use_expand.name</a></li> + <li><a onclick="updateDropdown(this);">useflags.use_expand.description</a></li> + <li><a onclick="updateDropdown(this);">useflags.use_expand.use_expand_prefix</a></li> + <li class="divider"></li> + <li><a onclick="updateDropdown(this);">metadata_hash</a></li> + </ul> + </div> + <div class="btn-group"> + <button type="button" class="pgo-query-operator btn btn-default dropdown-toggle" data-toggle="dropdown"><span>should match</span> + <span class="caret"></span></button> + <ul class="dropdown-menu" role="menu"> + <li><a onclick="updateDropdown(this);">should match</a></li> + <li><a onclick="updateDropdown(this);">must match</a></li> + <li><a onclick="updateDropdown(this);">must not match</a></li> + </ul> + </div> + </div><!-- /btn-group --> + <input type="text" class="form-control" placeholder="e.g. gentoo-sources"> + </div><!-- /input-group --> + + </div><!-- /.col-lg-6 --> + <div class="col-lg-2"> + <span class="pgo-query-delete-btn fa fa-trash pull-right" style="font-size: 20px;margin-top:5px;" onclick="deleteInput(this);"></span> + <span class="pgo-query-add-btn fa fa-plus pull-right" style="font-size: 20px;margin-top:5px;" onclick="addInput();"></span> + </div> + + </div><!-- /.row --> + + </div> + + Please refer to <a href="/about/queries">this page</a> for further information on advanced search queries and examples. + + </div> + <div class="modal-footer"> + <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> + <button type="button" class="btn btn-primary" onclick="buildAdvancedQuery();" data-dismiss="modal">Apply</button> + </div> + </div> + </div> +</div> + +<%= javascript_include_tag 'index/query_generator.js' %> + <%= javascript_include_tag 'index/typeahead.js' %> diff --git a/app/views/packages/_changed_package.html.erb b/app/views/packages/_changed_package.html.erb index 5b407fb..d6a0d00 100644 --- a/app/views/packages/_changed_package.html.erb +++ b/app/views/packages/_changed_package.html.erb @@ -46,10 +46,10 @@ </small> <% unless (changelog_entry = matching_changelog_entry(change)).nil? %> <div class="kk-inline-changelog-entry"> - <a href="<%= gitweb_commit_url(changelog_entry[:id]) %>" title="<%= t :git_commit %>"> + <a href="<%= gitweb_commit_url(changelog_entry.id) %>" title="<%= t :git_commit %>"> <span class="octicon octicon-git-pull-request"></span> <span class="kk-commit-message"> - <%= changelog_entry[:message].lines.first %> + <%= changelog_entry.message.lines.first %> </span> </a> </div> diff --git a/app/views/packages/_changelog_entry.html.erb b/app/views/packages/_changelog_entry.html.erb index e592f89..17a6e66 100644 --- a/app/views/packages/_changelog_entry.html.erb +++ b/app/views/packages/_changelog_entry.html.erb @@ -1,28 +1,29 @@ <li class="list-group-item"> - <strong><%= annotate_bugs changelog[:message].lines.first %></strong> + <strong><%= annotate_bugs changelog.message.lines.first %></strong> <div class="kk-byline"> - <%= mail_to changelog[:email], changelog[:author] %>, - <%= i18n_date(changelog[:date]) %>, - commit <%= link_to_gitweb_commit changelog[:id]%> + <%= mail_to changelog.email, changelog.author %>, + <%= i18n_date(changelog.date) %>, + commit <%= link_to_gitweb_commit changelog.id%> </div> <table class="table table-condensed kk-changelog-diffstat"> - <% unless changelog[:files][:added].empty? %> + + <% unless changelog.files["added"].empty? %> <tr class="success"> <td class="kk-changelog-diffstat-icon"><span class="octicon octicon-diff-added"></span></td> - <td><%= safe_join(changelog[:files][:added].map {|f| link_to_gitweb_ebuild_diff(f, changelog[:id], @package.category, @package.name) }, ', ') %></td> + <td><%= safe_join(changelog.files["added"].select { |file| file.include?(@package.category + '/' + @package.name) }.map {|f| link_to_gitweb_ebuild_diff(f.split('/').last, changelog.id, @package.category, @package.name) }, ', ') %></td> </tr> <% end %> - <% unless changelog[:files][:modified].empty? %> + <% unless changelog.files["modified"].empty? %> <tr class="warning"> <td class="kk-changelog-diffstat-icon"><span class="octicon octicon-diff-modified"></span></td> - <td><%= safe_join(changelog[:files][:modified].map {|f| link_to_gitweb_ebuild_diff(f, changelog[:id], @package.category, @package.name) }, ', ') %></td> + <td><%= safe_join(changelog.files["modified"].select { |file| file.include?(@package.category + '/' + @package.name) }.map {|f| link_to_gitweb_ebuild_diff(f.split('/').last, changelog.id, @package.category, @package.name) }, ', ') %></td> </tr> <% end %> - <% unless changelog[:files][:deleted].empty? %> + <% unless changelog.files["deleted"].empty? %> <tr class="danger"> <td class="kk-changelog-diffstat-icon"><span class="octicon octicon-diff-removed"></span></td> - <td><%= safe_join(changelog[:files][:deleted].map {|f| link_to_gitweb_ebuild_diff(f, changelog[:id], @package.category, @package.name) }, ', ') %></td> + <td><%= safe_join(changelog.files["deleted"].select { |file| file.include?(@package.category + '/' + @package.name) }.map {|f| link_to_gitweb_ebuild_diff(f.split('/').last, changelog.id, @package.category, @package.name) }, ', ') %></td> </tr> <% end %> diff --git a/app/views/packages/_metadata.html.erb b/app/views/packages/_metadata.html.erb index 426afd9..5568c08 100644 --- a/app/views/packages/_metadata.html.erb +++ b/app/views/packages/_metadata.html.erb @@ -3,7 +3,7 @@ <h3 class="panel-title"><%= t :box_metadata %></h3> </div> <ul class="list-group kk-metadata-list"> - <% if package.homepage.size > 1 %> + <% if !package.homepage.nil? && package.homepage.size > 1 %> <li class="kk-metadata-item list-group-item"> <div class="row"> <div class="col-xs-12 col-md-3 kk-metadata-key"> diff --git a/app/views/packages/_metadata_use.html.erb b/app/views/packages/_metadata_use.html.erb index d33b751..fdf2b0b 100644 --- a/app/views/packages/_metadata_use.html.erb +++ b/app/views/packages/_metadata_use.html.erb @@ -7,7 +7,7 @@ <%= render partial: 'useflag', object: useflags['global'], as: 'useflags' %> <% end %> <% unless useflags['use_expand'].empty? %> - <% useflags['use_expand'].each_pair do |flag, values| %> + <% useflags['use_expand'].group_by { |u| u['use_expand_prefix'] }.each_pair do |flag, values| %> <span class="kk-useflag-group"><%= t :use_expand_flag, flag: flag %></span> <%= render partial: 'useflag', object: values, as: 'useflags' %> <% end %> diff --git a/app/views/packages/_package_header.html.erb b/app/views/packages/_package_header.html.erb index 1b7876b..8c611da 100644 --- a/app/views/packages/_package_header.html.erb +++ b/app/views/packages/_package_header.html.erb @@ -25,7 +25,7 @@ <%= package.description %> </p> - <% unless package.homepage.empty? || package.homepage.first.nil? || package.homepage.first.empty? %> + <% unless package.homepage.nil? || package.homepage.first.nil? || package.homepage.first.empty? %> <p class="kk-package-homepage"> <%= content_tag :a, package.homepage.first, href: package.homepage.first, rel: 'nofollow' %> </p> diff --git a/app/views/packages/_resources.html.erb b/app/views/packages/_resources.html.erb index 40a2547..34c8486 100644 --- a/app/views/packages/_resources.html.erb +++ b/app/views/packages/_resources.html.erb @@ -7,9 +7,9 @@ <span class="fa fa-fw fa-bug"></span> <%= t :res_bugs %> </a> - <a href="https://wiki.gentoo.org/index.php?title=Special%3ASearch&fulltext=Search&search=<%= u package.name %>" class="list-group-item" target="_blank"> + <a href="https://wiki.gentoo.org/wiki/Special:Search/<%= u package.name %>" class="list-group-item" target="_blank"> <span class="fa fa-fw fa-book"></span> - <%= t :res_docs %> + <%= documentation_label(package.name) %> </a> <a href="https://forums.gentoo.org/search.php?search_terms=all&show_results=topics&search_keywords=<%= u package.name %>&mode=results" class="list-group-item" target="_blank"> <span class="fa fa-fw fa-comments-o"></span> diff --git a/app/views/packages/_useflag.html.erb b/app/views/packages/_useflag.html.erb index a60e589..04ed0cd 100644 --- a/app/views/packages/_useflag.html.erb +++ b/app/views/packages/_useflag.html.erb @@ -1,5 +1,5 @@ <ul class="kk-useflag-container <%= useflags.size > 10 ? 'kk-useflag-container-many' : 'kk-useflag-container-few' %>"> -<% useflags.each_pair do |flag, flag_data| %> - <li class="kk-useflag"><%= link_to flag, useflag_path(id: flag_data['name']), :title => strip_tags(flag_data['description']), 'data-toggle' => 'tooltip' %></li> +<% useflags.each do |flag_data| %> + <li class="kk-useflag"><%= link_to flag_data['use_expand_prefix'].nil? ? flag_data['name'] : flag_data['name'].gsub(flag_data['use_expand_prefix'] + '_', '') , useflag_path(id: flag_data['name']), :title => strip_tags(flag_data['description']), 'data-toggle' => 'tooltip' %></li> <% end %> </ul> diff --git a/app/views/packages/added.html.erb b/app/views/packages/added.html.erb index 97d5cb6..589226a 100644 --- a/app/views/packages/added.html.erb +++ b/app/views/packages/added.html.erb @@ -12,7 +12,7 @@ <% cache("added-full-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'changed_package', object: change, as: 'change', locals: { package: _package, version: _package.latest_version } %> <% end %> </ul> diff --git a/app/views/packages/keyworded.html.erb b/app/views/packages/keyworded.html.erb index ff5b60c..a83a558 100644 --- a/app/views/packages/keyworded.html.erb +++ b/app/views/packages/keyworded.html.erb @@ -12,7 +12,7 @@ <% cache("keyworded-full-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> diff --git a/app/views/packages/search.html.erb b/app/views/packages/search.html.erb index fe77dd3..80f6bf3 100644 --- a/app/views/packages/search.html.erb +++ b/app/views/packages/search.html.erb @@ -1,20 +1,24 @@ -<h1 class="first-header">Search Results <small>for <%= params[:q] %></small></h1> +<h1 class="first-header">Search Results <small>for <%= params[:q] %></small> + <%= feed_icon search_packages_url(format: :atom, params: request.query_parameters) %></h1> <% if @packages.size > 0 %> <div class="panel panel-default"> <div class="panel-heading"> - Results <%= @offset + 1 %>—<%= [@offset + Package.default_search_size, @packages.total].min %> of <%= @packages.total %> + Results <%= @offset + 1 %>—<%= [@offset + PackageRepository.default_search_size, @packages.total].min %> of <%= @packages.total %> </div> <div class="list-group"> <%= render partial: 'package_result_row', collection: @packages, as: 'package' %> </div> <div class="panel-footer"> <div class="btn-group" role="group" aria-label="Result navigation"> - <%= link_to '< Prev', search_packages_path(q: params[:q], o: [@offset - Package.default_search_size, 0].max), class: 'btn btn-default' + (@offset > 0 ? '' : ' disabled') %> - <%= link_to 'Next >', search_packages_path(q: params[:q], o: @offset + Package.default_search_size), class: 'btn btn-default ' + ((@offset + Package.default_search_size) > @packages.total ? 'disabled' : '') %> + <%= link_to '< Prev', search_packages_path(q: params[:q], o: [@offset - PackageRepository.default_search_size, 0].max), class: 'btn btn-default' + (@offset > 0 ? '' : ' disabled') %> + <%= link_to 'Next >', search_packages_path(q: params[:q], o: @offset + PackageRepository.default_search_size), class: 'btn btn-default ' + ((@offset + PackageRepository.default_search_size) > @packages.total ? 'disabled' : '') %> </div> </div> </div> +<% content_for :head do %> + <%= alternate_feed_link(search_packages_url(format: :atom, params: request.query_parameters), t(:atom_feed)) %> +<% end %> <% else %> <div class="jumbotron"> <h2 class="site-welcome stick-top">Nothing found. :( Try again?</h2> diff --git a/app/views/packages/show.json.jbuilder b/app/views/packages/show.json.jbuilder index 3b8a012..703ed8b 100644 --- a/app/views/packages/show.json.jbuilder +++ b/app/views/packages/show.json.jbuilder @@ -21,20 +21,20 @@ end json.use do json.local @package.versions.first.useflags[:local] do |flag| - json.name flag[1][:name] - json.description strip_tags flag[1][:description] + json.name flag[:name] + json.description strip_tags flag[:description] end json.global @package.versions.first.useflags[:global] do |flag| - json.name flag[1][:name] - json.description strip_tags flag[1][:description] + json.name flag[:name] + json.description strip_tags flag[:description] end - json.use_expand @package.versions.first.useflags[:use_expand] do |flag| + json.use_expand @package.versions.first.useflags[:use_expand].group_by { |u| u['use_expand_prefix'] } do |flag| json.set! flag[0] do json.array! flag[1] do |expand_flag| - json.name expand_flag[0] - json.description strip_tags expand_flag[1][:description] + json.name expand_flag[:name].gsub(expand_flag[:use_expand_prefix] + '_', '') + json.description strip_tags expand_flag[:description] end end end diff --git a/app/views/packages/stable.html.erb b/app/views/packages/stable.html.erb index 7b230fe..d9654de 100644 --- a/app/views/packages/stable.html.erb +++ b/app/views/packages/stable.html.erb @@ -12,7 +12,7 @@ <% cache("stable-full-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> diff --git a/app/views/packages/updated.html.erb b/app/views/packages/updated.html.erb index b774c58..af54ce1 100644 --- a/app/views/packages/updated.html.erb +++ b/app/views/packages/updated.html.erb @@ -12,7 +12,7 @@ <% cache("updated-full-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> diff --git a/app/views/useflags/_useflag_result_row.html.erb b/app/views/useflags/_useflag_result_row.html.erb index 084669f..3bdcd30 100644 --- a/app/views/useflags/_useflag_result_row.html.erb +++ b/app/views/useflags/_useflag_result_row.html.erb @@ -1,4 +1,4 @@ -<a class="list-group-item" href="<%= slf useflag_path useflag[:name] %>"> - <h3 class="kk-search-result-header"><%= useflag[:name] %></h3> - <%= useflag[:description] %> +<a class="list-group-item" href="<%= slf useflag_path useflag.name %>"> + <h3 class="kk-search-result-header"><%= useflag.name %></h3> + <%= useflag.description %> </a> diff --git a/bin/first-run b/bin/first-run index dd06ce2..63130ed 100755 --- a/bin/first-run +++ b/bin/first-run @@ -1,8 +1,18 @@ #!/bin/bash -bundle install --deployment -bundle exec rake tmp:create -bundle exec rake assets:precompile -bundle exec rake kkuleomi:index:init -bundle exec rake kkuleomi:seed:all -./update-all.sh +# Wait for Elasticsearch to start up +sleep 30 + +bundler install +bundle exec rake tmp:create RAILS_ENV=${1:-development} +bundle exec rake assets:precompile RAILS_ENV=${1:-development} +bundle exec rake kkuleomi:index:init RAILS_ENV=${1:-development} +./bin/update-all.sh ${1:-development} + +if [[ "${1:-development}" == "production" ]] +then + crontab -l | { cat; echo "*/10 * * * * /var/www/packages.gentoo.org/htdocs/bin/update-all.sh ${1:-development}"; } | crontab - +fi + +# Finally start the http server when the index is initialized +bundle exec thin start -p 5000
\ No newline at end of file diff --git a/bin/test.sh b/bin/test.sh new file mode 100755 index 0000000..5397212 --- /dev/null +++ b/bin/test.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Wait for Elasticsearch to start up +sleep 30 + +bundler install +bundle exec rake tmp:create RAILS_ENV=test +bundle exec rake assets:precompile RAILS_ENV=test +bundle exec rake kkuleomi:index:init RAILS_ENV=test + +RAILS_ENV=test bundle exec rake test test/
\ No newline at end of file diff --git a/bin/update-all.sh b/bin/update-all.sh index efa9955..c05277d 100755 --- a/bin/update-all.sh +++ b/bin/update-all.sh @@ -1,3 +1,23 @@ +#!/bin/bash + +# This script runs as the gpackages user normally! + +# Outside of a docker environment, it cannot call emerge --sync because that +# requires the 'portage' group, and opens up attacks to escalate from gpackages +# to portage-owned files. However, in a Docker environment, the other files +# from Portage are NOT available unless --sync IS used. + +function in_docker() { + path=/proc/1/cgroups + [[ -e ${path} ]] && grep -qa docker "${path}" +} + +# Stuff that we have to do inside Docker: +if in_docker && [[ ${1} != "production" ]]; then + emerge --sync +fi + +# This is the copy of the tree used to run gpackages against. if [[ ! -d /mnt/packages-tree/gentoo/ ]]; then cd /mnt/packages-tree || exit 1 git clone https://anongit.gentoo.org/git/repo/gentoo.git @@ -10,4 +30,4 @@ fi /var/www/packages.gentoo.org/htdocs/bin/update-use.sh cd /var/www/packages.gentoo.org/htdocs || exit 1 -bundle exec rake kkuleomi:update:all RAILS_ENV=production &>/dev/null +bundle exec rake kkuleomi:update:all RAILS_ENV=${1:-development} &>/dev/null diff --git a/bin/update-changelogs.sh b/bin/update-changelogs.sh index ae64050..0b35989 100755 --- a/bin/update-changelogs.sh +++ b/bin/update-changelogs.sh @@ -1,5 +1,7 @@ #!/bin/bash +mkdir -p /var/cache/pgo-egencache + cd /mnt/packages-tree/gentoo/ || exit 1 egencache -j 6 --cache-dir /var/cache/pgo-egencache --repo gentoo --repositories-configuration '[gentoo] location = /mnt/packages-tree/gentoo' --update-changelogs diff --git a/bin/update-md5.sh b/bin/update-md5.sh index 05eca90..10c7d75 100755 --- a/bin/update-md5.sh +++ b/bin/update-md5.sh @@ -1,5 +1,7 @@ #!/bin/bash +mkdir -p /var/cache/pgo-egencache + cd /mnt/packages-tree/gentoo/ || exit 1 egencache -j 6 --cache-dir /var/cache/pgo-egencache --repo gentoo --repositories-configuration '[gentoo] location = /mnt/packages-tree/gentoo' --update diff --git a/bin/update-use.sh b/bin/update-use.sh index 7f32af2..33bacfb 100755 --- a/bin/update-use.sh +++ b/bin/update-use.sh @@ -1,5 +1,7 @@ #!/bin/bash +mkdir -p /var/cache/pgo-egencache + cd /mnt/packages-tree/gentoo/ || exit 1 egencache -j 6 --cache-dir /var/cache/pgo-egencache --repo gentoo --repositories-configuration '[gentoo] location = /mnt/packages-tree/gentoo' --update-use-local-desc diff --git a/config/application.rb b/config/application.rb index e5cb95e..661c4ee 100644 --- a/config/application.rb +++ b/config/application.rb @@ -20,7 +20,7 @@ Bundler.require(*Rails.groups) module Packages class Application < Rails::Application # Initialize configuration defaults for originally generated Rails version. - config.load_defaults 5.1 + config.load_defaults "6.0" # Settings in config/environments/* take precedence over those specified here. # Application configuration should go into files in config/initializers diff --git a/config/initializers/elasticsearch.rb b/config/initializers/elasticsearch.rb index 4ced5b5..1037b1f 100644 --- a/config/initializers/elasticsearch.rb +++ b/config/initializers/elasticsearch.rb @@ -1,9 +1,10 @@ -require 'elasticsearch/persistence/model' +require 'elasticsearch/persistence' + +DEFAULT_CLIENT = Elasticsearch::Client.new host: ENV['ELASTICSEARCH_URL'] || 'localhost:9200' -Elasticsearch::Persistence.client = Elasticsearch::Client.new host: ENV['ELASTICSEARCH_URL'] || 'localhost:9200' if Rails.env.development? or ENV['RAILS_DEBUG'] logger = ActiveSupport::Logger.new(STDERR) logger.level = Logger::DEBUG logger.formatter = proc { |s, d, p, m| "\e[2m#{m}\n\e[0m" } - Elasticsearch::Persistence.client.transport.logger = logger + DEFAULT_CLIENT.transport.logger = logger end diff --git a/config/initializers/kkuleomi_config.rb.dist b/config/initializers/kkuleomi_config.rb.dist index dc0e79d..7f7fe3b 100644 --- a/config/initializers/kkuleomi_config.rb.dist +++ b/config/initializers/kkuleomi_config.rb.dist @@ -2,7 +2,7 @@ KKULEOMI_PORTDIR='/var/db/repos/gentoo' # The location of the repository used for gathering runtime information -KKULEOMI_RUNTIME_PORTDIR='/var/db/repos/gentoo' +KKULEOMI_RUNTIME_PORTDIR='/mnt/packages-tree/gentoo/' # The first actual git commit # Set this to the second commit in the repo to avoid long changelog generation times diff --git a/config/locales/en.yml b/config/locales/en.yml index 9358da7..dcc2db9 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -13,6 +13,7 @@ en: architectures: "Architectures" about: "About" help: "Help" + queries: "Advanced Search Queries" feedback: "Feedback" find_packages: "Find Packages" atom_feed: "Atom feed" @@ -59,6 +60,7 @@ en: view_git_changelog: "View Git Changelog" res_bugs: "Related bugs" res_docs: "Documentation" + res_search_docs: "Search for Documentation" res_forums: "Forums posts" res_repo: "Git repository browser" res_log: "Git log" @@ -96,6 +98,7 @@ en: feed_added_arch: "Gentoo Packages: Added packages on %{arch}" feed_added_title: "%{atom} (%{description})" feed_added_content: "%{atom} is now available in Gentoo on these architectures: %{arches}" + feed_search_results: "Gentoo Packages for search query: %{query}" feed_updated: "Gentoo Packages: Updated packages" feed_updated_arch: "Gentoo Packages: Updated packages on %{arch}" feed_updated_title: "%{atom} (%{description})" @@ -108,5 +111,6 @@ en: feed_keyworded_arch: "Gentoo Packages: Newly keyworded packages on %{arch}" feed_keyworded_title: "%{atom} (%{description})" feed_keyworded_content: "%{atom} is now keyworded on these architectures: %{arches}" + feed_commit_content: "#%{hash}: %{message}" # <meta> descriptions desc_categories_show: "Gentoo package category %{category}: %{description}" diff --git a/config/routes.rb b/config/routes.rb index 570ce7c..84b0c9f 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -7,6 +7,7 @@ Rails.application.routes.draw do get 'about/feeds' get 'about/help' get 'about/changelog' + get 'about/queries' root 'index#index' diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 1f2710b..554bfd8 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -26,6 +26,12 @@ services: - type: "bind" source: "." target: "/var/www/packages.gentoo.org/htdocs/" + - type: volume + source: portage + target: /var/db/repos/gentoo + - type: volume + source: portage-git + target: /mnt/packages-tree environment: # "Redis:port" and "elasticsearch:port" refer to sibling containers. - REDIS_PROVIDER=REDIS_URL @@ -38,16 +44,19 @@ services: depends_on: - redis - elasticsearch - command: > - bash -c " bundler install - && bundle exec rake assets:precompile - && bundle exec thin start -p 5000" + command: bash -c "/var/www/packages.gentoo.org/htdocs/bin/first-run development" sidekiq: build: . volumes: - type: "bind" source: "." target: "/var/www/packages.gentoo.org/htdocs/" + - type: volume + source: portage + target: /var/db/repos/gentoo + - type: volume + source: portage-git + target: /mnt/packages-tree environment: - RAILS_ENV=development - RAILS_SERVE_STATIC_FILES=1 @@ -66,8 +75,7 @@ services: ports: - 11211 elasticsearch: - # TODO(antarus): We should build a docker image for this based on gentoo. - image: docker.elastic.co/elasticsearch/elasticsearch:6.0.1 + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.1 container_name: elasticsearch environment: - discovery.type=single-node @@ -82,9 +90,12 @@ services: - 9200 # elasticsearch browser dejavu: - image: appbaseio/dejavu:3.2.3 + image: appbaseio/dejavu:3.4.0 container_name: dejavu ports: - '1358:1358' links: - - elasticsearch
\ No newline at end of file + - elasticsearch +volumes: + portage: + portage-git: diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..29c51c0 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,98 @@ + +version: '3.2' +# This file is used for testing purposes. Call +# +# $ docker-compose -f docker-compose.test.yml up --exit-code-from http-serving +# +# to run all tests. +# +# +services: + http-serving: + # Build from Dockerfile in . + build: . + ports: + - 5000 + volumes: + - type: "bind" + source: "." + target: "/var/www/packages.gentoo.org/htdocs/" + - type: volume + source: portage + target: /var/db/repos/gentoo + - type: volume + source: portage-git + target: /mnt/packages-tree + environment: + # "Redis:port" and "elasticsearch:port" refer to sibling containers. + - REDIS_PROVIDER=REDIS_URL + - REDIS_URL=redis://redis:6379 + - ELASTICSEARCH_URL=elasticsearch:9200 + - RAILS_SERVE_STATIC_FILES=1 + - RAILS_ENV=development + - MEMCACHE_URL="memcache:11211" + - SECRET_KEY_BASE=6c9710aeb74dd88ff1d1b8f4bd6d7d8e0f340905d0974400fffd7246714aa703cf7bf4a98c0bc90317a3b803b82c0f9371e18ada19fc4eed9d6118077a249f50 + depends_on: + - redis + - elasticsearch + command: bash -c "/var/www/packages.gentoo.org/htdocs/bin/test.sh" + sidekiq: + build: . + volumes: + - type: "bind" + source: "." + target: "/var/www/packages.gentoo.org/htdocs/" + - type: volume + source: portage + target: /var/db/repos/gentoo + - type: volume + source: portage-git + target: /mnt/packages-tree + environment: + - RAILS_ENV=development + - RAILS_SERVE_STATIC_FILES=1 + - REDIS_URL=redis://redis:6379 + - MEMCACHE_URL="memcache:11211" + - ELASTICSEARCH_URL=elasticsearch:9200 + - SECRET_KEY_BASE=6c9710aeb74dd88ff1d1b8f4bd6d7d8e0f340905d0974400fffd7246714aa703cf7bf4a98c0bc90317a3b803b82c0f9371e18ada19fc4eed9d6118077a249f50 + depends_on: + - redis + - elasticsearch + command: > + bash -c " bundler install + && bundle exec sidekiq -c 5 -e test" + memcache: + image: memcached:latest + ports: + - 11211 + elasticsearch: + # TODO(antarus): We should build a docker image for this based on gentoo. + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.1 + container_name: elasticsearch + environment: + - discovery.type=single-node + - http.port=9200 + - http.cors.enabled=true + - http.cors.allow-origin=http://localhost:1358,http://127.0.0.1:1358 + - http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization + - http.cors.allow-credentials=true + - bootstrap.memory_lock=true + - 'ES_JAVA_OPTS=-Xms512m -Xmx512m' + ports: + - 9200 + # elasticsearch browser + dejavu: + image: appbaseio/dejavu:3.4.0 + container_name: dejavu + ports: + - '1358:1358' + links: + - elasticsearch + redis: + image: redis:4.0.6 + ports: + - 6379 + +volumes: + portage: + portage-git: diff --git a/docker-compose.yml b/docker-compose.yml index d3d5d58..59eb9d2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,13 @@ services: build: . ports: - 5000 + volumes: + - type: volume + source: portage + target: /var/db/repos/gentoo + - type: volume + source: portage-git + target: /mnt/packages-tree environment: # "Redis:port" and "elasticsearch:port" refer to sibling containers. - REDIS_PROVIDER=REDIS_URL @@ -33,9 +40,16 @@ services: depends_on: - redis - elasticsearch - command: bundle exec thin start -p 5000 + command: bash -c "/var/www/packages.gentoo.org/htdocs/bin/first-run production" sidekiq: build: . + volumes: + - type: volume + source: portage + target: /var/db/repos/gentoo + - type: volume + source: portage-git + target: /mnt/packages-tree environment: - RAILS_ENV=production - RAILS_SERVE_STATIC_FILES=1 @@ -53,7 +67,7 @@ services: - 11211 elasticsearch: # TODO(antarus): We should build a docker image for this based on gentoo. - image: docker.elastic.co/elasticsearch/elasticsearch:6.0.1 + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.1 # Run in single-node config. environment: - discovery.type=single-node @@ -63,3 +77,7 @@ services: image: redis:4.0.6 ports: - 6379 + +volumes: + portage: + portage-git: diff --git a/lib/core_ext/markdown_handler.rb b/lib/core_ext/markdown_handler.rb index f5ffa7b..a08ed18 100644 --- a/lib/core_ext/markdown_handler.rb +++ b/lib/core_ext/markdown_handler.rb @@ -5,8 +5,7 @@ module MarkdownHandler @erb ||= ActionView::Template.registered_template_handler(:erb) end - def self.call(template) - compiled_source = erb.call(template) + def self.call(template, source) "RDiscount.new(begin;#{compiled_source};end).to_html" end end diff --git a/lib/kkuleomi/store.rb b/lib/kkuleomi/store.rb index a1a2d93..ec27d7a 100644 --- a/lib/kkuleomi/store.rb +++ b/lib/kkuleomi/store.rb @@ -1,15 +1,13 @@ module Kkuleomi::Store - def self.refresh_index - Category.gateway.refresh_index! - end def self.create_index(force = false) - types = [ - Category, - Package, - Version, - Change, - Useflag, + repositories = [ + CategoryRepository, + PackageRepository, + VersionRepository, + ChangeRepository, + UseflagRepository, + CommitRepository ] base_settings = { @@ -33,15 +31,11 @@ module Kkuleomi::Store mapping: { total_fields: { limit: 50000 } } } + settings = JSON.parse('{ "mapping": { "total_fields": { "limit": 50000 } } }') + # In ES 1.5, we could use 1 mega-index. But in ES6, each model needs its own. - types.each { |type| - client = type.gateway.client - client.indices.delete(index: type.index_name) rescue nil if force - body = { - settings: type.settings.to_hash.merge(base_settings), - mappings: type.mappings.to_hash - } - client.indices.create(index: type.index_name, body: body) + repositories.each { |repository| + repository.instance.create_index!(force: true, settings: settings) } end end diff --git a/lib/kkuleomi/store/model.rb b/lib/kkuleomi/store/model.rb deleted file mode 100644 index 653884b..0000000 --- a/lib/kkuleomi/store/model.rb +++ /dev/null @@ -1,78 +0,0 @@ -module Kkuleomi::Store::Model - def self.included(base) - base.send :include, InstanceMethods - base.extend ClassMethods - end - - module ClassMethods - # Finds instances by exact IDs using the 'term' filter - def find_all_by(field, value, opts = {}) - search({ - size: 10_000, - query: { bool: { filter: { term: { field => value } } } } - }.merge(opts)) - end - - # Filter all instances by the given parameters - def filter_all(filters, opts = {}) - filter_args = [] - filters.each_pair { |field, value| filter_args << { term: { field => value } } } - - search({ - query: { - bool: { filter: { bool: { must: filter_args } } } - }, - size: 10_000 - }.merge(opts)) - end - - def find_by(field, value, opts = {}) - find_all_by(field, value, opts).first - end - - def find_all_by_parent(parent, opts = {}) - search(opts.merge( - size: 10_000, - query: { - bool: { - filter: { - has_parent: { - parent_type: parent.class.document_type, - query: { term: { _id: parent.id } } - } - }, - must: { match_all: {} } - } - } - )) - end - - # Returns all (by default 10k) records of this class sorted by a field. - def all_sorted_by(field, order, options = {}) - all({ - query: { match_all: {} }, - sort: { field => { order: order } } - }, options) - end - end - - module InstanceMethods - # Converts the model to an OpenStruct instance - # - # @param [Array<Symbol>] fields Fields to export into the OpenStruct, or all fields if nil - # @return [OpenStruct] OpenStruct containing the selected fields - def to_os(*fields) - fields = all_fields if fields.empty? - OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }]) - end - - # Converts the model to a Hash - # - # @param [Array<Symbol>] fields Fields to export into the Hash, or all fields if nil - # @return [Hash] Hash containing the selected fields - def to_hsh(*fields) - fields = all_fields if fields.empty? - Hash[fields.map { |field| [field, send(field)] }] - end - end -end diff --git a/lib/kkuleomi/store/models/package_import.rb b/lib/kkuleomi/store/models/package_import.rb index 99ab433..8ae1e5d 100644 --- a/lib/kkuleomi/store/models/package_import.rb +++ b/lib/kkuleomi/store/models/package_import.rb @@ -30,15 +30,15 @@ module Kkuleomi::Store::Models::PackageImport set_basic_metadata(package_model, latest_ebuild) # Be sure to have an ID now - save + PackageRepository.save(self) import_useflags!(package_model) - Kkuleomi::Store.refresh_index + CategoryRepository.refresh_index! import_versions!(package_model, ebuilds, options) # Do this last, so that any exceptions before this point skip this step self.metadata_hash = package_model.metadata_hash - save + PackageRepository.save(self) if options[:package_state] == 'new' && !options[:suppress_change_objects] RecordChangeJob.perform_later( @@ -73,7 +73,7 @@ module Kkuleomi::Store::Models::PackageImport end def import_useflags!(package_model) - index_flags = Useflag.local_for(package_model.to_cp) + index_flags = UseflagRepository.local_for(package_model.to_cp) model_flags = package_model.metadata[:use] new_flags = model_flags.keys - index_flags.keys @@ -87,23 +87,23 @@ module Kkuleomi::Store::Models::PackageImport flag_doc.description = model_flags[flag] flag_doc.atom = package_model.to_cp flag_doc.scope = 'local' - flag_doc.save + UseflagRepository.save(flag_doc) end eql_flags.each do |flag| unless index_flags[flag].description == model_flags[flag] index_flags[flag].description = model_flags[flag] - index_flags[flag].save + UseflagRepository.save(index_flags[flag]) end end del_flags.each do |flag| - index_flags[flag].delete + UseflagRepository.delete(index_flags[flag]) end end def import_versions!(package_model, ebuilds, options) - index_v = Hash[Version.find_all_by(:package, package_model.to_cp).map { |v| [v.version, v] }] + index_v = Hash[VersionRepository.find_all_by(:package, package_model.to_cp).map { |v| [v.version, v] }] model_v = Hash[ebuilds.map { |v| [v.version, v] }] index_keys = index_v.keys @@ -128,7 +128,7 @@ module Kkuleomi::Store::Models::PackageImport if sort_key == 0 self.useflags = version_doc.useflags - save + VersionRepository.save(version_doc) end end @@ -144,12 +144,12 @@ module Kkuleomi::Store::Models::PackageImport if sort_key == 0 self.useflags = version_doc.useflags - save + VersionRepository.save(version_doc) end end del_v.each do |v| - index_v[v].delete + VersionRepository.delete(index_v[v]) end end end diff --git a/lib/kkuleomi/store/models/package_search.rb b/lib/kkuleomi/store/models/package_search.rb deleted file mode 100644 index ec0268c..0000000 --- a/lib/kkuleomi/store/models/package_search.rb +++ /dev/null @@ -1,161 +0,0 @@ -# Contains the search logic for packages -module Kkuleomi::Store::Models::PackageSearch - def self.included(base) - base.send :include, InstanceMethods - base.extend ClassMethods - end - - module ClassMethods - def suggest(q) - Package.search( - size: 20, - query: { - wildcard: { - name_sort: { - wildcard: q.downcase + '*' - } - } - } - ) - end - - # Tries to resolve a query atom to one or more packages - def resolve(atom) - [] if atom.nil? || atom.empty? - - Package.find_all_by(:atom, atom) + Package.find_all_by(:name, atom) - end - - # Searches the versions index for versions using a certain USE flag. - # Results are aggregated by package atoms. - def find_atoms_by_useflag(useflag) - Version.search( - size: 0, # collect all packages. - query: { - bool: { - must: { match_all: {} }, - filter: { term: { use: useflag } } - } - }, - aggs: { - group_by_package: { - terms: { - field: 'package', - order: { '_key' => 'asc' }, - # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html - # ES actually dislikes large sizes like this (it defines 10k buckets basically) and it will be *very* expensive but lets try it and see. - # Other limits in this app are also 10k mostly to 'make things fit kinda'. - size: 10000, - } - } - }, - ).response.aggregations['group_by_package'].buckets - end - - def default_search_size - 25 - end - - def default_search(q, offset) - return [] if q.nil? || q.empty? - - part1, part2 = q.split('/', 2) - - if part2.nil? - search(build_query(part1, nil, default_search_size, offset)) - else - search(build_query(part2, part1, default_search_size, offset)) - end - end - - def build_query(q, category, size, offset) - { - size: size, - from: offset, - query: { - function_score: { - query: { bool: bool_query_parts(q, category) }, - functions: scoring_functions - } - } - } - end - - def bool_query_parts(q, category = nil) - q_dwncsd = q.downcase - - query = { - must: [ - match_wildcard(q_dwncsd) - ], - should: [ - match_phrase(q_dwncsd), - match_description(q) - ] - } - - query[:must] << [match_category(category)] if category - - query - end - - def match_wildcard(q) - q = ('*' + q + '*') unless q.include? '*' - q.tr!(' ', '*') - - { - wildcard: { - name_sort: { - wildcard: q, - boost: 4 - } - } - } - end - - def match_phrase(q) - { - match_phrase: { - name: { - query: q, - boost: 5 - } - } - } - end - - def match_description(q) - { - match: { - description: { - query: q, - boost: 0.1 - } - } - } - end - - def match_category(cat) - { - match: { - category: { - query: cat, - boost: 2 - } - } - } - end - - def scoring_functions - [ - { - filter: { term: { category: 'virtual' } }, - weight: 0.6 - } - ] - end - end - - module InstanceMethods - end -end diff --git a/lib/kkuleomi/store/models/version_import.rb b/lib/kkuleomi/store/models/version_import.rb index b65b683..6ee6b64 100644 --- a/lib/kkuleomi/store/models/version_import.rb +++ b/lib/kkuleomi/store/models/version_import.rb @@ -38,7 +38,7 @@ module Kkuleomi::Store::Models::VersionImport self.masks = Portage::Util::Masks.for(ebuild_model) self.metadata_hash = ebuild_model.metadata_hash - save() + VersionRepository.save(self) # If keywords changed, calculate changes and record as needed (but only do that if we should) unless options[:suppress_change_objects] @@ -60,7 +60,7 @@ module Kkuleomi::Store::Models::VersionImport # @param [Package] parent Parent package model def set_sort_key!(key, parent) self.sort_key = key - save() + VersionRepository.save(self) end def strip_useflag_defaults(flags) diff --git a/lib/portage/util/history.rb b/lib/portage/util/history.rb index b2348b3..dfa7449 100644 --- a/lib/portage/util/history.rb +++ b/lib/portage/util/history.rb @@ -2,17 +2,23 @@ require 'time' class Portage::Util::History class << self - def for(category, package, limit = 20) + def update() return [] if KKULEOMI_DISABLE_GIT == true - files = "#{category}/#{package}/" + latest_commit_id = KKULEOMI_FIRST_COMMIT + latest_commit = CommitRepository.n_sorted_by(1, "date", "desc").first + + unless latest_commit.nil? + latest_commit_id = latest_commit.id + end + git = Kkuleomi::Util::Exec - .cmd(KKULEOMI_GIT) - .in(KKULEOMI_RUNTIME_PORTDIR) - .args( - 'log', '--name-status', '--no-merges', '--date=iso8601', "-n #{limit.to_i}", - "#{KKULEOMI_FIRST_COMMIT}..HEAD", files) - .run + .cmd(KKULEOMI_GIT) + .in(KKULEOMI_RUNTIME_PORTDIR) + .args( + 'log', '--name-status', '--no-merges', '--date=iso8601', "--reverse", + "#{latest_commit_id}..HEAD") + .run raw_log, stderr, status = git.stdout, git.stderr, git.exit_status fail "Cannot get git log: #{stderr}" unless status == 0 @@ -23,9 +29,11 @@ class Portage::Util::History private def parse(raw_log) - log_items = [] - raw_log.split("\n\ncommit ").each do |raw_commit| + count = raw_log.split("\n\ncommit ").slice(0, 10000).size + + raw_log.split("\n\ncommit ").slice(0, 10000).each do |raw_commit| + commit_lines = raw_commit.lines _id = commit_lines.shift.gsub('commit ', '').strip @@ -38,37 +46,46 @@ class Portage::Util::History commit_lines.shift _raw_message = [] - while (line = commit_lines.shift) != "\n" + while (line = commit_lines.shift) != "\n" && !line.nil? _raw_message << line end _raw_files = commit_lines _files = {added: [], modified: [], deleted: []} + _packages = [] _raw_files.each do |file| mode, file = file.split "\t" - filename = file.strip.split('/').last + + if file.strip.split('/').size >= 3 + _packages << (file.strip.split('/')[0] + '/' + file.strip.split('/')[1]) + end case mode when 'M' - _files[:modified] << filename + _files[:modified] << file.strip when 'D' - _files[:deleted] << filename + _files[:deleted] << file.strip when 'A' - _files[:added] << filename + _files[:added] << file.strip end end - log_items << { - id: _id, - author: _author, - email: _email, - date: _date, - message: _raw_message.map { |l| l.strip }.join("\n"), - files: _files - } + + commit = Commit.new + commit.id = _id + commit.author = _author + commit.email = _email + commit.date = _date + commit.message = _raw_message.map { |l| l.strip }.join("\n") + commit.files = _files + commit.packages = _packages.to_set + CommitRepository.save(commit) + end + + if count >= 10000 + CommitsUpdateJob.perform_later end - log_items end end end diff --git a/lib/tasks/kkuleomi.rake b/lib/tasks/kkuleomi.rake index 9b8bca0..9362b7a 100644 --- a/lib/tasks/kkuleomi.rake +++ b/lib/tasks/kkuleomi.rake @@ -50,4 +50,5 @@ end def initialize_caches MasksUpdateJob.perform_later UseflagsUpdateJob.perform_later + CommitsUpdateJob.perform_later end diff --git a/test/integration/about_routes_test.rb b/test/integration/about_routes_test.rb new file mode 100644 index 0000000..56c36a3 --- /dev/null +++ b/test/integration/about_routes_test.rb @@ -0,0 +1,30 @@ +require 'test_helper' + +class AboutRoutesTest < ActionDispatch::IntegrationTest + + test "can see the about page" do + get "/about" + assert_select "h1", "About packages.gentoo.org" + end + + test "can see the feedback page" do + get "/about/feedback" + assert_select "h1", "Feedback" + end + + test "can see the about feeds page" do + get "/about/feeds" + assert_select "h1", "Update Feeds" + end + + test "can see the about help page" do + get "/about/help" + assert_select "h1", "Help" + end + + test "can see the changelog page" do + get "/about/changelog" + assert_select "h1", "Changelog" + end + +end diff --git a/test/integration/arches_routes_test.rb b/test/integration/arches_routes_test.rb new file mode 100644 index 0000000..2e673e2 --- /dev/null +++ b/test/integration/arches_routes_test.rb @@ -0,0 +1,25 @@ +require 'test_helper' + +class ArchesRoutesTest < ActionDispatch::IntegrationTest + test "can see the arches page" do + get "/arches" + assert_select "h1", "Architectures" + end + + test "view keyworded packages for arch" do + arches = %w(alpha amd64 arm arm64 hppa ia64 ppc ppc64 sparc x86) + arches.each { |arch| + get ("/arches/" + arch + "/keyworded") + assert_select "h1", ("Keyworded Packages (" + arch + ")") + } + end + + test "view stable packages for arch" do + arches = %w(alpha amd64 arm arm64 hppa ia64 ppc ppc64 sparc x86) + arches.each { |arch| + get ("/arches/" + arch + "/stable") + assert_select "h1", ("Newly Stable Packages (" + arch + ")") + } + end + +end diff --git a/test/integration/categories_routes_test.rb b/test/integration/categories_routes_test.rb new file mode 100644 index 0000000..a9f46c2 --- /dev/null +++ b/test/integration/categories_routes_test.rb @@ -0,0 +1,10 @@ +require 'test_helper' + +class CategoriesRoutesTest < ActionDispatch::IntegrationTest + + test "can see the categories page" do + get "/categories" + assert_select "h1", "Packages" + end + +end diff --git a/test/integration/feeds_test.rb b/test/integration/feeds_test.rb new file mode 100644 index 0000000..12a9c2f --- /dev/null +++ b/test/integration/feeds_test.rb @@ -0,0 +1,29 @@ +require 'test_helper' + +class FeedsTest < ActionDispatch::IntegrationTest + + test "can see the added packages feed" do + get '/packages/added.atom' + assert_response :success + assert_equal 'application/atom+xml; charset=utf-8', @response.content_type + end + + test "can see the updates packages feed" do + get '/packages/updated.atom' + assert_response :success + assert_equal 'application/atom+xml; charset=utf-8', @response.content_type + end + + test "can see the newly stable packages feed" do + get '/packages/stable.atom' + assert_response :success + assert_equal 'application/atom+xml; charset=utf-8', @response.content_type + end + + test "can see the keyworded packages feed" do + get '/packages/keyworded.atom' + assert_response :success + assert_equal 'application/atom+xml; charset=utf-8', @response.content_type + end + +end diff --git a/test/integration/main_routes_test.rb b/test/integration/main_routes_test.rb new file mode 100644 index 0000000..71803aa --- /dev/null +++ b/test/integration/main_routes_test.rb @@ -0,0 +1,15 @@ +require 'test_helper' + +class MainRoutesTest < ActionDispatch::IntegrationTest + test "view landing page" do + get "/" + assert_select "h2", "Welcome to the Home of 1 Gentoo Packages" + end + + test "test route not present" do + assert_raises(ActionController::RoutingError) do + get '/larry' + end + end + +end diff --git a/test/integration/packages_routes_test.rb b/test/integration/packages_routes_test.rb new file mode 100644 index 0000000..465af4b --- /dev/null +++ b/test/integration/packages_routes_test.rb @@ -0,0 +1,51 @@ +require 'test_helper' + +class PackagesRoutesTest < ActionDispatch::IntegrationTest + + test "packages landing page" do + get "/packages" + assert_response :redirect + follow_redirect! + assert_response :success + assert_select "h1", "Packages" + end + + test "view existing package" do + get "/packages/virtual/packages" + assert_select ".kk-package-name", "packages" + end + + test "search for non existing package" do + get "/packages/search?q=larry" + assert_select "h2", "Nothing found. :( Try again?" + end + + test "search for existing package" do + get "/packages/search?q=packages" + assert_response :redirect + follow_redirect! + assert_response :success + assert_select ".kk-package-name", "packages" + end + + test "added package page" do + get "/packages/added" + assert_select "h1", "Added Packages" + end + + test "updated package page" do + get "/packages/updated" + assert_select "h1", "Updated Packages" + end + + test "newly stable packages page" do + get "/packages/stable" + assert_select "h1", "Newly Stable Packages" + end + + test "keyworded packages page" do + get "/packages/keyworded" + assert_select "h1", "Keyworded Packages" + end + +end
\ No newline at end of file diff --git a/test/integration/useflag_routes_test.rb b/test/integration/useflag_routes_test.rb new file mode 100644 index 0000000..3ed5718 --- /dev/null +++ b/test/integration/useflag_routes_test.rb @@ -0,0 +1,25 @@ +require 'test_helper' + +class UseflagRoutesTest < ActionDispatch::IntegrationTest + + test "can see the useflags page" do + get "/useflags" + assert_select "h1", "USE flags" + end + + test "search for multiple existing useflag" do + get "/useflags/search?q=systemd" + assert_select "h1", "USE Flag Search Results for systemd" + end + + test "search for non existing useflag" do + get "/useflags/search?q=larry" + assert_select "h1", "USE Flag Search Results for larry" + end + + test "view existing useflag" do + get "/useflags/systemd" + assert_select "h1", "systemd" + end + +end diff --git a/test/test_helper.rb b/test/test_helper.rb index c71aa37..db135c7 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -6,6 +6,10 @@ class ActiveSupport::TestCase # Import some test data into the test indices category = Portage::Repository::Category.new('test/fixtures/repo/virtual') Category.new.import!(category) + + package = Portage::Repository::Package.new('test/fixtures/repo/virtual/packages') + Package.new.import!(package, { package_state: 'new' }) + UseflagsUpdateJob.new.perform # Add more helper methods to be used by all tests here... end |