shard packages AGAIN by distro (#435)

ros-infrastructure · Oct 16, 2024 · b7c380a · b7c380a
1 parent 5e0c913
commit b7c380a
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 21 deletions.
diff --git a/_plugins/rosindex_generator.rb b/_plugins/rosindex_generator.rb
@@ -1530,8 +1530,12 @@ def generate(site)
     unless site.config['skip_search_index']
       puts ("Generating packages search index...").blue
 
-      packages_index = []
+      packages_index = {}
+      $all_distros.each do |distro|
+        packages_index[distro] = []
+      end
 
+      index = 0
       @all_repos.each do |instance_id, repo|
         repo.snapshots.each do |distro, repo_snapshot|
 
@@ -1545,8 +1549,9 @@ def generate(site)
 
             readme_filtered = if p['readme'] then self.strip_stopwords(p['readme']) else "" end
 
-            packages_index << {
-              'id' => packages_index.length,
+            index += 1
+            packages_index[distro] << {
+              'id' => index,
               'baseurl' => site.config['baseurl'],
               'url' => File.join('/p',package_name,instance_id)+"#"+distro,
               'last_commit_time' => repo_snapshot.data['last_commit_time'],
@@ -1556,7 +1561,7 @@ def generate(site)
               'released' => if repo_snapshot.released then 'is:released' else '' end,
               'unreleased' => if repo_snapshot.released then 'is:unreleased' else '' end,
               'version' => p['version'],
-              'description' => p['description'],
+              'description' => p['description'].strip,
               'maintainers' => p['maintainers'] * " ",
               'authors' => p['authors'] * " ",
               'distro' => distro,
@@ -1569,10 +1574,6 @@ def generate(site)
         end
       end
 
-      sorted_packages_index = packages_index.sort do |a, b|
-        $all_distros.index(a['distro']) <=> $all_distros.index(b['distro'])
-      end
-
       puts ("Precompiling lunr index for packages...").blue
       reference_field = 'id'
       indexed_fields = [
@@ -1581,8 +1582,8 @@ def generate(site)
         'distro','readme', 'released', 'unreleased'
       ]
       site.static_files.push(*precompile_lunr_index(
-        site, sorted_packages_index, reference_field, indexed_fields,
-        "search/packages/", site.config['search_index_shards'] || 1
+        site, packages_index, reference_field, indexed_fields,
+        "search/packages/", $all_distros
       ).to_a)
 
       puts ("Generating system dependencies search index...").blue
@@ -1633,9 +1634,12 @@ def generate(site)
       puts ("Precompiling lunr index for system dependencies...").blue
       reference_field = 'id'
       indexed_fields = ['name', 'platforms', 'dependants']
+      slice_length = system_deps_index.length / site.config['search_index_shards'] || 1
+      slices = {}
+      system_deps_index.each_slice(slice_length).with_index.map { |item, i| slices[i.to_s] = item }
       site.static_files.push(*precompile_lunr_index(
-        site, system_deps_index, reference_field, indexed_fields,
-        "search/deps/", site.config['search_index_shards'] || 1
+        site, slices, reference_field, indexed_fields,
+        "search/deps/", slices.keys
       ).to_a)
     end
 

diff --git a/_ruby_libs/lunr.rb b/_ruby_libs/lunr.rb
@@ -3,7 +3,7 @@
 require 'json'
 require_relative './common'
 
-def precompile_lunr_index(site, index, ref, fields, output_dir, shard_count = 1)
+def precompile_lunr_index(site, index, ref, fields, output_dir, shard_names)
   build_index_cmd = File.join(
     site.source, 'node_modules',
     'lunr-index-build', 'bin',
@@ -15,17 +15,16 @@ def precompile_lunr_index(site, index, ref, fields, output_dir, shard_count = 1)
   output_dirpath = File.join(site.dest, output_dir)
   FileUtils.mkdir_p(site.dest) unless File.directory?(site.dest)
   FileUtils.mkdir_p(output_dirpath) unless File.directory?(output_dirpath)
-  shard_size = index.length / shard_count
-  if shard_size == 0 then shard_size = 1 end
 
   Enumerator.new do |enum| 
-    shards = index.each_slice(shard_size).with_index.collect do |index_slice, i|
-      dputs("Building lunr index shard #{i}...")
-      index_filename = "index.#{i}.json"
-      data_filename = "data.#{i}.json"
+    shards = []
+    shard_names.each do |name|
+      dputs("Building lunr index shard #{name}...")
+      index_filename = "index.#{name}.json"
+      data_filename = "data.#{name}.json"
       data_filepath = File.join(output_dirpath, data_filename)
       File.open(data_filepath, 'w') do |data_file|
-        data_file.write(JSON.generate(index_slice))
+        data_file.write(JSON.generate(index[name]))
       end
       enum.yield SearchIndexFile.new(site, site.dest, output_dir, data_filename)
       dputs("Index data written to #{data_filename}.")
@@ -35,7 +34,7 @@ def precompile_lunr_index(site, index, ref, fields, output_dir, shard_count = 1)
       Process.waitpid(pid)
       enum.yield SearchIndexFile.new(site, site.dest, output_dir, index_filename)
       dputs("Index written to #{output_dir}/#{index_filename}.")
-      {:index => index_filename, :data => data_filename}
+      shards << {:index => index_filename, :data => data_filename}
     end
     shards_filename = "shards.json"
     shards_filepath = File.join(output_dirpath, shards_filename)