qa: fairly split integration tests using hints

ci/partition-files.lib.sh

@@ -2,8 +2,10 @@
 
 # partition_files returns a consistent partition of the filenames given on stdin
 # Usage: partition_files <partition_index> <partition_count=2> < <(ls files)
-# partition_index: the zero-based index of the partition to select `[0,partition_count)`
-# partition_count: the number of partitions `[2,#files]`
+#   partition_index: the zero-based index of the partition to select `[0,partition_count)`
+#   partition_count: the number of partitions `[2,#files]`
+# Environment:
+#   PARTITION_FILES_DEFAULT_COST <integer>: the cost if no SPLIT_ESTIMATE can be found
 partition_files() (
     set -e
 
@@ -25,21 +27,60 @@ partition_files() (
       _error "partition_index(${partition_index}) must be a number that is greater 0 and less than partition_count(${partition_count})"
     fi
 
-    # round-robbin emit those in our selected partition
-    for index in "${!files[@]}"; do
-      partition="$(( index % partition_count ))"
-      if (( partition == partition_index )); then
-        echo "${files[$index]}"
-      fi
+    # function for finding the split estimate for a file.
+    # seeks file for `SPLIT_ESTIMATE: INTEGER` hint, which may be in a comment.
+    # uses PARTITION_FILES_DEFAULT_COST (default 20) if no hint is found.
+    _find_split_estimate () {
+      local file="${1:?}"
+      local cost=$(awk 'match($0, /SPLIT_ESTIMATE: ?[[:digit:]]+/){print substr($0,RSTART+16,RLENGTH); exit}' "${file}")
+
+      echo "${cost:-${PARTITION_FILES_DEFAULT_COST:-20}}"
+    }
+
+    # index and sort by estimated cost.
+    # the result is an array of filename-tab-cost entries with default values and the most expensive entries first.
+    local indexed=()
+    for file in "${files[@]}"; do
+      indexed+=("${file}"$'\t'"$(_find_split_estimate "${file}")")
+    done
+    IFS=$'\n' indexed=($(printf "%s\n" "${indexed[@]}" | sort -k2rn -k1)); unset IFS
+
+    # assign our files to their partitions
+    local costs=()
+    local selected=()
+    for index in "${!indexed[@]}"; do
+      local file
+      local cost
+      IFS=$'\t' read -r file cost <<< "${indexed[$index]}"
+
+      # find the partition with the current lowest cost
+      # and allocate the current file to it
+      local partition=0
+      for i in $(seq 1 $(( partition_count - 1 ))); do
+        (( "${costs[$i]:-0}" < "${costs[${partition}]:-0}" )) && partition=$i
+      done
+      costs[$partition]=$(( costs[$partition] + $cost ))
+
+      # output if it matches the selected partition index
+      (( $partition == $partition_index )) && selected+=("${file}")
     done
+
+    echo "${selected[@]}"
+
+    # print a summary
+    local total_cost="$((${costs[@]/%/ +} 0))"
+    >&2 echo "[PARTITION id=${partition_index}/${partition_count}"\
+                        "files=${#selected[@]}/${#files[@]}"\
+                        "cost=${costs[$partition_index]}/${total_cost}]"
 )
 
 if [[ "$0" == "${BASH_SOURCE[0]}" ]]; then
   if [[ "$1" == "test" ]]; then
+    # for testing purposes SPLIT_ESTIMATE: 31
     status=0
 
     SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-    file_list="$( cd "${SCRIPT_DIR}"; find . -type f )"
+    file_list="$( find "${SCRIPT_DIR}/." -type f )"
 
     # for any legal partitioning into N partitions, we ensure that
     # the combined output of `partition_files I N` where `I` is all numbers in

qa/integration/rspec.rb

@@ -34,7 +34,46 @@
 RSpec.clear_examples
 
 RSpec.configure do |c|
-    c.filter_run_excluding skip_fips: true if java.lang.System.getProperty("org.bouncycastle.fips.approved_only") == "true"
+  timer = Class.new do
+    def initialize
+      @timings = Hash.new(0)
+      @mutex = Mutex.new
+    end
+
+    def record(example)
+      start_time = now_millis
+      example.run
+    ensure
+      duration_millis = (now_millis - start_time)
+      spec_path = Pathname.new(example.file_path).cleanpath.to_s
+      @mutex.synchronize { @timings[spec_path] += duration_millis }
+    end
+
+    def write
+      @mutex.synchronize do
+        @timings.sort.each do |filename, time_millis|
+          $stderr.puts("[TIME #{filename}](#{(time_millis/1000.0).ceil})")
+        end
+      end
+    end
+
+    private
+
+    ##
+    # Get the current time in millis directly from Java,
+    # bypassing any ruby time-mocking libs
+    # @return [Integer]
+    def now_millis
+      java.lang.System.currentTimeMillis()
+    end
+  end.new
+
+  c.around(:example) { |example| timer.record(example) }
+  c.after(:suite) { timer.write }
+end
+
+RSpec.configure do |c|
+  c.filter_run_excluding skip_fips: true if java.lang.System.getProperty("org.bouncycastle.fips.approved_only") == "true"
 end
 
 return RSpec::Core::Runner.run($JUNIT_ARGV).to_i

qa/integration/specs/01_logstash_bin_smoke_spec.rb

@@ -25,6 +25,7 @@
 require 'json'
 require 'open-uri'
 
+# SPLIT_ESTIMATE: 98
 describe "Test Logstash instance" do
   before(:all) {
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/beats_input_spec.rb

@@ -23,6 +23,7 @@
 require "yaml"
 require "fileutils"
 
+# SPLIT_ESTIMATE: 80
 describe "Beat Input" do
   before(:all) do
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/cli/install_spec.rb

@@ -40,6 +40,7 @@ def plugin_filename_re(name, version)
 
 INSTALLATION_ABORTED_RE = /Installation aborted/
 
+# SPLIT_ESTIMATE: 284
 describe "CLI > logstash-plugin install" do
   before(:each) do
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/cli/keystore_spec.rb

@@ -24,6 +24,7 @@
 require "fileutils"
 require "open3"
 
+# SPLIT_ESTIMATE: 10
 describe "CLI > logstash-keystore" do
   before(:all) do
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/cli/list_spec.rb

@@ -21,6 +21,7 @@
 require_relative '../../framework/helpers'
 require "logstash/devutils/rspec/spec_helper"
 
+# SPLIT_ESTIMATE: 24
 describe "CLI > logstash-plugin remove" do
   before(:all) do
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/cli/prepare_offline_pack_spec.rb

@@ -20,6 +20,7 @@
 require_relative "../../services/logstash_service"
 require_relative "../../framework/helpers"
 
+# SPLIT_ESTIMATE: 60
 describe "CLI > logstash-plugin prepare-offline-pack", :skip_fips do
   before(:all) do
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/cli/remove_spec.rb

@@ -22,6 +22,7 @@
 require_relative "pluginmanager_spec_helper"
 require "logstash/devutils/rspec/spec_helper"
 
+# SPLIT_ESTIMATE: 240
 describe "CLI > logstash-plugin remove", :skip_fips do
 
   include_context "pluginmanager validation helpers"

qa/integration/specs/cli/update_spec.rb

@@ -22,6 +22,7 @@
 require_relative "pluginmanager_spec_helper"
 require "logstash/devutils/rspec/spec_helper"
 
+# SPLIT_ESTIMATE: 40
 describe "CLI > logstash-plugin update", :skip_fips do
 
   include_context "pluginmanager validation helpers"

qa/integration/specs/command_line_spec.rb

@@ -19,6 +19,7 @@
 require_relative "../framework/settings"
 require_relative "../framework/helpers"
 
+# SPLIT_ESTIMATE: 12
 describe "CLI >" do
   before(:all) do
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/deprecation_log_spec.rb

@@ -22,6 +22,7 @@
 require "logstash/devutils/rspec/spec_helper"
 require "yaml"
 
+# SPLIT_ESTIMATE: 8
 describe "Test Logstash Pipeline id" do
   before(:all) {
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/direct_heap_allocator_flag_spec.rb

@@ -23,6 +23,7 @@
 require "yaml"
 require "manticore"
 
+# SPLIT_ESTIMATE: 40
 describe "Test Logstash buffer allocation setting" do
   before(:all) {
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/dlq_spec.rb

@@ -23,6 +23,7 @@
 
 require "logstash/devutils/rspec/spec_helper"
 
+# SPLIT_ESTIMATE: 30
 describe "Test Dead Letter Queue" do
   # template with an ip field
   let(:template) { serverless? ? { "index_patterns": ["te*"], "template": {"mappings": { "properties": { "ip": { "type": "ip" }}}} } :

qa/integration/specs/env_variables_condition_spec.rb

@@ -30,6 +30,7 @@
 # tag2 = mytag2
 # tag3 = mytag3
 ####################################
+# SPLIT_ESTIMATE: 19
 describe "Support environment variable in condition" do
   before(:all) {
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/env_variables_config_spec.rb

@@ -21,6 +21,7 @@
 require_relative '../framework/helpers'
 require "logstash/devutils/rspec/spec_helper"
 
+# SPLIT_ESTIMATE: 7
 describe "Test Logstash configuration" do
   before(:all) {
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/fatal_error_spec.rb

@@ -21,6 +21,7 @@
 require_relative '../services/logstash_service'
 require "logstash/devutils/rspec/spec_helper"
 
+# SPLIT_ESTIMATE: 15
 describe "uncaught exception" do
   before(:all) do
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/install_java_plugin_spec.rb

@@ -21,6 +21,7 @@
 require "logstash/devutils/rspec/spec_helper"
 require "stud/temporary"
 
+# SPLIT_ESTIMATE: 40
 describe "Install and run java plugin", :skip_fips do
   before(:all) do
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/java_api_spec.rb

@@ -24,6 +24,7 @@
 require "yaml"
 require "fileutils"
 
+# SPLIT_ESTIMATE: 20
 describe "Java plugin API" do
   before(:all) do
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/kafka_input_spec.rb

@@ -21,6 +21,7 @@
 require "rspec/wait"
 require "logstash/devutils/rspec/spec_helper"
 
+# SPLIT_ESTIMATE: 8
 describe "Test Kafka Input" do
   let(:num_retries) { 60 }
   let(:num_events) { 37 }

qa/integration/specs/logstash_to_logstash_spec.rb

@@ -23,6 +23,7 @@
 require 'stud/temporary'
 require 'logstash/devutils/rspec/spec_helper'
 
+# SPLIT_ESTIMATE: 20
 describe "Logstash to Logstash communication Integration test" do
 
   before(:all) {

qa/integration/specs/mixed_codec_spec.rb

@@ -24,6 +24,7 @@
 require "fileutils"
 require "logstash/devutils/rspec/spec_helper"
 
+# SPLIT_ESTIMATE: 26
 describe "Ruby codec when used in" do
   before(:all) {
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/monitoring_api_spec.rb

@@ -22,6 +22,7 @@
 require "logstash/devutils/rspec/spec_helper"
 require "stud/try"
 
+# SPLIT_ESTIMATE: 180
 describe "Test Monitoring API" do
   before(:each) do |example|
     $stderr.puts("STARTING: #{example.full_description} (#{example.location})")

qa/integration/specs/multiple_pipeline_spec.rb

@@ -23,6 +23,7 @@
 require "socket"
 require "yaml"
 
+# SPLIT_ESTIMATE: 30
 describe "Test Logstash service when multiple pipelines are used" do
   before(:all) {
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/pipeline_log_spec.rb

@@ -23,6 +23,7 @@
 require "yaml"
 require "fileutils"
 
+# SPLIT_ESTIMATE: 50
 describe "Test Logstash Pipeline id" do
   before(:all) {
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/plugin_name_log_spec.rb

@@ -22,6 +22,7 @@
 require "logstash/devutils/rspec/spec_helper"
 require "yaml"
 
+# SPLIT_ESTIMATE: 11
 describe "Test Logstash Pipeline id" do
   before(:all) {
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/pq_drain_spec.rb

@@ -23,6 +23,7 @@
 
 require 'stud/temporary'
 
+# SPLIT_ESTIMATE: 40
 if ENV['FEATURE_FLAG'] == 'persistent_queues'
   describe "Test logstash queue draining" do
     before(:all) { @fixture = Fixture.new(__FILE__) }

qa/integration/specs/reload_config_spec.rb

@@ -24,6 +24,7 @@
 require "json"
 require "logstash/util"
 
+# SPLIT_ESTIMATE: 20
 describe "Test Logstash service when config reload is enabled" do
   define_negated_matcher :exclude, :include
 

qa/integration/specs/reserved_tags_field_spec.rb

@@ -21,6 +21,7 @@
 require_relative '../framework/helpers'
 require "logstash/devutils/rspec/spec_helper"
 
+# SPLIT_ESTIMATE: 21
 # reserved tags should accept string and array of string only
 describe "Guard reserved tags field against incorrect use" do
   before(:all) {

qa/integration/specs/secret_store_spec.rb

@@ -30,6 +30,7 @@
 # tag2 = mytag2
 # tag3 = mytag3
 ####################################
+# SPLIT_ESTIMATE: 41
 describe "Test that Logstash" do
   before(:all) {
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/settings_spec.rb

@@ -22,6 +22,7 @@
 require "logstash/devutils/rspec/spec_helper"
 require "yaml"
 
+# SPLIT_ESTIMATE: 76
 describe "Test Logstash instance whose default settings are overridden" do
   before(:all) {
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/slowlog_spec.rb

@@ -22,6 +22,7 @@
 require "logstash/devutils/rspec/spec_helper"
 require "yaml"
 
+# SPLIT_ESTIMATE: 12
 describe "Test Logstash Slowlog" do
   before(:all) {
     @fixture = Fixture.new(__FILE__)

qa/integration/specs/webserver_spec.rb

@@ -22,6 +22,7 @@
 require "stud/try"
 require "manticore"
 
+# SPLIT_ESTIMATE: 5
 describe 'api webserver', :skip_fips do
   let!(:logger) { double("Logger").as_null_object }
   let!(:agent) { double("Agent").as_null_object }