diff --git a/.github/workflows/cbrain_ci.yaml b/.github/workflows/cbrain_ci.yaml index 9e251134f..a4d034bb8 100644 --- a/.github/workflows/cbrain_ci.yaml +++ b/.github/workflows/cbrain_ci.yaml @@ -49,6 +49,16 @@ jobs: with: ruby-version: 2.7.2 + ########################################################### + - name: Setup python + uses: actions/setup-python@v1 + + ########################################################### + - name: Setup python package Boutiques to install 'bosh' + uses: BSFishy/pip-action@v1 + with: + packages: boutiques + ########################################################### - name: Setup BrainPortal And Bourreau Names run: | diff --git a/Bourreau/config/application.rb b/Bourreau/config/application.rb index eacf73b1b..acac432fb 100644 --- a/Bourreau/config/application.rb +++ b/Bourreau/config/application.rb @@ -32,15 +32,6 @@ class Application < Rails::Application # properly set up all tasks installed from plugins (and the defaults tasks). config.autoload_paths += Dir["#{config.root}/cbrain_plugins/installed-plugins/cbrain_task"] - # CBRAIN Plugins load paths: add directory for descriptor-based CbrainTask - # models. This directory, similarly to the one above, contains symbolic - # links to a special loader code which will call a task generator to - # generate the requested CbrainTask subclass on the fly. - # - # The rake task cbrain:plugins:install:all also takes care of creating the - # symlinks for this location. - config.autoload_paths += Dir["#{config.root}/cbrain_plugins/installed-plugins/cbrain_task_descriptors"] - # Only load the plugins named here, in the order given (default is alphabetical). # :all can be used as a placeholder for all plugins not explicitly named. # config.plugins = [ :exception_notification, :ssl_requirement, :all ] diff --git a/Bourreau/lib/boutiques.schema.json b/Bourreau/lib/boutiques.schema.json new file mode 120000 index 000000000..42e9e1352 --- /dev/null +++ b/Bourreau/lib/boutiques.schema.json @@ -0,0 +1 @@ +../../BrainPortal/lib/boutiques.schema.json \ No newline at end of file diff --git a/Bourreau/lib/cbrain_task_generators b/Bourreau/lib/cbrain_task_generators deleted file mode 120000 index 6cc485eef..000000000 --- a/Bourreau/lib/cbrain_task_generators +++ /dev/null @@ -1 +0,0 @@ -../../BrainPortal/lib/cbrain_task_generators \ No newline at end of file diff --git a/Bourreau/spec/boutiques/boutiques_tester_spec.rb b/Bourreau/spec/boutiques/boutiques_tester_spec.rb index 9743c4ad2..0955a8f6a 100644 --- a/Bourreau/spec/boutiques/boutiques_tester_spec.rb +++ b/Bourreau/spec/boutiques/boutiques_tester_spec.rb @@ -42,39 +42,95 @@ # Run before block to create required input files before(:all) do + Userfile.all.each{ |uf| uf.destroy } + DataProvider.all.each { |dp| dp.destroy } # The group, provider, and user ids used downstream GID, UID, DPID = Group.everyone.id, User.admin.id, 9 @ftype = lambda { |fname| Userfile.suggested_file_type(fname) || SingleFile } + + @execer = RemoteResource.current_resource + # Get the schema and json descriptor + desc_path = File.join(__dir__, TestScriptDescriptor) + @descriptor = BoutiquesSupport::BoutiquesDescriptor.new_from_file(desc_path) + + @unrestricted_descriptor = @descriptor.dup + @unrestricted_descriptor.inputs.each { |input| input.optional = true; input.delete "requires-inputs" ; input.delete "disables-inputs" } + @unrestricted_descriptor.groups.each { |group| group.delete "mutually-exclusive" ; group.delete "one-is-required" } + + # Create tool and tool config and task class + tool = Tool.create_from_descriptor(@descriptor) + @tool_config = ToolConfig.create_from_descriptor(@execer, tool, @descriptor) + if ! BoutiquesTask.const_defined?(:BoutiquesTest) + BoutiquesBootIntegrator.link_from_json_file(desc_path) rescue nil + end + key = [ tool.name, @tool_config.version_name ] + @switchDescriptor = ->(desc) { + ToolConfig.instance_eval { @_descriptors_[key] = desc.dup } + } + @switchDescriptor.(@descriptor) # resets to full standard + # Create new data provider + @provider = FlatDirLocalDataProvider.new({ :online => true, :read_only => false, :remote_dir => '.' }) + @provider.id, @provider.name, @provider.user_id, @provider.group_id = DPID, 'test_provider', UID, GID + @provider.save! + @userFiles = InputFilesList.map { |f| uf=Userfile.new(:type => @ftype.(f), name: File.basename(f), data_provider_id: DPID, user_id: UID, group_id: GID); uf.save!; uf } + + @minimalDescriptor = NewMinimalTask.() + min_tool = Tool.create_from_descriptor(@minimalDescriptor) + @min_tool_config = ToolConfig.create_from_descriptor(@execer, min_tool, @minimalDescriptor) + if ! BoutiquesTask.const_defined?(:MinimalTask) + BoutiquesBootIntegrator.link_from_descriptor(@minimalDescriptor) rescue nil + end + + # Adjust numbers; tries to guess what they should + # be. We leave bad values alone (e.g. bad strings) for tests + @adjustNumbers = ->(task) do + task.boutiques_descriptor.inputs + .select { |input| input.type == 'Number' } + .each do |input| + val = task.invoke_params[input.id] + next if val.blank? + if val.is_a?(String) && val =~ /\A-?[0-9.]+\z/ + val = (input.integer ? Integer(val) : Float(val)) rescue val + task.invoke_params[input.id] = val + elsif val.is_a?(Array) + val = val.map do |x| + next x if x !~ /\A-?[0-9.]+\z/ + (input.integer ? Integer(x) : Float(x)) rescue x + end + task.invoke_params[input.id] = val + end + end + end end # Post-test cleanup via after block after(:all) do - destroyInputFiles - destroyOutputFiles + Dir.chdir(TempStore) do + destroyInputFiles + destroyOutputFiles + end end context "Cbrain external" do # Run before block to create ClusterTask object before(:each) do - # Get the schema and json descriptor - schema = SchemaTaskGenerator.default_schema - descriptor = File.join(__dir__, TestScriptDescriptor) - # Generate a new task class via the Boutiques framework and integrate it into cbrain - @boutiquesTask = SchemaTaskGenerator.generate(schema, descriptor) - @boutiquesTask.integrate if File.exists?(descriptor) # Create a new instance of the generated task class - @task = CbrainTask::BoutiquesTest.new - @task.tool_config = ToolConfig.new - @task.params = {} + @task = BoutiquesTask::BoutiquesTest.new + @task.cluster_workdir = 'fcw' + @task.tool_config_id = @tool_config.id + @task.params = {}.with_indifferent_access + @task.params[:invoke] = {}.with_indifferent_access # Assign it a bourreau resource = RemoteResource.current_resource @task.bourreau_id = resource.id # Give access to the generated task class itself - @task_const = "CbrainTask::#{SchemaTaskGenerator.classify(@task.name)}".constantize + @task_const = BoutiquesTask::BoutiquesTest end before(:all) do - createInputFiles + Dir.chdir(TempStore) do + createInputFiles + end end # Tests expected behaviour of the auto-generated cluster task @@ -83,7 +139,7 @@ # Check necessary properties of the descriptor object describe "descriptor" do it "has the right name" do - expect( @boutiquesTask.descriptor['name'] ).to eq( 'BoutiquesTest' ) + expect( @task.boutiques_descriptor.name ).to eq( 'BoutiquesTest' ) end end @@ -96,96 +152,31 @@ expect( @task_const.tool ).not_to be_nil end - # Test that the apply_template method works as expected - # for some representative (or previously buggy) cases - describe "apply_template" do - - # Define some default parameters to use in the tests - before(:each) do - @template = 'cmd [1] [2] [3] [4] [5]' - @def_keys = {'[1]' => 'one.txt', '[2]' => 2, '[3]' => 't.csv', '[4]' => nil, '[5]' => nil} - @flags = {'[1]' => '-1', '[2]' => '--long-flag', '[3]' => '-t', '[4]' => nil, '[5]' => nil} - @seps = {'[1]' => '=', '[2]' => '~', '[3]' => ' ', '[4]' => nil, '[5]' => nil} - end - - it "handles string substitutions" do - s = @task.apply_template(@template, @def_keys) - expect( s.strip ).to eq( 'cmd one.txt 2 t.csv' ) - end - - it "handles string substituions with spaces" do - s = @task.apply_template(@template, @def_keys.merge({'[4]' => '4 4'})) - expect( s.strip ).to eq( "cmd one.txt 2 t.csv '4 4'" ) - end - - # Characters special to the shell or ruby's gsub should not interfere - # In this case, ensure that ' is escaped properly - it "handles special meaning characters" do - s = @task.apply_template(@template, @def_keys.merge({'[4]' => " '; arg"})) - expect( s.strip ).to eq( "cmd one.txt 2 t.csv ' '\\''; arg'" ) - end - - it "handles substitutions with command-line flags" do - s = @task.apply_template(@template, @def_keys, flags: @flags) - expect( s.strip ).to eq( "cmd -1 one.txt --long-flag 2 -t t.csv" ) - end - - it "handles substitution with flag-type inputs (when true)" do - s = @task.apply_template(@template, @def_keys.merge({'[4]' => true}), flags: {'[4]' => '-f'}) - expect( s.strip ).to eq( "cmd one.txt 2 t.csv -f" ) - end - - it "handles substitution with flag-type inputs (when false)" do - s = @task.apply_template(@template, @def_keys.merge({'[4]' => false}), flags: {'[4]' => '-f'}) - expect( s.strip ).to eq( "cmd one.txt 2 t.csv" ) - end - - it "handles substitution with list-type inputs" do - s = @task.apply_template(@template, @def_keys.merge({'[4]' => ['a', 'b', 'c']}), flags: {'[4]' => '-l'}) - expect( s.strip ).to eq( "cmd one.txt 2 t.csv -l a b c" ) - end - - it "handles special flag separator substitution" do - s = @task.apply_template(@template, - @def_keys.merge({'[4]' => true}), - flags: @flags.merge({'[4]' => '-f'}), - separators: @seps) - expect( s.strip ).to eq( "cmd -1=one.txt --long-flag~2 -t t.csv -f" ) - end - - it "properly strips endings" do - s = @task.apply_template(@template, - @def_keys.merge({'[4]' => true, '[5]' => '9.tex'}), - flags: @flags.merge({'[4]' => '-f', '[5]' => '-tex'}), - separators: @seps, - strip: [ '.txt', '.tex' ]) - expect( s.strip ).to eq( "cmd -1=one --long-flag~2 -t t.csv -f -tex 9" ) - end - - end - # Test that creating a basic cluster command in isolation works it "can create cluster commands" do - @task.params[:A] = "A_VAL" - expect( NormedTaskCmd.(@task) ).to eq( './' + TestScriptName + ' -A A_VAL' ) + @switchDescriptor.(@unrestricted_descriptor) + @task.invoke_params[:A] = "A_VAL" + expect( NormedTaskCmd.(@task) ).to eq( './' + TestScriptName + ' -A A_VAL -r r.txt' ) end # Test that creating cluster commands with string lists works it "can create cluster commands with lists" do - @task.params[:A] = "A_VAL" - @task.params[:p] = ['e1', 'e2', 'e3'] - expect( NormedTaskCmd.(@task) ).to eq( './' + TestScriptName + ' -A A_VAL -p e1 e2 e3' ) + @task.invoke_params[:A] = "A_VAL" + @task.invoke_params[:p] = ['e1', 'e2', 'e3'] + expect( NormedTaskCmd.(@task) ).to eq( './' + TestScriptName + ' -A A_VAL -p e1 e2 e3 -r r.txt' ) end # Test that export commands work and are in the right place it "exports environment variables" do - expect( @task.cluster_commands[0] ).to eq("export ev1='nice_value'") + @switchDescriptor.(@unrestricted_descriptor) + expect( @task.cluster_commands.join('') ).to include("export ev1=nice_value\n") end # It properly escapes environment variables # Note that we only have to worry about inappropriate values that are still valid JSON it "escapes environment variables" do - expect( @task.cluster_commands[1] ).to eq("export ev2='ta- 9\"\\'\\''_%^&$@]['") + @switchDescriptor.(@unrestricted_descriptor) + expect( @task.cluster_commands.join('') ).to include("export ev2='ta- 9\"\\'\\''_%^&$@]['") end end @@ -198,44 +189,24 @@ # Generate a descriptor @descriptor = NewMinimalTask.() # Generates a task object from the minimal mock app - @generateTask = -> params { - genTask = SchemaTaskGenerator.generate(SchemaTaskGenerator.default_schema, @descriptor, false).integrate - task = CbrainTask::MinimalTest.new - task.params = params + @generateTask = -> (params) { + task = BoutiquesTask::MinimalTest.new(:tool_config_id => @min_tool_config.id, :bourreau_id => @execer.id) + task.cluster_workdir = "task_wd" + task.params = {}.with_indifferent_access + task.params[:invoke] = params.with_indifferent_access + key = [ @min_tool_config.tool.name, @min_tool_config.version_name ] + desc = @descriptor + ToolConfig.instance_eval { @_descriptors_[key] = desc } task } end - context 'cbrain integration' do - - # Simple test to ensure integration is correct - it "should work correctly" do - genTask = SchemaTaskGenerator.generate(SchemaTaskGenerator.default_schema, @descriptor, false).integrate - expect( (CbrainTask::MinimalTest.new).name ).to eq( "MinimalTest" ) # Check for task instance - expect( genTask.name ).to eq( "CbrainTask::MinimalTest" ) # Check for generated task class instance - end - - end - context 'cluster_command substitution' do # Test basic command substitution correctness it "should correctly substitute cluster_commands with default settings" do task = @generateTask.( { a: 'value' } ) - expect( task.cluster_commands[0].strip ).to eq( '/minimalApp -a value' ) - end - - # Test that optional "shell" in descriptor trigger the creation of a wrapper - it "should create wrapper commands for alternate shells" do - @descriptor['shell'] = 'myFakeShell' - task = @generateTask.( { a: 'value' } ) - expect( task.cluster_commands[0].strip ).to match( - / - myFakeShell\s*<<'BoutiquesShellWrapper'\s+ - \/minimalApp\s-a\svalue\s+ - BoutiquesShellWrapper - /x - ) + expect( task.cluster_commands[0].strip ).to include( "/minimalApp -a value\n" ) end # Test output flag substitution @@ -243,7 +214,7 @@ @descriptor['command-line'] += ' [OUT-KEY]' @descriptor['output-files'][0].merge!( { 'value-key' => '[OUT-KEY]', 'command-line-flag' => '-o' } ) task = @generateTask.( { a: 'value' } ) - expect( task.cluster_commands[0].strip ).to eq( '/minimalApp -a value -o value' ) + expect( task.cluster_commands[0].strip ).to include( "/minimalApp -a value -o value" ) end # Test output flag separator substitution @@ -254,7 +225,7 @@ 'command-line-flag' => '-o', 'command-line-flag-separator' => '=' } ) task = @generateTask.( { a: 'value' } ) - expect( task.cluster_commands[0].strip ).to eq( '/minimalApp -a value -o=value' ) + expect( task.cluster_commands[0].strip ).to include( "/minimalApp -a value -o=value" ) end # Test output flag separator substitution with prior path-template substitution @@ -267,7 +238,16 @@ 'path-template' => '[A]+[B]', 'command-line-flag-separator' => '/' } ) task = @generateTask.( { a: 'val', b: 9 } ) - expect( task.cluster_commands[0].strip ).to eq( '/minimalApp -a val -b 9 -o/val+9' ) + expect( task.cluster_commands[0].strip ).to include( "/minimalApp -a val -b 9 -o/val+9" ) + end + + # Test absolute file path + it "should correctly generate full paths if uses-absolute-path is set" do + @descriptor['command-line'] += ' [F1] [F2]' + @descriptor['inputs'] << GenerateJsonInputDefault.('f1','File','basename file', { 'uses-absolute-path' => false } ) + @descriptor['inputs'] << GenerateJsonInputDefault.('f2','File','abs path file', { 'uses-absolute-path' => true } ) + task = @generateTask.( { f1: Userfile.first.id , f2: Userfile.first.id } ) + expect( task.cluster_commands[0].strip ).to match( /minimalApp -f1 ([a-zA-Z0-9\.]+) -f2 \/.*\/\1/ ) end end @@ -280,21 +260,28 @@ # The cluster_commands method requires userfiles to exist before running now before(:each) do # Clean userfiles and data providers - Userfile.all.each{ |uf| uf.destroy } - DataProvider.all.each { |dp| dp.destroy } + #Userfile.all.each{ |uf| uf.destroy } + #DataProvider.all.each { |dp| dp.destroy } # Create new data provider - @provider = FlatDirLocalDataProvider.new({ :online => true, :read_only => false, :remote_dir => '.' }) - @provider.id, @provider.name, @provider.user_id, @provider.group_id = DPID, 'test_provider', UID, GID - @provider.save! + #@provider = FlatDirLocalDataProvider.new({ :online => true, :read_only => false, :remote_dir => '.' }) + #@provider.id, @provider.name, @provider.user_id, @provider.group_id = DPID, 'test_provider', UID, GID + #@provider.save! # Generate files used downstream @userFiles = InputFilesList.map { |f| @task.safe_userfile_find_or_new(@ftype.(f), name: File.basename(f), data_provider_id: DPID, user_id: UID, group_id: GID) } @userFiles.each { |f| f.save! } @idsForFiles = @userFiles.map { |f| f.id } + @switchDescriptor.(@descriptor) # resets to full standard + Dir.chdir(TempStore) do + destroyTaskSupportFiles + end end # After each local test, destroy the output files, so they don't interfere with downstream tests after(:each) do - destroyOutputFiles + Dir.chdir(TempStore) do + destroyOutputFiles + destroyTaskSupportFiles + end end # Check that userfiles are discoverable @@ -302,41 +289,51 @@ expect( @idsForFiles.all? { |t| Userfile.find_by_id( t ) } ).to be true end - # Note: -C is a file of the mock task with uses-absolute-path equaling true. -d doesn't specify. - it "handles uses-absolute-paths as intended" do - # Mock the location of the full cluster workdir - allow_any_instance_of( CbrainTask::BoutiquesTest ).to receive( :full_cluster_workdir ).and_return( File.join(Dir.pwd, TempStore) ) - s1, s2 = @task.apply_template('[C]', { '[C]' => @idsForFiles[0] }), @task.apply_template('[d]', { '[d]' => @idsForFiles[1] }) - expect( (Pathname.new(s1)).absolute? && ( !(Pathname.new(s2)).absolute? ) ).to be true - end - # Ensure that the `setup` method does not replace ids with hashes it "works with ids rather than objects" do - @task.params = ArgumentDictionary.( "-A a -B 9 -C #{C_file} -v s -n 7 ", @idsForFiles ) + @task.params[:invoke] = ArgumentDictionary.( "-A a -B 9 -C #{C_file} -v s -n 7 ", @idsForFiles ) @task.cluster_workdir = 'fcw' @task.setup - expect( @task.params[:C] ).to eq( @idsForFiles[0] ) + expect( @task.invoke_params[:C] ).to eq( @idsForFiles[0] ) end # Perform tests by running the cmd line given by cluster_commands and checking the exit code BasicTests.each do |test| + test_name = test[0] +#next unless test_name == "fails when a required argument is missing (A: flag + value)" + test_args = test[1] + test_status = test[2] + test_fnames = test[3] || [] + test_cberror = test[4] # message in exception when the failure is in cluster_commands # Testing for unrecognized inputs will not work here, since apply_template will ignore them - next if test[0].include?( "unrecognized" ) + next if test_name.include?( "unrecognized" ) # The apply_template method adds the separator on its own, so we need only check that works - next if test[0].include?( "fails when special separator" ) + next if test_name.include?( "fails when special separator" ) # Run the test - it "#{test[0]}" do + it "#{test_name}" do + Dir.chdir(TempStore) do # Mock the location of the full cluster workdir - allow_any_instance_of( CbrainTask::BoutiquesTest ).to receive( :full_cluster_workdir ).and_return( File.join(Dir.pwd, TempStore) ) + allow_any_instance_of( BoutiquesTask::BoutiquesTest ).to receive( :full_cluster_workdir ).and_return( File.join(Dir.pwd, TempStore) ) begin # Convert string arg to params dict - @task.params = ArgumentDictionary.( test[1], @idsForFiles ) + @task.params[:invoke] = ArgumentDictionary.( test_args, @idsForFiles ) + @adjustNumbers.(@task) rescue OptionParser::MissingArgument => e next # after_form does not need to check this here, since rails puts a value in the hash end # Run the generated command line from cluster_commands (-2 to ignore export lines and the echo log at -1) - exit_code = runTestScript( FileNamesToPaths.( @task.cluster_commands[-2].strip.gsub('./'+TestScriptName,'') ), test[3] || [] ) - # Check that the exit code is appropriate - expect( exit_code ).to eq( test[2] ) + if test_cberror.present? + expect { @task.cluster_commands }.to raise_error(Regexp.new(test_cberror)) + else + opts = extractOptions( @task.cluster_commands.join(""), TestScriptName ) + exit_code = runTestScript( + FileNamesToPaths.( + #@task.cluster_commands.join("").strip.gsub('./'+TestScriptName,'') + opts + ), test_fnames ) + # Check that the exit code is appropriate + expect( exit_code ).to eq( test_status ) + end + end # chdir end end @@ -349,7 +346,7 @@ # Define some useful constants (constants get redefined in before(:each) blocks) before(:all) do # The warning message used when unable to find optional output files - OptOutFileNotFoundWarning = "Unable to find optional output file: " + OptOutFileNotFoundWarning = "Skipped optional missing output file" # Current rails pwd PWD = Dir.pwd end @@ -357,6 +354,7 @@ # Note: FactoryGirl methods should be used instead, but only if their db changes get rolled back in the before(:each) block # This is currently not happening, possibly due to some versioning issues before(:each) do + Dir.chdir TempStore # Destroy any pre-existing userfiles Userfile.all.each{ |uf| uf.destroy } # Create a mock task required output file @@ -366,51 +364,54 @@ # Use helper method for getting filetype classes @userfileClass = @ftype.(@fname) # Get the schema and json descriptor - descriptor = File.join(__dir__, TestScriptDescriptor) - # Generate a new task class via the Boutiques framework and integrate it into cbrain - @boutiquesTask = SchemaTaskGenerator.generate(SchemaTaskGenerator.default_schema, descriptor) - @boutiquesTask.integrate + descriptor = @descriptor.dup # Instantiate an object of the new class type - @task = CbrainTask::BoutiquesTest.new + @task = BoutiquesTask::BoutiquesTest.new + @task.tool_config_id = @tool_config.id + @task.bourreau_id = @tool_config.bourreau_id # Destroy any prior existing data providers (so we use a clean, lone one) DataProvider.all.each { |dp| dp.destroy } # Create a local data_provider to hold our files @provider = FlatDirLocalDataProvider.new({ :online => true, :read_only => false, :remote_dir => '.' }) @provider.id, @provider.name, @provider.user_id, @provider.group_id = DPID, 'test_provider', UID, GID @provider.save! + @task.results_data_provider_id = @provider.id # Change base directory so checks for simulated files go to the right temp storage place # This is because some checks by e.g. save_results expect files from the task to be in the pwd # Passing a block to chdir would be preferable but then one would have to do it in every test - Dir.chdir TempStore allow( @task ).to receive( :full_cluster_workdir ).and_return( Dir.pwd ) # Add a local input file to the data provider (allows smarter lookup in userfile_exists) @file_c, @ft = 'c', @ftype.(@file_c) newFile = @task.safe_userfile_find_or_new(@ft, name: @file_c, data_provider_id: DPID, user_id: UID, group_id: GID) newFile.save! # Fill in data necessary for the task to check for and save the output file '@fname' - @task.params = {:ro => @fname_base, :interface_userfile_ids => [Userfile.all.first.id]} + @task.params ||= { :interface_userfile_ids => [Userfile.all.first.id] }.with_indifferent_access + @task.params[:invoke] = {}.with_indifferent_access + @task.params[:invoke].merge!({:r => @fname_base }) @task.user_id, @task.group_id = UID, GID # Generate a simulated exit file, as if the task had run - @simExitFile = @task.exit_cluster_filename + @simExitFile = @task.exit_status_filename File.write( @simExitFile, "0\n" ) # The basic properties for the required output file @reqOutfileProps = {:name => @fname_base, :data_provider_id => @provider.id} # Optional output file properties @optOutFileName = File.basename(OptOutName) # Implicitly in temp storage @optFileClass = @ftype.( @optOutFileName ) + @switchDescriptor.(@unrestricted_descriptor) end # Clean up after each test after(:each) do # Also need to get rid of the (simulated) exit file - File.delete( @simExitFile ) if File.exists?( @simExitFile ) + File.delete( @simExitFile ) if @simExitFile.present? && File.exists?( @simExitFile ) # Destroy the registered userfiles and the data_provider, so as not to affect downstream tests # Needed to destroy actual output files written to the filesystem Userfile.all.each{ |uf| uf.destroy } - # Return to rails base dir - Dir.chdir PWD # Delete any generated output files destroyOutputFiles + destroyTaskSupportFiles + # Return to rails base dir + Dir.chdir PWD end # Test that the generated Boutiques task can handle output file saving and renaming @@ -427,9 +428,9 @@ File.delete(@simExitFile) expect( @task.save_results ).to be false end - it "save_results is false if the exit status file has invalid content" do + it "save_results raises an exception if the exit status file has invalid content" do File.write( @simExitFile, "abcde\n" ) - expect( @task.save_results ).to be false + expect { @task.save_results }.to raise_error(/Exit status file.*has unexpected content/) end it "save_results is false if the exit status file contains a value greater than 1" do File.write( @simExitFile, "3\n" ) @@ -439,10 +440,11 @@ # Check that save_results works as expected for existent files it "can save results files" do # Make sure the file on the filesystem exists - expect( File.exists? @fname_base ).to be true + expect( File.exists?(@fname_base) ).to be true # Ensure the file has not been registered/created yet expect( @task.userfile_exists(@userfileClass, @reqOutfileProps) ).to be false # Ensure that saving the results occurs error-free + @task.cluster_commands expect( @task.save_results ).to be true # Ensure that the file now exists in the data_provider # Output name now contains the run_id inside it... @@ -457,6 +459,7 @@ # Destroy the required output file File.delete( @fname_base ) # Attempting to save_results should return a 'failure' error code + @task.cluster_commands expect( @task.save_results ).to be false end @@ -465,10 +468,11 @@ # Create the optional output file FileUtils.touch( @optOutFileName ) # Inform the generated task to look for the optional output file - @task.params = @task.params.merge({:oo => @optOutFileName}) + @task.params[:invoke] = @task.invoke_params.merge!({:o => @optOutFileName}) # Ensure the file exists expect( File.exists? @optOutFileName ).to be true # Attempt to save both the optional and required output files. Should succeed. + @task.cluster_commands expect( @task.save_results ).to be true # Both files should exist in the data_provider newReqName = "r-#{@task.run_id}.txt" @@ -483,10 +487,12 @@ # However, a logging should occur to note that fact it "handles absent optional output files properly" do # Inform the generated task to look for the optional output file - @task.params = @task.params.merge({:oo => @optOutFileName}) + @task.params[:invoke] = @task.invoke_params.merge!({:o => @optOutFileName}) + @switchDescriptor.(@unrestricted_descriptor) # resets to full standard # Ensure the file does not exist expect( File.exists? @optOutFileName ).to be false # Attempt to save both the optional and required output files. Should succeed (in terms of return value). + @task.cluster_commands expect( @task.save_results ).to be true # Only the required file should exist in the data_provider newReqName = "r-#{@task.run_id}.txt" @@ -494,7 +500,7 @@ expect( @task.userfile_exists(@userfileClass, {:name => newReqName, :data_provider_id => @provider.id}) ).to be true expect( @task.userfile_exists(@optFileClass, {:name => newOptName, :data_provider_id => @provider.id}) ).to be false # Check that a notice was logged to warn of the missing optional output file - expect( @task.getlog.include?( OptOutFileNotFoundWarning ) ).to be true + expect( @task.getlog ).to include( OptOutFileNotFoundWarning ) end # Ensure the files do not survive between tests @@ -504,117 +510,6 @@ end # End output file handling tests - # Test that a Tool Config is created iff both the bourreau and the descriptor specify docker - # Also ensure move commands are added to cluster_commands when needed - context 'Containerized Behaviour' do - - before(:each) do - # Ensure the Bourreau does not have docker installed by default - resource = RemoteResource.current_resource - resource.docker_executable_name = "docker" - resource.save! - # Get the schema and json descriptor - @schema = SchemaTaskGenerator.default_schema - @descriptor = File.join(__dir__, TestScriptDescriptor) - # Generate a new task class via the Boutiques framework, without integrating it - @boutiquesTask = SchemaTaskGenerator.generate(@schema, @descriptor) - @boutiquesTask.descriptor["container-image"] = nil # tool does not use docker by default - @task_const_name = "CbrainTask::#{SchemaTaskGenerator.classify(@boutiquesTask.name)}" - # Destroy any tools/toolconfigs for the tool, if any exist - ToolConfig.where(tool_id: CbrainTask::BoutiquesTest.tool.id).destroy_all rescue nil - Tool.where(:cbrain_task_class_name => @task_const_name).destroy_all rescue nil - # Placeholder image - @dockerImg = { "type" => "docker", "image" => "image" } - # Check for mv commands - @nMvCmds = lambda { |a| a.reduce(0) { |t,s| t + ( s.start_with?("mv ") ? 1 : 0 ) } } - @setupForMvTests = lambda do |wd, outfile1, outfile2 = nil| - # Tell the Bourreau to use docker - resource = RemoteResource.current_resource - resource.docker_executable_name = "docker" - resource.save! - # Make the task execute in a specific container dir - descriptor = SchemaTaskGenerator.expand_json(@descriptor) - descriptor['container-image'] = @dockerImg.merge( { "working-directory" => wd } ) - # Force an output file to be generated outside of that dir or its subtree - descriptor["output-files"][0]["path-template"] = outfile1 - descriptor["output-files"][1]["path-template"] = outfile2 unless outfile2.nil? - boutiquesTask = SchemaTaskGenerator.generate(@schema, descriptor) - # Generate the task class via templating - boutiquesTask.integrate if File.exists?(@descriptor) - # Ensure cluster_commands accounts for the problem by mocking the environment - allow_any_instance_of( CbrainTask::BoutiquesTest ).to receive( :use_docker? ).and_return( true ) - allow_any_instance_of( CbrainTask::BoutiquesTest ).to receive( :full_cluster_workdir ).and_return( File.join(Dir.pwd, TempStore) ) - task = CbrainTask::BoutiquesTest.new - userFiles = InputFilesList.map { |f| task.safe_userfile_find_or_new(@ftype.(f), name: File.basename(f), data_provider_id: DPID, user_id: UID, group_id: GID) } - userFiles.each { |f| f.save! } - idsForFiles = userFiles.map { |f| f.id } - return task, idsForFiles - end - end - - after(:each) do - # Destroy any tools/toolconfigs for the tool, if any exist - ToolConfig.where(tool_id: @task_const_name.constantize.tool.id).destroy_all - Tool.where(:cbrain_task_class_name => @task_const_name).destroy_all - # Ensure the Bourreau does not have docker installed by default - resource = RemoteResource.current_resource - resource.docker_executable_name = "docker" - resource.save! - end - - it "correctly adds mv commands when relative optional files are specified" do - task, idsForFiles = @setupForMvTests.( "/launch/", "/far/away/[r]") - task.params = ArgumentDictionary.( "-A a -B 9 -C #{C_file} -v s -n 7 -r r -o optOutFile", idsForFiles ) - expect( @nMvCmds.( task.cluster_commands ) ).to eq( 1 ) # It's a relative path, so only one mv need be done - end - - it "correctly adds mv commands when optional files are not specified" do - task, idsForFiles = @setupForMvTests.( "/launch/", "/far/away/[r]") - task.params = ArgumentDictionary.( "-A a -B 9 -C #{C_file} -v s -n 7 -r r", idsForFiles ) - expect( @nMvCmds.( task.cluster_commands ) ).to eq( 1 ) # Only mv required file - end - - it "correctly adds mv commands when absolute optional files are specified" do - task, idsForFiles = @setupForMvTests.( "/launch/", "/far/away/[r]", "/another/evil/dir/[o]" ) - task.params = ArgumentDictionary.( "-A a -B 9 -C #{C_file} -v s -n 7 -r r -o optOutFile", idsForFiles ) - expect( @nMvCmds.( task.cluster_commands ) ).to eq( 2 ) # Need to mv both files - end - - - it "is not created when Bourreau does not support Docker" do - resource = RemoteResource.current_resource - resource.docker_executable_name = "" - resource.save! - @boutiquesTask.descriptor['container-image'] = @dockerImg - @boutiquesTask.integrate if File.exists?(@descriptor) - expect( ToolConfig.exists?( :tool_id => @task_const_name.constantize.tool.id ) ).to be false - end - - it "is not created when descriptor has no Docker image" do - RemoteResource.current_resource.docker_executable_name = "docker" - RemoteResource.current_resource.save! - @boutiquesTask.integrate if File.exists?(@descriptor) - expect( ToolConfig.exists?( :tool_id => @task_const_name.constantize.tool.id ) ).to be false - end - - it "is created when descriptor has Docker image and Bourreau has Docker" do - @boutiquesTask.descriptor['container-image'] = @dockerImg - resource = RemoteResource.current_resource - resource.docker_executable_name = "docker" - resource.save! - @boutiquesTask.integrate if File.exists?(@descriptor) - expect( ToolConfig.exists?( :tool_id => @task_const_name.constantize.tool.id ) ).to be true - end - - # When neither criteria is met, check that a ToolConfig is not made - # Also checks that the integration changes were rolled back - it "is not created when Bourreau does not support Docker and descriptor has no Docker image" do - @boutiquesTask.integrate if File.exists?(@descriptor) - expect( ToolConfig.exists?( :tool_id => @task_const_name.constantize.tool.id ) ).to be false - end - - end - end end diff --git a/Bourreau/spec/rails_helper.rb b/Bourreau/spec/rails_helper.rb index d2cc5dc83..b12a68a90 100644 --- a/Bourreau/spec/rails_helper.rb +++ b/Bourreau/spec/rails_helper.rb @@ -13,6 +13,7 @@ # DATABASE_URL=mysql2://prioux:mypassword@localhost/prioux_test # require 'yaml' +require 'uri' 1.times do # a block to encapsulate local variables and not pollute anything env = ENV['RAILS_ENV'] dbconfig_file = "../BrainPortal/config/database.yml" diff --git a/BrainPortal/app/models/boutiques_cluster_task.rb b/BrainPortal/app/models/boutiques_cluster_task.rb index 47ee0b72e..770b5d5a7 100644 --- a/BrainPortal/app/models/boutiques_cluster_task.rb +++ b/BrainPortal/app/models/boutiques_cluster_task.rb @@ -118,22 +118,33 @@ def cluster_commands #:nodoc: descriptor = self.descriptor_for_cluster_commands invoke_struct = self.invoke_params.dup # as it is in the task's params + # Used when building absolute paths + workdir = Pathname.new(self.full_cluster_workdir) + # Replace userfile IDs for file basenames in the invoke struct descriptor.file_inputs.each do |input| userfile_id = invoke_params[input.id] next if userfile_id.blank? # that happens when it's an optional file - userfile = Userfile.find(userfile_id) + userfiles = Array(Userfile.find(userfile_id)) # can be one or many + jsonfnames = userfiles.map(&:name) # by default, the basename + jsonfnames = jsonfnames.map { |base| (workdir + base).to_s } if input.uses_absolute_path # Most common situation - if ! input.list || ! userfile.is_a?(CbrainFileList) - invoke_struct[input.id] = (input.list ? [ userfile.name ] : userfile.name) + if ! input.list || ! userfiles.first.is_a?(CbrainFileList) + invoke_struct[input.id] = (input.list ? jsonfnames : jsonfnames.first ) next end # In case the input is a list and is assigned a CbrainFileList + userfile = userfiles.first + cb_error "Expected a CbrainFileList as a single entry for input '#{input.name}'" if + userfiles.size > 1 || ! userfile.is_a?(CbrainFileList) userfile.sync_to_cache userfile_list = userfile.userfiles_accessible_by_user!(user, nil, nil, file_access_symbol) subnames = userfile_list.compact.map(&:name) # [ 'userfilename1', 'userfilename2' ... ] + if input.uses_absolute_path + subnames = subnames.map { |base| (workdir + base).to_s } + end invoke_struct[input.id] = subnames end @@ -174,10 +185,25 @@ def cluster_commands #:nodoc: end simul_status = $? # a Process::Status object if ! simul_status.success? - cb_error "The 'bosh exec simulate' command failed with return code #{simul_status.exitstatus}" + bosh_message = simulout.split("\n").detect { |line| line =~ /ERROR/ } + bosh_message &&= ": #{bosh_message}" + cb_error "The 'bosh exec simulate' command failed with return code #{simul_status.exitstatus}#{bosh_message}" end simulout.sub!(/^Generated.*\n/,"") # header junk from simulate - commands = <<-COMMANDS + + commands = "" + if descriptor.environment_variables.present? + commands = "# Environment variables defined by Boutiques descriptor\n" + descriptor.environment_variables.each do |struct| + ename = (struct['name'] || struct[:name] ).to_s + evalue = (struct['value'] || struct[:value]).to_s + next if ename.blank? || ! ename.match(/\A[a-zA-Z]\w+\z/) + commands += "export #{ename.strip}=#{evalue.bash_escape}\n" + end + commands += "\n" + end + + commands += <<-COMMANDS # Main tool command, generated with bosh exec simulate #{simulout.strip} status=$? diff --git a/BrainPortal/app/models/boutiques_portal_task.rb b/BrainPortal/app/models/boutiques_portal_task.rb index 90727c07d..a27116e76 100644 --- a/BrainPortal/app/models/boutiques_portal_task.rb +++ b/BrainPortal/app/models/boutiques_portal_task.rb @@ -115,10 +115,12 @@ def before_form # return "Warning: you selected more files than this task requires, so you won't be able to assign them all." # Not available in case of descriptor qualified to launch multiple task - if !descriptor.qualified_to_launch_multiple_tasks? && (num_in_files < num_needed_inputs || num_in_files > num_needed_inputs+num_opt_inputs) - message = "This task requires #{num_needed_inputs} mandatory file(s) and #{num_opt_inputs} optional file(s)\n" + - input_infos - cb_error message + if num_in_files < num_needed_inputs || num_in_files > num_needed_inputs+num_opt_inputs + if (! descriptor.qualified_to_launch_multiple_tasks? || num_in_files == 0) + message = "This task requires #{num_needed_inputs} mandatory file(s) and #{num_opt_inputs} optional file(s)\n" + + input_infos + cb_error message + end end "" @@ -299,6 +301,7 @@ def final_task_list #:nodoc: # -------------------------------------- if descriptor.file_inputs.size == 1 || descriptor.qualified_to_launch_multiple_tasks? input = descriptor.file_inputs.first + input = descriptor.sole_mandatory_file_input if descriptor.qualified_to_launch_multiple_tasks? fillTask = lambda do |userfile,tsk,extra_params=nil| tsk.params[:interface_userfile_ids] |= [ userfile.id.to_s ] @@ -445,6 +448,8 @@ def cbcsv_files(descriptor = self.descriptor_for_after_form) next if isInactive(input) userfile_id = invoke_params[input.id] next if userfile_id.blank? + next if userfile_id.is_a?(Array) && userfile_id.size > 1 # list = true + userfile_id = userfile_id.first if userfile_id.is_a?(Array) userfile = Userfile.find_accessible_by_user(userfile_id, self.user, :access_requested => file_access_symbol()) next unless ( userfile.is_a?(CbrainFileList) || (userfile.suggested_file_type || Object) <= CbrainFileList ) [ input, userfile ] @@ -576,16 +581,18 @@ def sanitize_param(input) # Nothing special required for strings, bar for symbols being acceptable strings. when :string - value = value.to_s if value.is_a?(Symbol) - params_errors.add(invokename, " is not a string (#{value})") unless value.is_a?(String) - value.strip! if value.is_a?(String) - params_errors.add(invokename, " is blank") if value.blank? && !empty_string_allowed - # The following three checks are to prevent cases when - # a string param is used as a path - params_errors.add(invokename, " cannot contain newlines") if value =~ /[\n\r]/ - params_errors.add(invokename, " cannot start with this character") if value =~ /^[\.\/]+/ - params_errors.add(invokename, " cannot move up dirs") if value.include? "/../" - if value.present? && input.value_choices.blank? + value = value.to_s if value.is_a?(Symbol) + params_errors.add(invokename, " is not a string") unless value.is_a?(String) + value = value.to_s.strip # now force it + params_errors.add(invokename, " is blank") if value.blank? && !empty_string_allowed + # The following checks are to prevent cases when a string param is used as a path + if value.present? + params_errors.add(invokename, " cannot contain newlines") if value =~ /[\n\r]/ + params_errors.add(invokename, " cannot start with this character") if value =~ /^[\.\/]+/ + params_errors.add(invokename, " cannot move up dirs") if value.include? "/../" + end + # Finally, check allowed characters + if value.present? && input.value_choices.blank? # valid value choices are checked elsewhere params_errors.add(invokename, " contains invalid characters") unless value.match?(charset_regex) # we can use a string in the match method end @@ -628,7 +635,7 @@ def sanitize_param(input) end def check_enum_param(input) - value = invoke_params[input.id] + value = invoke_params[input.id] || input.default_value string_values = Array(value).map(&:to_s) allowed_values = input.value_choices.map(&:to_s) return if (string_values - allowed_values).empty? # I hope that comparing the sets as strings is OK @@ -714,7 +721,7 @@ def check_allornone_group(group, descriptor = self.descriptor_for_after_form) # MAYBE IN COMMON def invoke_params - self.params[:invoke] ||= {} + self.params[:invoke] ||= {}.with_indifferent_access end # In the case of a misconfiguration of the portal, or if the file for diff --git a/BrainPortal/app/models/cbrain_task.rb b/BrainPortal/app/models/cbrain_task.rb index bc634d8c2..94eeb01a1 100644 --- a/BrainPortal/app/models/cbrain_task.rb +++ b/BrainPortal/app/models/cbrain_task.rb @@ -1109,12 +1109,9 @@ def struct_runtime_info(runtime_textfile=self.runtime_info) #:nodoc: # Patch: pre-load all model files for the subclasses def self.preload_subclasses - [ CBRAIN::TasksPlugins_Dir, CBRAIN::TaskDescriptorsPlugins_Dir ].each do |dir| + [ CBRAIN::TasksPlugins_Dir ].each do |dir| Dir.chdir(dir) do Dir.glob("*.rb").each do |rubyfile| - next if rubyfile == 'cbrain_task_class_loader.rb' # skip that - next if rubyfile == 'cbrain_task_descriptor_loader.rb' # skip that - model = rubyfile.sub(/.rb\z/, '') require_dependency "#{dir}/#{model}.rb" unless [ model.classify, model.camelize ].any? { |m| CbrainTask.const_defined?(m) rescue nil } diff --git a/BrainPortal/app/views/tasks/_params.html.erb b/BrainPortal/app/views/tasks/_params.html.erb index b03074874..d9a387e84 100644 --- a/BrainPortal/app/views/tasks/_params.html.erb +++ b/BrainPortal/app/views/tasks/_params.html.erb @@ -24,17 +24,10 @@ <% locals = { :params => @task.params, :form => form } - raw_partial = lambda do |partial| - @task.raw_partial(partial) if @task.respond_to?(:raw_partial) - end %>

Task Parameters - <% if raw_partial.(:edit_help) %> - <%= overlay_content_link "(Help)", :class => "task_help_link", :enclosing_element => "span" do %> - <%= render :inline => raw_partial.(:edit_help), :locals => locals %> - <% end %> - <% elsif (public_path = @task.public_path("edit_params_help.html")) %> + <% if (public_path = @task.public_path("edit_params_help.html")) %> <%= overlay_ajax_link "(Help)", public_path.to_s, :class => "task_help_link" %> <% elsif (edit_help_partial = task_partial(:edit_help) rescue nil) %> <%= overlay_content_link "(Help)", :class => "task_help_link", :enclosing_element => "span" do %> @@ -46,9 +39,5 @@
<%= error_messages_for(@task) %> -<% if raw_partial.(:task_params) %> - <%= render :inline => raw_partial.(:task_params), :locals => locals %> -<% else %> - <%= render :partial => task_partial('task_params'), :locals => locals %> -<% end %> +<%= render :partial => task_partial('task_params'), :locals => locals %>
diff --git a/BrainPortal/app/views/tasks/show.html.erb b/BrainPortal/app/views/tasks/show.html.erb index 8f0f06cfe..f3c5cb1e5 100644 --- a/BrainPortal/app/views/tasks/show.html.erb +++ b/BrainPortal/app/views/tasks/show.html.erb @@ -203,15 +203,8 @@ <% begin %> <% locals = { :task => @task, :params => @task.params } - raw_partial = lambda do |partial| - @task.raw_partial(partial) if @task.respond_to?(:raw_partial) - end %> - <% if raw_partial.(:show_params) %> - <%= render :inline => raw_partial.(:show_params), :locals => locals %> - <% else %> - <%= render :partial => task_partial(:show_params), :locals => locals %> - <% end %> + <%= render :partial => task_partial(:show_params), :locals => locals %> <% rescue ActionView::MissingTemplate %> 
Problem loading summary view (no template provided by task author).
<% rescue => ex %> diff --git a/BrainPortal/cbrain_plugins/cbrain-plugins-base/cbrain_task_descriptors/boutiques_demo.json b/BrainPortal/cbrain_plugins/cbrain-plugins-base/cbrain_task_descriptors/boutiques_demo.json deleted file mode 100644 index 746ab0ee4..000000000 --- a/BrainPortal/cbrain_plugins/cbrain-plugins-base/cbrain_task_descriptors/boutiques_demo.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "name": "BoutiquesDemo", - "tool-version": "5.1.2", - "schema-version": "0.5", - "author": "Pierre Rioux", - "description": "A demo CBRAIN task using a Boutiques descriptor. This program runs the UNIX 'du' command on a file or directory.", - "descriptor-url": "https://github.com/aces/cbrain/blob/master/BrainPortal/cbrain_plugins/cbrain-plugins-base/cbrain_task_descriptors/boutiques_demo.json", - "command-line": "du [ALL] [HUMAN] [FOLLOW_LINKS] [CBRAIN_INPUT] | tee [DU_OUTPUT_NAME]", - "inputs": [ - { - "name": "Input file or directory", - "id": "my_input", - "description": "Any file or directory in CBRAIN", - "type": "File", - "optional": false, - "list": false, - "value-key": "[CBRAIN_INPUT]" - }, - { - "name": "Name of report", - "id": "my_output_name", - "description": "The name of the text report containing the output of the 'du' command", - "type": "String", - "optional": false, - "list": false, - "value-key": "[DU_OUTPUT_NAME]" - }, - { - "name": "All files", - "id": "option_a", - "description": "Whether or not to provide a breakdown of each and every file", - "type": "Flag", - "optional": true, - "command-line-flag": "-a", - "value-key": "[ALL]" - }, - { - "name": "Human readable values", - "id": "option_h", - "description": "If true, units will be shown in readable human values, otherwise in blocks", - "type": "Flag", - "optional": true, - "command-line-flag": "-h", - "value-key": "[HUMAN]" - }, - { - "name": "Follow initial symlinks", - "id": "option_upper_h", - "description": "If true, initial symlinks are followed. Needed for file collections in CBRAIN", - "type": "Flag", - "optional": true, - "default-value": true, - "command-line-flag": "-H", - "value-key": "[FOLLOW_LINKS]" - } - ], - "output-files": [ - { - "name": "Output report", - "id": "du_report_out", - "description": "The output of the 'du' command", - "optional": false, - "list": false, - "path-template": "[DU_OUTPUT_NAME]" - } - ], - "groups": [ - { - "id": "files", - "name": "Files", - "members": [ - "my_output_name", - "my_input" - ] - }, - { - "id": "options", - "name": "Options", - "members": [ - "option_a", - "option_h", - "option_upper_h" - ] - } - ], - "tags": { - "domain": [ - "boutiques", - "testing", - "cbrain", - "platform", - "internal" - ] - }, - "suggested-resources": { - "cpu-cores": 1, - "ram": 1, - "walltime-estimate": 60 - }, - "custom": { - "cbrain:readonly-input-files": true - } -} diff --git a/BrainPortal/cbrain_plugins/cbrain-plugins-base/cbrain_task_descriptors/multi_boutiques_demo.json b/BrainPortal/cbrain_plugins/cbrain-plugins-base/cbrain_task_descriptors/multi_boutiques_demo.json deleted file mode 100644 index 3896b7b53..000000000 --- a/BrainPortal/cbrain_plugins/cbrain-plugins-base/cbrain_task_descriptors/multi_boutiques_demo.json +++ /dev/null @@ -1,132 +0,0 @@ -{ - "name": "MultiBoutiquesDemo", - "tool-version": "6.1.2", - "schema-version": "0.5", - "author": "Pierre Rioux", - "description": "A demo CBRAIN task using a Boutiques descriptor. This program runs the UNIX 'du' command on some files or directories.", - "descriptor-url": "https://github.com/aces/cbrain/blob/master/BrainPortal/cbrain_plugins/cbrain-plugins-base/cbrain_task_descriptors/multi_boutiques_demo.json", - "command-line": "du [ALL] [HUMAN] [FOLLOW_LINKS] [SINPUT1] [MINPUT1] [SINPUT2] [MINPUT2] | tee [DU_OUTPUT_NAME]", - "inputs": [ - { - "name": "Required single input 1", - "id": "sinput1", - "description": "Any file or directory in CBRAIN", - "type": "File", - "optional": false, - "list": false, - "value-key": "[SINPUT1]" - }, - { - "name": "Required multi input 1", - "id": "minput1", - "description": "Any file or directory in CBRAIN, or even a CbrainFileList", - "type": "File", - "optional": false, - "list": true, - "value-key": "[MINPUT1]" - }, - { - "name": "Optional single input 2", - "id": "sinput2", - "description": "Any file or directory in CBRAIN", - "type": "File", - "optional": true, - "list": false, - "value-key": "[SINPUT2]" - }, - { - "name": "Optional multi input 2", - "id": "minput2", - "description": "Any file or directory in CBRAIN, or even a CbrainFileList", - "type": "File", - "optional": true, - "list": true, - "value-key": "[MINPUT2]" - }, - { - "name": "Name of report", - "id": "my_output_name", - "description": "The name of the text report containing the output of the 'du' command", - "type": "String", - "optional": false, - "list": false, - "value-key": "[DU_OUTPUT_NAME]" - }, - { - "name": "All files", - "id": "option_a", - "description": "Whether or not to provide a breakdown of each and every file", - "type": "Flag", - "optional": true, - "command-line-flag": "-a", - "value-key": "[ALL]" - }, - { - "name": "Human readable values", - "id": "option_h", - "description": "If true, units will be shown in readable human values, otherwise in blocks", - "type": "Flag", - "optional": true, - "command-line-flag": "-h", - "value-key": "[HUMAN]" - }, - { - "name": "Follow initial symlinks", - "id": "option_upper_h", - "description": "If true, initial symlinks are followed. Needed for file collections in CBRAIN", - "type": "Flag", - "optional": true, - "default-value": true, - "command-line-flag": "-H", - "value-key": "[FOLLOW_LINKS]" - } - ], - "output-files": [ - { - "name": "Output report", - "id": "du_report_out", - "description": "The output of the 'du' command", - "optional": false, - "list": false, - "path-template": "[DU_OUTPUT_NAME]" - } - ], - "groups": [ - { - "id": "files", - "name": "Files", - "members": [ - "sinput1", - "minput1", - "sinput2", - "minput2" - ] - }, - { - "id": "options", - "name": "Options", - "members": [ - "option_a", - "option_h", - "option_upper_h" - ] - } - ], - "tags": { - "domain": [ - "boutiques", - "testing", - "cbrain", - "platform", - "internal" - ] - }, - "suggested-resources": { - "cpu-cores": 1, - "ram": 1, - "walltime-estimate": 60 - }, - "custom": { - "cbrain:readonly-input-files": true - } -} diff --git a/BrainPortal/cbrain_plugins/installed-plugins/cbrain_task_descriptors/.gitignore b/BrainPortal/cbrain_plugins/installed-plugins/cbrain_task_descriptors/.gitignore deleted file mode 100644 index 8947531f0..000000000 --- a/BrainPortal/cbrain_plugins/installed-plugins/cbrain_task_descriptors/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!.gitignore -!cbrain_task_descriptor_loader.rb diff --git a/BrainPortal/cbrain_plugins/installed-plugins/cbrain_task_descriptors/cbrain_task_descriptor_loader.rb b/BrainPortal/cbrain_plugins/installed-plugins/cbrain_task_descriptors/cbrain_task_descriptor_loader.rb deleted file mode 100644 index 95f510bc0..000000000 --- a/BrainPortal/cbrain_plugins/installed-plugins/cbrain_task_descriptors/cbrain_task_descriptor_loader.rb +++ /dev/null @@ -1,71 +0,0 @@ - -# -# CBRAIN Project -# -# Copyright (C) 2008-2012 -# The Royal Institution for the Advancement of Learning -# McGill University -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -# -# CbrainTask descriptor loader -# - -1.times do # just starts a block so local variables don't pollute anything - - basename = File.basename(__FILE__) - if basename == 'cbrain_task_descriptor_loader.rb' # usually, the symlink destination - # This can happen with eager loading. - #puts "Weird. Trying to load the loader?!?" - break - end - - schema = SchemaTaskGenerator.default_schema - descriptor = __FILE__.sub(/.rb\z/,'.json') - next unless File.exists?(descriptor) # Bad or broken symlink? Missing json? Ignore. - - descriptor_basename = Pathname.new(descriptor).basename - - begin - generator = SchemaTaskGenerator.generate(schema, descriptor) - rescue StandardError => e - generator = nil - #puts "=================" - puts "C> Failed to generate CbrainTask from descriptor '#{descriptor_basename}'." - puts "C> Error Message: #{e.class} #{e.message}" - #puts e.backtrace.join("\n"); - #puts "=================" - end - - # This is a check performed while we transition from the old integrator to the new one - new_integrated_tool = Tool.where(:cbrain_task_class_name => "BoutiquesTask::#{generator.name}").first - if new_integrated_tool - puts "C> Skipping integration of CbrainTask::#{generator.name} : new integration already present." - break - end - - begin - generator.integrate if generator - puts "C> [DEPRECATED] Integrated CbrainTask::#{generator.name} from descriptor '#{descriptor_basename}'" - rescue StandardError => e - #puts "=================" - puts "C> Failed to integrate CbrainTask from descriptor '#{descriptor_basename}'." - puts "C> Error Message: #{e.class} #{e.message}" - #puts e.backtrace.join("\n"); - #puts "=================" - end - -end diff --git a/BrainPortal/config/application.rb b/BrainPortal/config/application.rb index cd0387823..bbdcf3944 100644 --- a/BrainPortal/config/application.rb +++ b/BrainPortal/config/application.rb @@ -32,15 +32,6 @@ class Application < Rails::Application # properly set up all tasks installed from plugins (and the defaults tasks). config.autoload_paths += Dir["#{config.root}/cbrain_plugins/installed-plugins/cbrain_task"] - # CBRAIN Plugins load paths: add directory for descriptor-based CbrainTask - # models. This directory, similarly to the one above, contains symbolic - # links to a special loader code which will call a task generator to - # generate the requested CbrainTask subclass on the fly. - # - # The rake task cbrain:plugins:install:all also takes care of creating the - # symlinks for this location. - config.autoload_paths += Dir["#{config.root}/cbrain_plugins/installed-plugins/cbrain_task_descriptors"] - config.action_controller.include_all_helpers = true end diff --git a/BrainPortal/config/initializers/cbrain.rb b/BrainPortal/config/initializers/cbrain.rb index 3a69b1df6..c09a0b4af 100644 --- a/BrainPortal/config/initializers/cbrain.rb +++ b/BrainPortal/config/initializers/cbrain.rb @@ -49,8 +49,6 @@ class CBRAIN TasksPlugins_Dir = "#{Plugins_Dir}/installed-plugins/cbrain_task" # singular; historical ViewsPlugins_Dir = "#{Plugins_Dir}/installed-plugins/views" - # Original integrator - TaskDescriptorsPlugins_Dir = "#{Plugins_Dir}/installed-plugins/cbrain_task_descriptors" # New integrator BoutiquesDescriptorsPlugins_Dir = "#{Plugins_Dir}/installed-plugins/boutiques_descriptors" diff --git a/BrainPortal/lib/cbrain_task_generators/schemas/boutiques.schema.json b/BrainPortal/lib/boutiques.schema.json similarity index 100% rename from BrainPortal/lib/cbrain_task_generators/schemas/boutiques.schema.json rename to BrainPortal/lib/boutiques.schema.json diff --git a/BrainPortal/lib/boutiques_boot_integrator.rb b/BrainPortal/lib/boutiques_boot_integrator.rb index 96ad7c358..02c9e65e5 100644 --- a/BrainPortal/lib/boutiques_boot_integrator.rb +++ b/BrainPortal/lib/boutiques_boot_integrator.rb @@ -34,6 +34,11 @@ class BoutiquesBootIntegrator def self.link_from_json_file(path) descriptor = BoutiquesSupport::BoutiquesDescriptor.new_from_file(path) + self.link_from_descriptor(descriptor) + end + + def self.link_from_descriptor(descriptor) + path = descriptor.from_file.presence || "unknown.json" tool_name = descriptor.name tool_version = descriptor.tool_version myself = RemoteResource.current_resource @@ -42,8 +47,7 @@ def self.link_from_json_file(path) tool = Tool.create_from_descriptor(descriptor) # does nothing if it already exists if tool.cbrain_task_class_name =~ /^CbrainTask::/ basename = Pathname.new(path).basename - puts "B> SKIPPING old integraton of Boutiques JSON: #{basename} Class: #{tool.cbrain_task_class_name}" - return + raise "ERROR: old integraton of Boutiques JSON: #{basename} Class: #{tool.cbrain_task_class_name}" end # Create ToolConfig if necessary diff --git a/BrainPortal/lib/boutiques_support.rb b/BrainPortal/lib/boutiques_support.rb index 534a495a1..75b5d27e2 100644 --- a/BrainPortal/lib/boutiques_support.rb +++ b/BrainPortal/lib/boutiques_support.rb @@ -107,7 +107,7 @@ module BoutiquesSupport Revision_info=CbrainFileRevision[__FILE__] #:nodoc: # Descriptor schema - SCHEMA_FILE = "#{Rails.root.to_s}/lib/cbrain_task_generators/schemas/boutiques.schema.json" + SCHEMA_FILE = "#{Rails.root.to_s}/lib/boutiques.schema.json" # Read schema, store it in the module. @schema = JSON.parse(File.read(SCHEMA_FILE)) diff --git a/BrainPortal/lib/cbrain_task_generators/schema_task_generator.rb b/BrainPortal/lib/cbrain_task_generators/schema_task_generator.rb deleted file mode 100644 index 4c79657c7..000000000 --- a/BrainPortal/lib/cbrain_task_generators/schema_task_generator.rb +++ /dev/null @@ -1,596 +0,0 @@ - -# -# CBRAIN Project -# -# Copyright (C) 2008-2012 -# The Royal Institution for the Advancement of Learning -# McGill University -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -require 'fileutils' - -# This module handles generation of CbrainTasks from schema and descriptor -# files. The generated code can be added right away to CBRAIN's available tasks -# or written to file for later modification. -# -# NOTE: Only JSON and a single schema (boutiques) is currently supported -module SchemaTaskGenerator - - Revision_info=CbrainFileRevision[__FILE__] #:nodoc: - - # Directory where descriptor schemas are located - SCHEMA_DIR = "#{Rails.root.to_s}/lib/cbrain_task_generators/schemas" - - # Default schema file to use when validating auto-loaded descriptors - DEFAULT_SCHEMA_FILE = 'boutiques.schema.json' - - # Represents a schema to validate task descriptors against - class Schema - - # Creates a new Schema from either a file path, a string or a hash - # representing the schema. - def initialize(schema) - @schema = SchemaTaskGenerator.expand_json(schema) - end - - # Validates +descriptor+ against the schema. Returns a list of validation - # errors or nil if +descriptor+ is valid. - def validate(descriptor) - JSON::Validator.fully_validate( - @schema, - SchemaTaskGenerator.expand_json(descriptor), - :errors_as_objects => true - ) - end - - # Same as +validate+, but throws exceptions on validation errors instead. - # Still returns nil if +descriptor+ is valid. - def validate!(descriptor) - JSON::Validator.validate!( - @schema, - SchemaTaskGenerator.expand_json(descriptor), - :errors_as_objects => true - ) - end - - # A Schema essentially behaves like a hash, as to allow accessing schema - # properties. Forwards all other unknown method calls to Hash, if they - # exist. - def method_missing(method, *args) #:nodoc: - if @schema.respond_to?(method) - @schema.send(method, *args) - else - super - end - end - - end - - # Encapsulates a CbrainTask generated by this generator - class GeneratedTask - - # Name of the generated class. Usually a camel-case form - # of descriptor[:name]. - attr_accessor :name - # Descriptor used to generate this task, in hash form. - attr_accessor :descriptor - # Path to the descriptor, if it was provided as a file - attr_accessor :descriptor_path - # Schema instance used to generate this task. - attr_accessor :schema - # Validation errors produced when validating the task's descriptor, if any. - attr_accessor :validation_errors - # Generated Ruby/HTML source for this task. Hash with at least the keys: - # [:portal] BrainPortal side task class Ruby source. Contains a class - # inheriting/implementing PortalTask. - # [:bourreau] Bourreau side task class Ruby source. Contains a class - # inheriting/implementing ClusterTask. - # [:task_params] Ruby ERB form template for the task's params (edit) page. - # [:show_params] Ruby ERB template for the task's show page. - # [:edit_help] Ruby ERB template for the task's parameter help popup. - attr_accessor :source - - # Create a new encapsulated generated CbrainTask from the output of the - # generator (+SchemaTaskGenerator+::+generate+ method). +attributes+ is - # expected to be a hash matching this object's attributes (:source, :name, - # :descriptor, etc.) - def initialize(attributes) - attributes.each do |name, value| - instance_variable_set("@#{name}", value) - end - end - - # Integrates the encapsulated CbrainTask in this CBRAIN installation. - # Unless +register+ is specified to be false, this method will add the - # required Tool if necessary for the CbrainTask to be - # useable right away (since almost all information required to make the - # Tool and ToolConfig objects is available in the spec). - # Also, if +multi_version+ is specified, this method will wrap the - # encapsulated CbrainTask in a version switcher class to allow different - # CbrainTask classes for each tool version. - # Returns the newly generated CbrainTask subclass. - def integrate(register: true, multi_version: false) - # Make sure the task class about to be generated does not already exist, - # to avoid mixing the classes up. - name = SchemaTaskGenerator.classify(@name) - if CbrainTask.const_defined?(name) - Rails.logger.warn( - "WARNING: #{name} is already defined in CbrainTask; " + - "undefining to avoid collisions" - ) unless multi_version - - CbrainTask.send(:remove_const, name) - end - - # As the same code is used to dynamically load tasks descriptors and - # create task templates, the class definitions are generated as strings - # (Otherwise the source wouldn't be available to write down the generated - # templates). This forces the use of eval instead of the much nicer, - # faster and easier to maintain alternatives. :( - app_type = Rails.root.to_s =~ /BrainPortal\z/ ? :portal : :bourreau - rb_source = @source[app_type] - fake_rb_name = "generated/#{@name}/#{app_type}/" + - Pathname.new(@descriptor_path || "internal_#{@name}.json").basename.to_s + ".rb" # .json.rb !!! - eval rb_source, binding, fake_rb_name - - # Try and retrieve the just-generated task class - task = "CbrainTask::#{name}".constantize - - # Since the task class doesn't have a matching cbrain_plugins directory - # tree, some methods need to be added/redefined to ensure the cooperation - # of views and controllers. - generated = self - - # The task class has no public_path. - task.define_singleton_method(:public_path) { |public_file| nil } - - # Make sure the task class still has access to its generated source - task.define_singleton_method(:generated_from) { generated } - - # Offer access to the raw string version of the view partials for use - # in views instead of the cbrain_plugins paths. - task.define_singleton_method(:raw_partial) do |partial| - ({ - :task_params => generated.source[:task_params], - :show_params => generated.source[:show_params], - :edit_help => generated.source[:edit_help] - })[partial] - end - task.class_eval do - define_method(:raw_partial) do |partial| - self.class.raw_partial(partial) - end - end - - # Add a helper method for accessing the help file (for use on the tools page) - # Written by the rake task cbrain:plugins:install, within the subtask public_assets - helpFileName = name + "_help.html" - helpFileDir = File.join( "cbrain_plugins", "cbrain_tasks", "help_files/" ) - task.define_singleton_method(:help_filepath){ File.join(helpFileDir, helpFileName) } - - # If multi-versioning is enabled, replace the task class object constant - # in CbrainTask by a version switcher wrapper class. - if multi_version - # Build the corresponding switcher and add the task's version and class - # to it. - version = @descriptor['tool-version'] - switcher = SchemaTaskGenerator.version_switcher(name) - switcher.known_versions[version] = task - - # Redefine the CbrainTask or Object constant pointing to the task's - # class to point to the switcher instead. - if CbrainTask.const_defined?(name) - CbrainTask.send(:remove_const, name) - CbrainTask.const_set(name, switcher) - end - end - - # With the task class and descriptor, we have enough information to - # generate a Tool and ToolConfig to register the tool into CBRAIN. - register(task) if register - - # For debugging templates, it helps to view the code generated. - # You can simply create an empty directory some place and provide its - # path to the method to_directory() below. - # - # Adjust path here, get the dumps of ALL Boutiques tasks in it. - #to_directory("/tmp/dump") - # - # Adjust path here, if you're interested in a single task's (CbrainTask::AbcdXyz) files. - #to_directory("/tmp/dump") if name == 'AbcdXyz' - # - # Alternatively, you can trigger the dumps using environment variables - # CBRAIN_BOUTIQUES_DUMPDIR="/path/to/dir" - # CBRAIN_BOUTIQUES_DUMPTASK="Abcd" # if not set, all tasks are dumped - if ((dumpdir = ENV['CBRAIN_BOUTIQUES_DUMPDIR']) && dumpdir.present? && File.directory?(dumpdir)) - if ENV['CBRAIN_BOUTIQUES_DUMPTASK'].blank? || name == ENV['CBRAIN_BOUTIQUES_DUMPTASK'] - puts_red "Dumping Boutiques task code for #{name} in #{dumpdir}" - to_directory(dumpdir) - end - end - - task - end - - # Register a newly generated CbrainTask subclass (+task+) in this CBRAIN - # installation, creating the appropriate Tool object from the information - # contained in the descriptor. The newly created Tool and ToolConfig - # will initially belong to the core admin. - def register(task) - name = @descriptor['name'] - version = @descriptor['tool-version'] || '(unknown)' - description = @descriptor['description'] || '' - container_engine = (@descriptor['container-image'] || {})['type'] - container_image = (@descriptor['container-image'] || {})['image'] - container_index = (@descriptor['container-image'] || {})['index'] - resource = RemoteResource.current_resource - - # Create and save a new Tool for the task, unless there's already one. - Tool.new( - :name => name, - :user_id => User.admin.id, - :group_id => User.admin.own_group.id, - :category => "scientific tool", - :cbrain_task_class_name => task.to_s, - :description => description - ).save! unless - Tool.exists?(:cbrain_task_class_name => task.to_s) - - # Create and save a new ToolConfig for the task on this server, unless - # theres already one. Only applies to Bourreaux (as it would make no - # sense on the portal). - return if Rails.root.to_s =~ /BrainPortal\z/ - # Create a ToolConfig iff - # (1) the Bourreau has a container executable and - # (2) the descriptor specifies a container image - return if container_engine.blank? # Singularity or Docker - return if container_image.blank? # Container name or url - container_engine.capitalize! - return if container_engine == "Singularity" && !resource.singularity_present? - return if container_engine == "Docker" && (!resource.docker_present? && !resource.singularity_present?) - - # If Docker engine isn't present use Singularity - container_engine = "Singularity" if (container_engine == "Docker" && !resource.docker_present?) - container_index = 'docker://' if container_index == 'index.docker.io' # old convention - - ToolConfig.new( - :tool_id => task.tool.id, - :bourreau_id => resource.id, - :group_id => User.admin.own_group.id, - :version_name => version, - :description => "#{name} #{version} on #{resource.name}", - :container_engine => container_engine, - :containerhub_image_name => container_image, - :container_index_location => container_index, - ).save! unless - ToolConfig.exists?( - :tool_id => task.tool.id, - :bourreau_id => resource.id, - :version_name => version - ) - end - - # Writes the encapsulated CbrainTask as a directory tree under +path+ under - # the CBRAIN plugin format; - # source[:portal] -> /portal/.rb - # source[:bourreau] -> /bourreau/.rb - # source[:task_params] -> /views/_task_params.html.erb - # source[:show_params] -> /views/_show_params.html.erb - # source[:edit_help] -> /views/public/edit_params_help.html - def to_directory(path) - name = @name.underscore - path = Pathname.new(path.to_s) + name - - FileUtils.mkpath(path) - Dir.chdir(path) do - ['portal', 'bourreau', 'views/public'].each { |d| FileUtils.mkpath(d) } - - File.write("portal/#{name}.rb", @source[:portal]) - File.write("bourreau/#{name}.rb", @source[:bourreau]) - File.write("views/_task_params.html.erb", @source[:task_params]) - File.write("views/_show_params.html.erb", @source[:show_params]) - File.write("views/public/edit_params_help.html", @source[:edit_help]) - end - end - - end - - # Generates a CbrainTask from +descriptorInput+, which is expected to validate - # against +schema+. +schema+ is expected to be either a +Schema+ instance, - # a path to a schema file, the schema in string format or a hash - # representing the schema. - # Similarly to +schema+, +descriptorInput+ is expected to be either a path to a - # descriptor file, the descriptor in string format or a hash representing - # the descriptor. - # By default, the validation of +descriptorInput+ against +schema+ is strict - # and +generate+ will abort at any validation error. Set +strict_validation+ - # to false if you wish for the generator to try and generate the task despite - # validation issues. - # If the +descriptorInput+ is not a path to a file, an alternative +file_for_revision_info+ - # can be provided to be used as the source for the CBRAIN Revision_info constant - # that will be created inside the generated task class. - def self.generate(schema, descriptorInput, strict_validation = true, file_for_revision_info = nil) - descriptor = self.expand_json(descriptorInput) - name = self.classify(descriptor['name']) - schema = Schema.new(schema) unless schema.is_a?(Schema) - errors = [] - - # Find path of descriptor, if it was provided as a file - descriptor_path = nil - [ descriptorInput, file_for_revision_info ].each do |filepath| - descriptor_path ||= filepath.to_s if - (filepath.is_a?(String) || filepath.is_a?(Pathname)) && - filepath.to_s.ends_with?(".json") && - File.file?(filepath.to_s) - end - - # Check for validation errors, but don't let them implode the whole cbrain app - begin - errors = schema.send( - strict_validation ? :'validate!' : :validate, - descriptor - ) || [] - rescue StandardError => e - errors = [e] # Thrown exceptions form the error list in that case - end - - # This lets us skip the task generation if a validation error occurred regardless of whether - # we were in strict or non-strict mode. - if (errors.length rescue 0) > 0 - # Error message - msg = "Encountered JSON schema validation error(s)\n " + - Array(errors).map(&:to_s).join("\n") + "\n\n" + - "WARNING: Boutiques application descriptor #{descriptor['name']} failed validation!\n" + - "\tSkipping task generation. Check " + descriptorInput.to_s + "." - # Log it - Rails.logger.warn( msg ) - puts msg # in rake tasks, we have no Rails.logger - # Return now to skip task generation (prevent catastrophic failure of cbrain) - return - end - - apply_template = lambda do |template| - ERB.new(File.read( - Rails.root.join('lib/cbrain_task_generators/templates', template).to_s - ), nil, '%-').result(binding) - end - - # file_version_path is a full path to any CBRAIN file - # that describe where this task comes from. It is usually - # a JSON file but could also be a Ruby file. The variable is used - # in the portal and bourreau templates below to record - # the version number of the generated code. - file_version_path ||= descriptor_path # the most common situation - file_version_path ||= file_for_revision_info # provided when the task is built from an internal JSON record instead of a file - # Fallbacks: extract the path of the most recent CBRAIN Ruby file in caller stack - file_version_path ||= caller.to_a.detect { |c| c.starts_with? Rails.root.to_s }.try(:sub,/:.*/,"") - file_version_path ||= __file__ # Final fallback: the generator himself - - GeneratedTask.new( - :name => name, - :descriptor => descriptor, - :descriptor_path => descriptor_path, - :schema => schema, - :validation_errors => errors, - :source => { - :portal => apply_template.('portal.rb.erb'), - :bourreau => apply_template.('bourreau.rb.erb'), - :task_params => apply_template.('task_params.html.erb.erb'), - :show_params => apply_template.('show_params.html.erb.erb'), - :edit_help => apply_template.('edit_help.html.erb') - }, - ) - end - - # Generate (or retrieve if it has been generated already) a version switcher - # class for CbrainTask subclasses named +name+. The version switcher class - # will behave just like a blank CbrainTask subclass until it is assigned - # a ToolConfig. It will then replace its methods with the ones from the - # CbrainTask subclass corresponding to that particular version: - # class A < PortalTask - # def f; :a; end - # end - # - # class B < PortalTask - # def f; :b; end - # end - # - # s = version_switcher('A') - # s.known_versions['1.1'] = A - # s.known_versions['1.2'] = B - # - # s.tool_config = ToolConfig.new(:version => '1.1') - # s.f # :a - def self.version_switcher(name) - base = Rails.root.to_s =~ /BrainPortal\z/ ? PortalTask : ClusterTask - @@version_switchers ||= {} - @@version_switchers[name] ||= Class.new(base) do - - # Versions known to this version switcher and their associated CbrainTask - # subclasses. - def self.known_versions - class_variable_set(:@@known_versions, {}) unless - class_variable_defined?(:@@known_versions) - - class_variable_get(:@@known_versions) - end - - # Add a few singleton methods on the object to perform a version switch - # once the tool config is set. - after_initialize do - # FIXME: the simplest and most straightforward way to make the version - # switcher task instance become an instance of the version-specific - # class would be to directly change the instance's class. - # At the time of writing, this is impossible in Ruby. - # This method (+as_version+) tries its best to mimic the missing - # functionality. - - # FIXME: unfortunately, while the technique +as_version+ uses is - # more-or-less sound Ruby-wise (it bulk-imports the version class - # instance methods into the version switcher instance's singleton - # class), it apparently overrides/messes up some sensitive core Ruby - # methods which make Ruby segfault when the object is garbage collected. - - # As such, this version switching functionality is not currently in use, - # for lack of a working technique to try to 'convert' the version - # switcher instance. - - # Convert this blank CbrainTask object (instance of the version switcher - # class) to a more-or-less real instance of the class corresponding to - # +version+ by including all of its methods in, replacing the defaults - # from PortalTask or ClusterTask. - define_singleton_method(:as_version) do |version| - # Try to get the version-specific class corresponding to +version+, - # falling back on the first defined version in known_versions if it - # cannot be found. - known = self.class.known_versions - unless (version_class = known[version]) - cb_error "No known versions for #{self.class.name}!?" unless - known.present? - - logger.warn( - "WARNING: Unknown version #{version} for #{self.class.name}, " + - "using #{known.first[0]} instead." - ) - - version, version_class = known.first - end - - # An object can only be given methods for a single version, and - # exactly once. Conflicts and odd issues could occur otherwise. - # Thus, there is no longer a need for :as_version or the tool_config - # setter hooks. - [ :as_version, :tool_config=, :tool_config_id= ].each do |m| - self.singleton_class.send(:remove_method, m) rescue nil - end - - # Use the Ruby 2.0 refinement API to include version_class methods - # inside this object's singleton class (or metaclass) - self.singleton_class.include(Module.new do - include refine(version_class) { } - end) - - # And try to make the object appear to be a version_class. - define_singleton_method(:class) { version_class } - define_singleton_method(:kind_of?) { |klass| is_a?(klass) } - define_singleton_method(:is_a?) do |klass| - klass <= version_class || super(klass) - end - define_singleton_method(:instance_of?) do |klass| - klass == version_class || super(klass) - end - end - - # If we dont have a tool config already, try to catch the exact moment - # when the version switcher instance gets assigned its tool config and - # invoke as_version when it happens. - if self.tool_config - self.as_version(self.tool_config.version_name) - else - [ :tool_config=, :tool_config_id= ].each do |method| - define_singleton_method(method) do |*args| - value = super(*args) - self.as_version(self.tool_config.version_name) if self.tool_config - value - end - end - end - end - - # Just like generated task classes, the version switcher doesn't have a - # cbrain_plugins directory structure and needs a few methods for views - # and controllers, adjusted to reflect that a ToolConfig is needed to - # access the real task class. - - # No public path - def self.public_path(public_file) - nil - end - - # No generated source (yet) - def self.generated_from - nil - end - - # Stubbed out raw view partials - def self.raw_partial(partial) - ({ - :task_params => %q{ No version specified }, - :show_params => %q{ No version specified } - })[partial] - end - - end - end - - # Returns the default Schema instance to use when validating descriptors - # without a specific schema or when auto-loading descriptors. - # (constructed from DEFAULT_SCHEMA_FILE) - def self.default_schema - @@default_schema ||= Schema.new("#{SCHEMA_DIR}/#{DEFAULT_SCHEMA_FILE}") - end - - # Utility method to convert a JSON string or file path into a hash. - # Returns the hash directly if a hash is given. - def self.expand_json(obj) - return obj unless obj.is_a?(String) - - JSON.parse!(File.exists?(obj) ? File.read(obj) : obj) - end - - # Utility method to convert a string (+str+) to an identifier suitable for a - # Ruby class name. Similar to Rails' classify, but tries to handle more cases. - def self.classify(str) - str.gsub!('-', '_') - str.gsub!(/\W/, '') - str.gsub!(/\A\d/, '') - str.camelize - end - - private - - # Utility/helper methods used in templates. - - # Create a function call formatter for +func+ with possible arguments lists - # +args+. The generated formatter will accept a list of arguments to format - # a call with. (formatter.(['a', 'b']) -> 'func(a, b)') - # If given, +block+ will be used to convert each value in +args+ - # (and the argument passed to the generated function) to an argument list. - # - # Example: - # a = [{ :a => 1, :b => 2 }, { :a => 2, :b => 4 }] - # f = format_call('f', a) { |a| [ a[:a], a[:b] ] } - # f.({ :a => 1, :b => 2}) # gives 'f(1, 2)' - def self.format_call(func, args, &block) - args = args.map { |a| block.(a) } if block - - widths = (args.first rescue []).zip(*args).map do |array| - array.map { |v| v.to_s.length rescue 0 }.max + 1 - end - - lambda do |args| - inner = (block ? block.(args) : args) - .reject(&:blank?) - .each_with_index - .map { |v, i| "%-#{widths[i]}s" % (v.to_s + ',') } - .join(' ') - .gsub(/,\s*\z/, '') - - "#{func}(#{inner})" - end - end - -end diff --git a/BrainPortal/lib/cbrain_task_generators/templates/bourreau.rb.erb b/BrainPortal/lib/cbrain_task_generators/templates/bourreau.rb.erb deleted file mode 100644 index 83f74410d..000000000 --- a/BrainPortal/lib/cbrain_task_generators/templates/bourreau.rb.erb +++ /dev/null @@ -1,748 +0,0 @@ - -# -# CBRAIN Project -# -# Copyright (C) 2008-2012 -# The Royal Institution for the Advancement of Learning -# McGill University -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -# NOTE: This is a working template generated from a descriptor: -# [Schema] <%= schema['id'] %> -# [Schema version] <%= descriptor['schema-version'] %> -# [Tool] <%= descriptor['name'] %> -# [Version] <%= descriptor['tool-version'] || '?' %> -# See the CbrainTask Programmer Guide (CBRAIN Wiki) for a more complete picture -# of how CbrainTasks are constructed. -% # NOTE: This template's weird indentation is there to try and make the -% # generated code as legible as possible. - -# Bourreau-side CbrainTask subclass to launch <%= name %> -class CbrainTask::<%= name %> < <%= (descriptor['custom'] || {})['cbrain:inherits-from-class'] || 'ClusterTask' %> - - Revision_info=CbrainFileRevision["<%= file_version_path %>"] #:nodoc: - - # Descriptor-based tasks are, by default, easily restartable and recoverable - include RestartableTask - include RecoverableTask - -% # Maximum width of a given +key+'s value in a +list+ of hashes -% max_width = lambda do |list, key| -% list.map { |i| i[key].to_s.length rescue 0 }.max -% end -% -% # Parameter groups -% inputs = (descriptor['inputs'].dup || []) rescue [] -% outputs = (descriptor['output-files'].dup || []) rescue [] -% eVars = (descriptor['environment-variables'].dup || []) rescue [] -% -% # Subsets -% files = inputs.select { |i| i['type'] == 'File' } -% required = outputs.select { |o| ! o['optional'] } -% globs = outputs.select { |o| o['list'] } -% in_keys = inputs.select { |i| i['value-key'] } -% out_keys = outputs.select { |o| o['value-key'] } -% flags = (inputs + outputs).select { |p| p['command-line-flag'].present? } -% seps = (inputs + outputs).select { |p| p['command-line-flag-separator'] } -% list_seps = (inputs + outputs).select { |p| p['list-separator'] } -% - # Task properties are special boolean properties of your task, returned as a - # hash table. Used internally by CBRAIN to enable/disable special task - # handling. All properties are unset (false) by default. - # - # Both generated task classes (ClusterTask for the Bourreau, PortalTask for - # the Portal) have different properties as they have different use cases. - # The properties below are for the Bourreau-side class (ClusterTask). - def self.properties #:nodoc: - super.merge({ - # The can_submit_new_tasks property allows a task to submit new tasks. - # To submit a new task, a task has to create a JSON file called - # new-task-*.json at the root of its working directory. Here is an - # example of JSON content for this file: - # { - # "tool-class": "CbrainTask::TestTool", - # "description": "A task running TestTool", - # "parameters": { - # "important_number": "123", - # "dummy_paramet4er": "432" - # } - # } - # The corresponding JSON schema is maintained in the validate_json_string - # method validate_json_string of the BourreauWorker class. - :can_submit_new_tasks => <%= !!(descriptor['custom'] || {})['cbrain:can-submit-new-tasks'] %>, - }) - end - - # Setup the cluster's environment to execute <%= name %>; create the relevant - # directories, prepare symlinks to input files, set environment variables, - # etc. Returns true if the task was correctly set up. - def setup #:nodoc: - - return false unless super - -% unless files.empty? - params = self.params - - # Set results data provider if missing via file inputs - # Check that the file can be resolved as well - isFile = lambda { |p| [<%= files.map { |f| ":'#{f['id']}'" }.join(',') %>].include?(p) } - params.each do |k,v| - if isFile.(k) - ( v.is_a?(Enumerable) ? v : [v] ).each do |t| - f = Userfile.find_by_id(t) - cb_error("Unable to resolve file (id: #{k}, input: #{t})") unless f - self.results_data_provider_id ||= f.data_provider_id rescue nil - end - end - end - - # And make them all available to <%= name %> -% make_available = format_call('multi_make_available_to_dir', files) { |file| [ -% "params[:'#{file['id']}']", -% "'.'", -% "nil", -% "nil", -% "#{file['list'] || false}", -% "'#{file['id']}'", -% ] } -%# Special Case: launching docker tasks from a specific directory (i.e. not the original -%# task directory) breaks the symbolic links generated by make_available. In this case, -%# we alter the location from which the relative path used by the symlink is computed. -% containerImage = descriptor['container-image'] -% launchDir = containerImage['working-directory'] rescue nil -% make_all_available = lambda do |changeBase| -% baseChanged = lambda do |f| -% "multi_make_available_to_dir(" + -% "params[:'#{f['id']}']," + -% "'.'," + -% "nil," + -% "\"#{launchDir}\"," + -% "\"#{f['list'] || false}\"," + -% "'#{f['id']}'" + -% ")" -% end -% files.each do |file| - <%= changeBase ? baseChanged.(file) : make_available.(file) %> -% end -% end -% if launchDir - # If there is a specified container working directory, but docker is not present, - # use make_available to symlink the local file to the location of the file in dp_cache_dir. -%# Note: whether docker is available is not known at templating time, hence the branching - if self.use_docker? -% make_all_available.(true) - # Otherwise, point the symlink to a relative path based off the launch directory - # specified by the boutiques descriptor - else -% make_all_available.(false) - end -% else -%# If no container working directory is specified, simply use make_available directly -% make_all_available.(false) -% end - -% end - true - end - - # If a working directory is specified for use in a container environment, - # return it; else, return nil. - def container_working_directory -% cimg = descriptor['container-image'] || {} -% wd = cimg["working-directory"] - <%= wd.nil? ? "nil" : "\"" + wd + "\"" %> - end - - # The set of shell commands to run on the cluster to execute <%= name %>. - # Any output on stdout or stderr will be captured and logged for information - # or debugging purposes. - # Note that this function also generates the list of output filenames - # in params. - def cluster_commands #:nodoc: - - # If we have a superclass that knows how to generate the commands - # better than we do, we just delegate all the work to it. - if self.class.superclass.respond_to?(:override_cluster_commands) && - self.class.superclass.override_cluster_commands.present? - return super - end - -% shell = descriptor['shell'].presence -% cmdline_template = descriptor['command-line'] -% shell_wrap_start = "#{shell} <<'BoutiquesShellWrapper'" -% shell_wrap_end = "BoutiquesShellWrapper" -% shell_wrap_start=shell_wrap_end="" if shell.nil? || shell =~ /\A(bash|\/bin\/bash|sh|\/bin\/sh)\z/ - # Command template from the descriptor. -% if shell_wrap_end.present? - # The descriptor specified that it is to be run with: <%= shell %> -% end - command_template = <<-'COMMAND_TEMPLATE' -<%= shell_wrap_start %> -<%= cmdline_template %> -<%= shell_wrap_end %> - COMMAND_TEMPLATE - - # Environment variables from Boutiques descriptor -% if eVars.empty? - envVars = [] -% else - envVars = [ -% eVars.each do |v| -%# The value is escaped for bash; the whole cmd is then escaped to be written into the ruby code - <%= ("export %s=%s" % [v['name'], v['value'].bash_escape(true)]).inspect %>, -% end - ] -% end - -% if in_keys.empty? && out_keys.empty? -% unless outputs.empty? - # Output filenames - self.params.merge!({ -% id_width = max_width.(outputs, 'id') + ":''".length -% outputs.each do |output| - <%= - "%-#{id_width}s => %s" % [ - ":'#{output['id']}'", - "'#{output['path-template']}'," - ] - %> -% end - }) - -% end -% if (descriptor['custom'] || {})['cbrain:ignore-exit-status'] - # Command-line - envVars + [ <<-'CMD' ] -% else - # Command-line to run <%= name %> and save its exit status - envVars + [ command_template, "echo $? > ./#{exit_cluster_filename.bash_escape}" ] -% end -% else - params = self.params - - # <%= name %>'s command line and output file names is constructed from a - # set of key-value pairs (keys) which are substituted in the command line - # and output templates. For example, if we have { '[1]' => '5' } for keys, - # a command line such as "foo [1] -e [1]" would turn into "foo 5 -e 5". - -% unless in_keys.empty? - # Substitution keys for input parameters - keys = { -% key_width = max_width.(in_keys, 'value-key') + "'".length -% in_keys.each do |key| - <%= - "'%-#{key_width}s => params[:'_cbrain_preprocessed_%s'] || params[:'%s']," % [ - key['value-key'] + "'", - key['id'], - key['id'] - ] - %> -% end - } - -% end -% unless out_keys.empty? - # Substitution keys for output files -% if in_keys.empty? - keys = { -% else - keys.merge!({ -% end -% -% key_width = max_width.(out_keys, 'value-key') + "''".length -% path_width = max_width.(out_keys, 'path-template') + "'',".length -% out_keys.each do |key| -% stripped = key['path-template-stripped-extensions'] - <%= - "%-#{key_width}s => apply_template(%-#{path_width}s keys%s" % [ - "'#{key['value-key']}'", - "'#{key['path-template']}',", - stripped ? ', strip: [' : '),' - ] - %> -% if stripped -% stripped.each do |ext| - '<%= ext %>', -% end - ]), -% end -% end - }) -% end - -% unless flags.empty? - # Input/output command-line flags used with keys in command-line - # substitution. - flags = { -% key_width = max_width.(flags, 'value-key') + "''".length -% flags.each do |flag| - <%= - "%-#{key_width}s => %s" % [ - "'#{flag['value-key']}'", - "'#{flag['command-line-flag']}'," - ] - %> -% end - } -% end - -% unless seps.empty? - # Input command-line separators (with flags) used with keys in command-line - # substitution. - seps = { -% key_width = max_width.(seps, 'value-key') + "''".length -% seps.each do |sep| - <%= - "%-#{key_width}s => %s" % [ - "'#{sep['value-key']}'", - "'#{sep['command-line-flag-separator']}'," - ] - %> -% end - } -% end - -% unless list_seps.empty? - # Input command-line separators (with list-separator) used with keys in command-line - # substitution. - list_seps = { -% key_width = max_width.(list_seps, 'value-key') + "''".length -% list_seps.each do |list_sep| - <%= - "%-#{key_width}s => %s" % [ - "'#{list_sep['value-key']}'", - "'#{list_sep['list-separator']}'," - ] - %> -% end - } -% end - - outfileMoveCommands = [] -% unless outputs.empty? - # Generate output filenames - params.merge!({ -% id_width = max_width.(outputs, 'id') + ":''".length -% path_width = max_width.(outputs, 'path-template') + "'',".length -% outputs.each do |output| -% stripped = output['path-template-stripped-extensions'] -% strp_str = "strip: [" + stripped.map{ |s| "'#{s}'" }.join(",") + "]" rescue nil - <%= - "%-#{id_width}s => apply_template(%-#{path_width}s keys#{stripped ? ", " + strp_str : ''})," % [ - ":'#{output['id']}'", - "'#{output['path-template']}'," - ] - %> -% end - }) - -%# If we are going to execute in a container, we need to detect output files outside the mounted directory, -%# add commands to mv them there, and then change the file paths to look there when saving results. -% containerImage = descriptor['container-image'] -% if containerImage - # If execution will occur in a container, alter the output file paths to compensate - if self.use_docker? - # Task launch directory (mounted into as pwd if specified) - # Note: assumes container_wd is absolute -% customWd = containerImage['working-directory'] - container_wd = <%= customWd ? "\"" + customWd + "\"" : "self.full_cluster_workdir" %> -% outids = outputs.map { |p| ":'#{p['id']}'" }.join(", ") - is_underneath = lambda { |base,potential| (Pathname.new(potential)).fnmatch?(File.join(base,'**')) } - [<%= outids %>].each do |outid| - # Do not issue move commands for optional output files with empty path templates - next if params[ outid ].nil? or params[ outid ] == '' - # Relative paths are assumed to be with respect to the mounted directory - currentPath = File.absolute_path( params[ outid ], container_wd ) - # If the current path is not in the directory subtree of the mounted directory, - # issue a move command at the end of cluster_commands and alter the path to search - # for output files accordingly - unless is_underneath.( container_wd, currentPath ) - # Move the file to the mounted dir (to be run post-execution, in the container) - # Warning: potential overwrite dangers if there is a same-named local file - newPosition = File.join( container_wd, File.basename(currentPath) ) - outfileMoveCommands << "mv #{currentPath.bash_escape} #{newPosition.bash_escape}" - # Alter where cbrain should look for the file (outside container) - params[ outid ] = File.basename(newPosition) - end - end - end -% end # if containerImage (i.e. output filepath changes for containerized tasks) -% end -% if (descriptor['custom'] || {})['cbrain:ignore-exit-status'] - # Generate the final command-line to run <%= name %> - envVars + [ apply_template( command_template, - keys<%= flags.empty? ? '' : ', flags: flags' %><%= seps.empty? ? '' : ', separators: seps' %><%= list_seps.empty? ? '' : ', list_separators: list_seps' %>) ] + - outfileMoveCommands -% else - # Generate the final command-line to run <%= name %> - command = apply_template( command_template, - keys<%= flags.empty? ? '' : ', flags: flags' %><%= seps.empty? ? '' : ', separators: seps' %><%= list_seps.empty? ? '' : ', list_separators: list_seps' %>) - - # And save its exit status - envVars + [ command, "echo $? > ./#{exit_cluster_filename.bash_escape}" ] + outfileMoveCommands -% end -% end - end # cluster_commands - - # Called after the task is done, this method ensures <%= name %> succeeded and - # saves its output files to the Bourreau's cache before registering them into - # CBRAIN for later retrieval. Returns true on success. - def save_results #:nodoc: - - # If we have a superclass that knows - # how to save results better than we do, we - # just delegate all the work to it. - if self.class.superclass.respond_to?(:override_save_results) && - self.class.superclass.override_save_results.present? - return super - end - -% unless outputs.empty? - # No matter how many errors occur, we need to save as many output - # files as possible and carry the error state to the end. - params = self.params - succeeded = true - -% end -% unless (descriptor['custom'] || {})['cbrain:ignore-exit-status'] - # Make sure <%= name %> completed successfully by checking its exit status - # in +exit_cluster_filename+. - if ! File.exists?(exit_cluster_filename) - self.addlog("Missing exit status file #{exit_cluster_filename}") -% if outputs.empty? - return false -% else - succeeded = false -% end - else # Check exit status file content is a number. - status_file_content = File.read(exit_cluster_filename).strip - if status_file_content.blank? || status_file_content !~ /\A^\d+\z/ - self.addlog("Exit status file #{exit_cluster_filename} has unexpected content") -% if outputs.empty? - return false -% else - succeeded = false -% end - else # Check exit status value - exit_status = status_file_content.to_i - unless SystemExit.new(exit_status).success? - self.addlog("Command failed, exit status #{exit_status}") -% if outputs.empty? - return false -% else - succeeded = false -% end - end # content is success - end # content exists - end # file exists - -% end - # Additional checks to see if <%= name %> succeeded would belong here. - -% if outputs.present? - # Identify the output files parameters from params. - outputs = params.slice(*[ -% outputs.each do |output| - :'<%= output['id'] %>', -% end - ]) - - # Helper for checking if an output is to be ignored outright -% ignoredOutputs = (descriptor['custom'] || {})['cbrain:ignore_outputs'] || [] - isIgnoredOutput = lambda do |f| - [<%= ignoredOutputs.map{ |x| ":'#{x}'" }.join(",") %>].include?( f.to_sym ) - end - -% unless required.empty? - # Make sure that every required output +path+ actually exists - # (or that its +glob+ matches something). - ensure_exists = lambda do |path| - if ! path_is_in_workdir?(path) # this also checks the existence - self.addlog("Output file is missing or outside of task work directory: #{path}") - succeeded = false - return - end - end - ensure_matches = lambda do |glob| - paths = Dir.glob(glob) - if paths.empty? - self.addlog("No output files matching #{glob}") - succeeded = false - end - paths.each do |path| - next if path_is_in_workdir?(path) - self.addlog("Output doesn't exist or is outside of task work directory: #{path}") - succeeded = false - end - end - -% required.select { |o| ! o['list'] }.each do |output| -% next if ignoredOutputs.include?(output['id'].to_s) - ensure_exists.(outputs[:'<%= output['id'] %>']) -% end -% required.select { |o| o['list'] }.each do |output| -% next if ignoredOutputs.include?(output['id'].to_s) - ensure_matches.(outputs[:'<%= output['id'] %>']) -% end - -% end -% unless globs.empty? - # Expand output file globs/patterns inside outputs for output file lists. - [ -% globs.each do |output| - :'<%= output['id'] %>', -% end - ].each do |param| - outputs[param] = Dir.glob(outputs[param]) - end - -% end - # Helper for checking whether outputs are optional -% optionalFiles = outputs.select { |o| o['optional'] } - isOptional = lambda do |f| - [<%= optionalFiles.map{ |x| ":'"+x['id']+"'" }.join(",") %>].include?( f.to_sym ) - end - - # Save (and register) all generated files to the results data provider - outputs.each do |param, paths| - if isIgnoredOutput.(param) - self.addlog("Output parameter '#{param}' is meant to be ignored") - next - end - paths = [paths] unless paths.is_a?(Enumerable) - paths.each do |path| - - # Print a warning if the file is not present - # Note that, since the output may be optional, the process may still be successful - unless path.present? && File.exists?(path) - self.addlog("Unable to find optional output file: #{path}") if isOptional.(param) - succeeded = false if ! isOptional.(param) - next - end - - next unless path_is_in_workdir?(path) # addlog message already added earlier - - # Get name and filetype - self.addlog("Attempting to save result file #{path}") - name = File.basename(path) - userfile_class = Userfile.suggested_file_type(name) - userfile_class ||= ( File.directory?(path) ? FileCollection : SingleFile ) - - # Add a run ID to the file name, to make sure the file doesn't exist. - name.sub!( /(\.\w+(\.gz|\.z|\.bz2|\.zip)?)?\z/i ) { |ext| "-#{self.run_id}" + ext } - - # Save the file (possible overwrite if race condition) - output = safe_userfile_find_or_new(userfile_class, :name => name) - - unless output.save - messages = output.errors.full_messages.join("; ") - self.addlog("Failed to save file #{path} as #{name}") - self.addlog(messages) if messages.present? - succeeded = false - next - end - - output.cache_copy_from_local_file(path) - params["_cbrain_output_#{param}"] ||= [] - params["_cbrain_output_#{param}"] << output.id - self.addlog("Saved result file #{name}") -% if (single_file = files.first if files.count == 1 && ! files.first['list']) - - # As all output files were generated from a single input file, - # the outputs can all be made children of the one parent input file. - parent = Userfile.find_by_id(params[:'<%= single_file['id'] %>']) - output.move_to_child_of(parent) if parent - self.addlog_to_userfiles_these_created_these([parent], [output]) if parent -% else -% input_ids = files.map { |i| i['id'] } # strings, like "minc", "baseline" etc -% if input_ids.present? - - # Add provenance data when multiple input files exist - parent_names = [ <%= input_ids.map { |x| ":'#{x}', " }.join %> ] - parent_ids = parent_names.map { |n| params[n] || params[n.to_s] }.compact - parents = parent_ids.map { |i| Userfile.where(:id => i).first }.compact - self.addlog_to_userfiles_these_created_these(parents, [output]) if parents.present? -% end -% end - end - end -% end - -% # This block saves back inputs back to their provider side -% # if the descriptor specifies that a successful task should do that. -% resync_inputs = (descriptor['custom'] || {})['cbrain:save_back_inputs'] || [] -% if resync_inputs.present? - # Resync inputs back to provider - if succeeded - rsync_input_ids = [ <%= resync_inputs.map { |x| ":'#{x}'" }.join(", ") %> ] - rsync_input_ids.each do |input_id| - userfile_id = params[input_id].presence - next if ! userfile_id - userfile = Userfile.find(userfile_id) - self.addlog "Attempting to save back input '#{userfile.name}' on DataProvider '#{userfile.data_provider.name}'" - userfile.cache_is_newer - userfile.sync_to_provider - self.addlog_to_userfiles_processed(userfile, "(content modified in place)") - end - end - -% end - succeeded - end - -% if descriptor['cbrain:walltime-estimate'] or descriptor['suggested-resources'].try(:[],'walltime-estimate') - # Conservative maximal run time estimate for <%= name %> when submitting a - # job on a cluster. This value should be somewhat larger than the longest - # expected run without being overly excessive; it will be submitted along - # with the job to the cluster management system for scheduling purposes. - def job_walltime_estimate - (<%= (descriptor['cbrain:walltime-estimate']) ? - descriptor['cbrain:walltime-estimate'].to_s : descriptor['suggested-resources'].try(:[], 'walltime-estimate').to_s %>).seconds - end - -% end - # Generic helper methods -% unless (descriptor['custom'] || {})['cbrain:ignore-exit-status'] - - # Filename used to hold the exit status of <%= name %>, computed similarly - # to +*_cluster_filename+. This file is generated as soon as the task is - # completed and is checked in +save_results+ to make sure the task succeeded. - def exit_cluster_filename - ".qsub.exit.#{self.name}.#{self.run_id}" - end -% end - - # Make a given set of userfiles +files+ available to <%= name %> at - # +directory+. Simple variation on +ClusterTask+::+make_available+ - # to allow +files+ to be an Enumerable of files to make available under - # +directory+. - # - # Note that this integration never invokes this method with anything - # in the userfile_sub_path argument (even if make_available() supports it) - # because the descriptor has currently no way to say it is interested in only - # a subpath inside a userfile. - def multi_make_available_to_dir(files, directory, userfile_sub_path = nil, start_dir = nil, isList = false, param_id = nil) - file_access = (self.class.properties[:readonly_input_files].present? || self.tool_config.inputs_readonly) ? :read : :write - files = [files] unless files.is_a?(Enumerable) - files - .map(&:presence) - .compact - .each do |file| # a userfile ID or a userfile object - userfile = Userfile.find(file) # transforms ID into real file object, if needed - - # The most common case - if ! isList || ! userfile.is_a?(CbrainFileList) - make_available(file, directory + '/', userfile_sub_path, start_dir) - next - end - - # When userfile is a CbrainFileList, we need to sync the files described *in it* instead - cbrainfilelist = userfile # just a new name - cbrainfilelist.sync_to_cache - userfile_list = cbrainfilelist.userfiles_accessible_by_user!(user, nil, nil, file_access) - userfile_list.each do |subfile| - make_available(subfile, directory + '/', userfile_sub_path, start_dir) - end - preproc_value_param_name = "_cbrain_preprocessed_#{param_id}".to_sym - params[preproc_value_param_name] = [ '!_cbrain_preprocessed_!' ] + userfile_list.map(&:name).sort - end - end - - # Apply substitution keys +keys+ to +template+ in order to format a - # command-line or output file name. - # Substitute each value in +keys+ in +template+, - # - prepended by the corresponding flag in +flags+ (if available), - # - stripped of the endings in +strip+, - # - separated by the corresponding string in +separators+ for flags - # - separated by the corresponding string in +list_separators+ for list - # Some examples: - # apply_template('f [1]', { '[1]' => 5 }) - # => 'f 5' - # - # apply_template('f [1]', { '[1]' => 5 }, - # flags: { '[1]' => '-z' } - # ) => 'f -z 5' - # - # apply_template('f [1]', { '[1]' => '5.z' }, - # flags: { '[1]' => '-z' }, - # strip: [ '.z' ] - # ) => 'f -z 5' - # - # apply_template('f [1]', { '[1]' => '5' }, - # flags: { '[1]' => '-z' }, - # separators: { '[1]' => '=' } - # ) => 'f -z=5' - # - # apply_template('f [1]', { '[1]' => '[1 2 3]' }, - # flags: { '[1]' => '-z' }, - # list_separators: {'[1]' => "*"} - # ) => 'f -z 1*2*3' - # - def apply_template(template, keys, flags: {}, strip: [], separators: {}, list_separators: {}) - # Set of properties from the descriptor for file-type inputs - fileKeys = [<%= files.map { |f| '"' + (f['value-key'] || 'nil') + '"' }.join(",") %>] - absPath = [<%= files.map { |f| f['uses-absolute-path'] || false }.join(",") %>] - # The working directory from which the task will be launched - # Used to fill in absolute paths for inputs requesting it - wd = self.container_working_directory || self.full_cluster_workdir - - keys.inject(template) do |template, (key, value)| - flag = flags[key] - sep_init = separators[key] - sep_list = list_separators[key] || ' ' - - # Flag type - isBool = (value.is_a?(TrueClass) || value.is_a?(FalseClass)) - next template.gsub(key, (flag && value==true) ? flag : '') if flag && isBool - - value = (value.is_a?(Enumerable) ? value.dup.to_a : [value]).reject(&:nil?) # blanks are allowed - - fetch_userfiles = true - if value[0] == '!_cbrain_preprocessed_!' # special token - value.shift # remove special token - fetch_userfiles = false - end - - value = value.map do |v| - - # Resolve file ids to names - if fileKeys.include?(key) - if fetch_userfiles - currFile = Userfile.find_by_id(v) - cb_error("Unable to find given userfile with id #{v}!") unless currFile - v = currFile.name - end - # If uses-absolute-path is true, convert the filename to a path - if absPath[ fileKeys.index(key) ] - v = File.join( File.absolute_path(wd), v) - end - end - - v = v.dup if v.is_a?(String) - - strip.find do |e| - v.sub!(/#{Regexp.quote(e)}\z/, '') - end if v.is_a?(String) - - v.to_s.bash_escape - end - .join(sep_list) - sep = (sep_init.nil?) ? ' ' : sep_init # Default separator is space - template.gsub(key){ (flag && value.present?) ? "#{flag}#{sep}#{value}" : value } - end - end - -end diff --git a/BrainPortal/lib/cbrain_task_generators/templates/edit_help.html.erb b/BrainPortal/lib/cbrain_task_generators/templates/edit_help.html.erb deleted file mode 100644 index 34681bedf..000000000 --- a/BrainPortal/lib/cbrain_task_generators/templates/edit_help.html.erb +++ /dev/null @@ -1,127 +0,0 @@ - - - - -<%- -# NOTE: This template's weird indentation is there to try and make the -# generated code as legible as possible. --%> - -<%- - # Parameter groups - params = descriptor['inputs'].select { |i| i['type'] != 'File' } - inputs = descriptor['inputs'].select { |i| i['type'] == 'File' } - outputs = descriptor['output-files'].dup --%> -<%# Format a parameter (+param+) attributes as a list element (
  • ) -%> -<%- format_param = lambda do |param| -%> -
  • - - <%= param['name'] %> - <%- if param['command-line-flag']-%> - (<%= param['command-line-flag'] %>) - <%- end -%> - <%- if param['description'] -%> - : - <%= param['description'] %> - <%- else -%> - - <%- end -%> -
    • -<%- end -%> -

    - <%= descriptor['name'] %> -<%- if descriptor['tool-version'] -%> - <%= descriptor['tool-version'] %> -<%- end -%> -

    -
    - -<%- if descriptor['description'] -%> -<%= descriptor['description'] %> -
    - -<%- end -%> -<%- unless params.empty? -%> -

    Parameters

    -
      - <%- params.each { |param| format_param.(param) } -%> -
    - -<%- end -%> -<%- unless inputs.empty? -%> -

    Input files

    -
      - <%- inputs.each { |input| format_param.(input) } -%> -
    - -<%- end -%> -<%- unless outputs.empty? -%> -

    Output files

    -
      - <%- outputs.each { |output| format_param.(output) } -%> -
    -<%- end -%> - -<%- unless inputs.empty? -%> -

    Launching Multiple Tasks

    -<%- single_file = (inputs.length == 1) && (inputs.first['optional'] == false) rescue false -%> -<%- if single_file -%> -

    - Because <%= descriptor['name'] %> has only a single, required file input, all files checked in Cbrain when the task was launched will be placed in this input. - A task will then be launched for each file input this way, with the other parameters remaining constant across the tasks. -

    -<%- end -%> -

    - Multiple instances of <%= descriptor['name'] %> can <%= "also" if single_file %> be launched simultaneously using the Cbrain filelist mechanism ("cbcsv" files). - Essentially, a cbcsv file is a csv file with a list of a user's files registered in Cbrain. - <%- if single_file -%> - Note that if one or more cbcsv files are automatically input to a file parameter (i.e. when there is only a single file input), - each cbcsv will be expanded to its constituent files (i.e. a task will be generated for each row). - <%- end -%> -

    -

    - A cbcsv file can be created in Cbrain by selecting a set of input files from which one wishes to launch a set of tasks, - clicking on the "More" button under the "Files" tab, and clicking "Create a file list". - A cbcsv file will be created and added to your Cbrain fileset, wherein each row represents a file and the first column represents its ID in Cbrain. - Selecting a cbcsv file for a file-type input parameter to the task will result in one copy of the task being launched for each file in the cbcsv, - with each task being given a different input file (from each row) for the parameter with the cbcsv (i.e. in its place). - If multiple cbcsv files in different file inputs are given, they must be the same length, as tasks will be generated by iterating over the rows of all the input cbcsv files simultaneously. - Should one wish to launch all combinations of input file parameters among several lists, the cbcsv files must be constructed explicitly for this. - It is also possible to download, manipulate, and re-upload a cbcsv file (e.g. if one has local ordered lists of files for a specific task already). -

    -

    - Note that rows with IDs of 0 (zero) in the first column are treated as "null" inputs, and no value will be given to the task for that row under that parameter - (this is useful if one wishes to have no input given to a certain parameter for some of the launched tasks). -

    -<%- end -%> - diff --git a/BrainPortal/lib/cbrain_task_generators/templates/portal.rb.erb b/BrainPortal/lib/cbrain_task_generators/templates/portal.rb.erb deleted file mode 100644 index ab6e1bcfc..000000000 --- a/BrainPortal/lib/cbrain_task_generators/templates/portal.rb.erb +++ /dev/null @@ -1,752 +0,0 @@ - -# -# CBRAIN Project -# -# Copyright (C) 2008-2012 -# The Royal Institution for the Advancement of Learning -# McGill University -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -# NOTE: This is a working template generated from a descriptor: -# [Schema] <%= schema['id'] %> -# [Schema version] <%= descriptor['schema-version'] %> -# [Tool] <%= descriptor['name'] %> -# [Version] <%= descriptor['tool-version'] || '?' %> -# See the CbrainTask Programmer Guide (CBRAIN Wiki) for a more complete picture -# of how CbrainTasks are constructed. -% # NOTE: This template's weird indentation is there to try and make the -% # generated code as legible as possible. - -# Portal-side CbrainTask subclass to launch <%= name %> -class CbrainTask::<%= name %> < <%= (descriptor['custom'] || {})['cbrain:inherits-from-class'] || 'PortalTask' %> - - Revision_info=CbrainFileRevision["<%= file_version_path %>"] #:nodoc: - -% # Maximum width of a given +key+'s value in a +list+ of hashes -% max_width = lambda do |list, key| -% list.map { |i| i[key].to_s.length rescue 0 }.max or 0 -% end -% -% # Parameter types -% params = descriptor['inputs'].dup -% outputs = descriptor['output-files'].dup -% required = params.select { |i| ! i['optional'] } -% optional = params.select { |i| i['optional'] } -% defaults = params.select { |i| i['default-value'] } -% files = params.select { |i| i['type'] == 'File' } -% file_lists = files.select { |i| i['list'] } -% -% # Parameter groups -% groups = (descriptor['groups'].dup || []) rescue [] -% noGroups = (groups.length == 0) -% gIdToMbrs = groups.inject({}){ |m,v| m.merge( v["id"] => v["members"] ) } -% gIdToPrms = gIdToMbrs.map { |k,v| [k, params.select{ |p| v.include? p["id"] }] }.to_h -% -% # Parameter Maps: ids => [ target_ids ] for disables/requires -% disables_map = params.inject({}){ |hmap,i| (res=i["disables-inputs"]) ? hmap.merge(i["id"]=>res) : hmap } -% requires_map = params.inject({}){ |hmap,i| (res=i["requires-inputs"]) ? hmap.merge(i["id"]=>res) : hmap } - -% # Special case; we only have one file input parameter and it only -% # allows single files. -% single_file = files.first if files.count == 1 && file_lists.empty? -% if single_file -% # The parameter's validation is made in final_task_list and is no longer -% # optional if it was. -% params.delete(single_file) -% required.delete(single_file) -% optional.delete(single_file) -% end -% - # Task properties are special boolean properties of your task, returned as a - # hash table. Used internally by CBRAIN to enable/disable special task - # handling. All properties are unset (false) by default. - # - # Both generated task classes (ClusterTask for the Bourreau, PortalTask for - # the Portal) have different properties as they have different use cases. - # The properties below are for the Portal-side class (PortalTask). - def self.properties #:nodoc: - super.merge({ - # The task's parameter view doesn't have a submit button, and one should - # be added automatically. Note that the views automatically generated - # along with this template *do* have a submit button. - :no_submit_button => false, - - # Disable the use of presets (saved parameters). If enabled, the preset - # panel is shown above the task's parameter view and allows users to - # save a task's parameters and re-use them later to launch another similar - # task. - :no_presets => false, - - # Allow CBRAIN to parallelize multiple instances of this task/job on a - # single cluster node; each job's generated shell script will be run - # as a background job on the node at the same time and CBRAIN will wait - # for all tasks to be done before moving to the data processing stage. - #:use_parallelizer => false, # use superclass default - - # Indicate that this task may alter its input files, and thus the task's - # owner must have write access to the input files to be allowed to launch - # the task. Most tasks do not alter their input files, but this is a safe - # default. - :readonly_input_files => <%= - if descriptor['custom'] && descriptor['custom'].has_key?('cbrain:readonly-input-files') - !!descriptor['custom']['cbrain:readonly-input-files'] - elsif descriptor['custom'] && descriptor['custom'].has_key?('cbrain:alters-input-files') - !descriptor['custom']['cbrain:alters-input-files'] - else - false - end - %>, - }) - end - - # This determines if the task expects to only read its input files, - # or modify them, and return respectively :read or :write (the default). - # The symbol can be passed to methods such as Userfile.find_accessible_by_user(). - # Depending on the value, more or less files are allowed to be processed. - def file_access - @_file_access ||= (self.class.properties[:readonly_input_files].present? || self.tool_config.try(:inputs_readonly) ? :read : :write) - end - -% unless defaults.empty? - # Default values for some (all?) of <%= name %>'s parameters. Those values - # reflect the defaults taken by the tool's developer; feel free to change - # them to match your platform's requirements. - def self.default_launch_args #:nodoc: - super.merge({ -% id_width = max_width.(defaults, 'id') + "'".length -% defaults.each do |default| - <%= - ":'%-#{id_width}s => %s," % [ - default['id'] + "'", - default['default-value'].inspect - ] - %> -% end - }) - end - -% end - # Callback called just before the task's form is rendered. At this point, - # the task's params hash contains at least the default list of input - # userfiles under the key :interface_userfile_ids. - def before_form #:nodoc: - - # If the superclass returns error messages, - # we don't even bother proceeding with the rest - # here. - super_messages = super - return super_messages if super_messages.present? - -% file_types = descriptor['inputs'] -% .select { |i| ! i['optional'] && i['type'] == 'File' } -% .map { |i| i['cbrain-file-type'] } -% .uniq -% -% input_infos = descriptor['inputs'] -% .select { |i| i['type'] == 'File' } -% .map { |i| -% iname = i['name'] -% ioptional = i['optional'] ? '(optional)' : '(mandatory)' -% itype = i['cbrain-file-type'].presence || '(any)' -% "#{iname} #{ioptional} Type: #{itype}\\n" -% }.join("") -% -% unless file_types.empty? - # Resolve interface_userfile_ids to actual userfile objects - files = Userfile.where(:id => self.params[:interface_userfile_ids]).all.to_a - -% if file_types.length == 1 && !file_types.first - # At least one file is required. - cb_error "Error: this task requires some input files:\n<%= input_infos %>" if files.empty? -% else - # Some input files are not optional and specific file types are - # required. Make sure the given input files are adequate. - - # Ensure that +files+ contains at least one file of type +type+ - ensure_one = lambda do |files, type| - type = type.constantize unless type.is_a?(Class) - cb_error "Error: this task requires at least one file of type '#{type.name}'" unless - files.any? { |f| f.is_a?(type) } - end - -% file_types.compact.each do |type| - ensure_one.(files, '<%= type %>') -% end -% end - -% end - "" - end - - # Callback called just after the task's form has been submitted by the user. - # At this point, all the task's params will be filled. This is where most - # validations happen. - def after_form #:nodoc: - - self.params[:interface_userfile_ids] ||= [] # make sure array is there - - # If the superclass returns error messages, - # we don't even bother proceeding with the rest - # here. - super_messages = super - return super_messages if super_messages.present? - -% unless params.empty? - params = self.params - - # Sanitize every input parameter according to their expected type - -% sanitize_param = format_call('sanitize_param', params) { |param| [ -% ":'#{param['id']}'", -% ":#{param['type'].downcase}", -% (param['cbrain-file-type'] ? ":file_type => '#{param['cbrain-file-type']}'" : nil) -% ] } -% -% unless required.empty? - # Required parameters -% required.each do |param| - <%= sanitize_param.(param) %> -% end - -% end -% unless optional.empty? - # Optional parameters -% calls = optional.map { |param| [ sanitize_param.(param), param ] } -% call_width = calls.map { |c, p| c.length }.max -% calls.each do |call, param| - <%= "%-#{call_width}s unless params[:'%s'].nil?" % [ call, param['id'] ] %> -% end - -% end -%# -%#### -%# Constraint and type checks -%# - # Helpers -% idsym = lambda { |param| ":'#{param['id']}'" } -% nilcheck = lambda { |param| "params[#{idsym.(param)}].nil?" } - list = lambda { |sym| (s = params[sym]).is_a?(Enumerable) ? s : [s] } - - # Helper function for detecting inactive parameters (or false for flag-type parameters) - # Note that empty strings are allowed and no parameter types except flags pass booleans - isInactive = lambda { |x| params[x].nil? || (params[x]==false) } - -%# -%# Enum Check -% has_val_choices = lambda { |p| not p['value-choices'].nil? } -% if params.any?{ |p| has_val_choices.( p ) } - # Check that any enum parameters have been given allowable values - errmsg = "was not given an acceptable value!" -% for param in params -% next unless has_val_choices.( param ) -% if param['type'].downcase == 'number' -% check = "(Array(params[#{idsym.(param)}])" + ' - ' + '[' + param['value-choices'].map{ |s| "#{s}" }.join(',') + ']' + ").empty?" -% else -% check = "(Array(params[#{idsym.(param)}])" + ' - ' + '[' + param['value-choices'].map{ |s| "'#{s}'" }.join(',') + ']' + ").empty?" -% end - params_errors.add(<%=idsym.(param)%>, errmsg) unless <%= nilcheck.(param) %> || <%= check %> -% end -% end # hasEnumVars check - -%# -%# Min/max/int Number check -% if params.any? { |p| p['minimum'] || p['maximum'] || p['integer'] } - # Check that number parameters with contraints have been given permissible values -% nerrmsg = lambda { |prm,str| "\"violates #{prm['exclusive-'+str] ? 'exclusive' : 'inclusive'} #{str} value #{prm[str].to_s}\"" } -% for p in params.select{ |q| q['type'].downcase == 'number'} # since we support arbitrary fields regardless of type -% if p['minimum'] -% ncheck = "list.(#{idsym.(p)}).all? { |v| v.to_f %s #{p["minimum"]} }" % (p['exclusive-minimum'] ? '>' : '>=') - params_errors.add(<%=idsym.(p) + ', ' + nerrmsg.(p,"minimum")%>) unless <%= nilcheck.(p) %> || <%= ncheck %> -% end -% if p['maximum'] -% ncheck = "list.(#{idsym.(p)}).all? { |v| v.to_f %s #{p["maximum"]} }" % (p['exclusive-maximum'] ? '<' : '<=') - params_errors.add(<%=idsym.(p) + ', ' + nerrmsg.(p,"maximum")%>) unless <%= nilcheck.(p) %> || <%= ncheck %> -% end -% if p['integer'] -% ncheck = "( list.(#{idsym.(p)}).all? { |x| Integer(x.to_s) } rescue false )" - params_errors.add(<%=idsym.(p)%>, "must be an integer") unless <%= nilcheck.(p) %> || <%= ncheck %> -% end -% end - -% end # min/max/int checks for numbers -%# -%# List properties check -% if params.any? { |p| p['list'] == true && (p['max-list-entries'] || p['min-list-entries']) } -% maxMap = params.inject({}){ |hm,p| (mle = p['max-list-entries']) ? hm.merge(p['id'] => mle) : hm } -% minMap = params.inject({}){ |hm,p| (mle = p['min-list-entries']) ? hm.merge(p['id'] => mle) : hm } -% lerrmsg = lambda { |m,val| "\"violates #{m} list length requirement (#{val})\"" } -% if maxMap.keys.length > 0 - # Check that max list lengths are not violated -% params.select{ |p| p['list'] == true && p['max-list-entries'] }.each do |p| -% lcheck = "( list.(#{idsym.(p)}).length <= #{maxMap[p['id']]} )" - params_errors.add(<%= idsym.(p) + ', ' + lerrmsg.("max", maxMap[p['id']]) %>) unless <%= nilcheck.(p) %> || <%= lcheck %> -% end - -% end -% if minMap.keys.length > 0 - # Check that min list lengths are not violated -% params.select{ |p| p['list'] == true && p['min-list-entries'] }.each do |p| -% lcheck = "( list.(#{idsym.(p)}).length >= #{minMap[p['id']]} )" - params_errors.add(<%= idsym.(p) + ', ' + lerrmsg.("min", minMap[p['id']]) %>) unless <%= nilcheck.(p) %> || <%= lcheck %> -% end - -% end -% end # min/max list length check -%# -%# -%#### -%# Write the checks for requires-inputs and disables-inputs. -%# Indentation is to preserve pretty output -%# -% # Helper for writing out a dictionary of keys to value arrays to a literal -% writeLiteralMap = lambda do |name,inmap| -% lenLongestKey = inmap.keys.map{ |x| x.to_s.length }.max + "''".length - <%= "%s = {" % name %> -% inmap.each do |key,valArr| -% vals = valArr.map{ |s| ":'%s'" % s }.join(", ") - <%= ":%-#{lenLongestKey}s => [%s]," % [ "'" + key + "'" , vals ] %> -% end - <%= "}" %> -% end -%# -% # Helper for writing out the checker loops -% # Can write a loop checking either disables (base="disable") or requires (base="require") violations -% generateCheckerLoop = lambda do |base| -% key, vals, req = base + 'r', base + 'ds', base == "require" - <%= base + "sMap.each do |"+key+", "+vals+"|" %> - unless isInactive.(<%= key %>) - for <%= base+'d' %> in <%= vals %> - msg = <%='\'is %s \' + %s.pretty_params_names[%s]' % [base + 'd ' + (req ? 'for' : 'by'), name, key] %> - params_errors.add(<%= base+'d' %>, msg) <%= req ? 'if' : 'unless' %> isInactive.(<%= base+'d' %>) - end - end - end -%# -% end -%# -%# Output to template -% unless requires_map.empty? and disables_map.empty? -% comment = "# A Map: id -> [ids] where ids are the parameters %s by the input id" - - <%= (comment % "required") unless requires_map.empty? %> -% writeLiteralMap.("requiresMap",requires_map) unless requires_map.empty? - - <%= (comment % "disabled") unless disables_map.empty? %> -% writeLiteralMap.("disablesMap",disables_map) unless disables_map.empty? - -% end -%# -% unless requires_map.empty? - # Check that requires-inputs is not violated - # If the parameter is filled in, the ones it requires must be too -% generateCheckerLoop.("require") -% end - -% unless disables_map.empty? - # Check that disables-inputs is not violated - # If the parameter is active, the ones it disables must not be -% generateCheckerLoop.("disable") -% end -%# -%#### -%# Check for violations pertaining to parameter groups -%# In particular, check that mutual exclusivity and one-is-required are satisfied -%# -% unless noGroups -%# Check if any groups require checking -% getGroupsWith = lambda { |prop| groups.select{ |g| g[prop] }.map{ |g| ":'"+g["id"]+"'" }.join(", ") } -% mutexGrps = getGroupsWith.("mutually-exclusive") -% oneReqGrps = getGroupsWith.("one-is-required") -% allNoneGrps = getGroupsWith.("all-or-none") -% hasMutex = (mutexGrps != "") -% hasOneReq = (oneReqGrps != "") -% hasAllNone = (allNoneGrps != "") -%# -% if (hasMutex || hasOneReq || hasAllNone) - - # Groups with the mutually exclusive, all-or-none or one-is-required - <%= 'mutexGroups = [%s]' % mutexGrps if hasMutex %> - <%= 'oneReqGroups = [%s]' % oneReqGrps if hasOneReq %> - <%= 'allNoneGroups = [%s]' % allNoneGrps if hasAllNone %> - # Mapping from groupId to members list -% writeLiteralMap.("gidToMbrs", gIdToMbrs) - # Mapping from groupId to group name - grpName = {<%= groups.map{ |g| ":'" + g["id"] + "\' => \'" + g["name"] + "'" }.join(", ") %>} -% end -% if hasMutex - # Lambda for checking mutual exclusivity - isMutex = lambda { |gid| gidToMbrs[gid].select{ |m| ! isInactive.(m.to_sym) }.count <= 1 } - mutexGroups.each do |group| # Check for violations of group mutex properties - errMsg = "violates group mutual exclusivity requirement" - params_errors.add(grpName[group], errMsg) unless isMutex.(group) - end -% end -% if hasOneReq - # Lambda for checking one-is-required (at least one is active) - hasOneActive = lambda { |gid| gidToMbrs[gid].select{ |m| ! isInactive.(m.to_sym) }.count > 0 } - oneReqGroups.each do |group| # Check for violations of group one-is-required properties - errMsg = "violates group one-is-required specification" - params_errors.add(grpName[group], errMsg) unless hasOneActive.(group) - end -% end -% if hasAllNone - # Lambda for checking all-or-none (either none or all are active) - hasAllNone = lambda { |gid| gidToMbrs[gid].select{ |m| ! isInactive.(m.to_sym) }.count == 0 || - gidToMbrs[gid].select{ |m| isInactive.(m.to_sym) }.count == 0} - allNoneGroups.each do |group| # Check for violations of group all-or-none properties - errMsg = "violates group all-or-none specification" - params_errors.add(grpName[group], errMsg) unless hasAllNone.(group) - end -% end -% end # noGroups check -%# -%# End parameter group properties check -%# -%# - ### Perform validation checks on cbcsv files, if any are present ### - - # Checks that the cbcsv is the correct type - # Current implementation will output an error here if a person uploads a cbcsv - # but forgets to change its type to cbcsv. I.e. we assume it is an error to use - # a .cbcsv for anything except generating a CbrainFileList object. - checkCbcsvType = lambda do |f,id| - isCbcsv = f.is_a?(CbrainFileList) - msg = " is not of type CbrainFileList (file #{f.name})! Please convert it with the file manager. (Type: #{f.class})" - params_errors.add(id, msg) unless isCbcsv - isCbcsv - end - - # Check that the user can access the cbcsv files - ascertainUserAccess = lambda do |f,id| - # Error message when a file cannot be found (e.g. non-existent id) - msg1 = lambda { |i| " - unable to find file with id #{i} in cbcsv #{f.name}. Ensure you own all the given files." } - # Error message when an exception is thrown - msg2 = lambda { |e| " cbcsv accessibility error in #{f.name}! Possibly due to cbcsv malformation. (Received error: #{e.inspect})" } - errFlag = true # Whether the error checking found a problem - begin # Check that the user has access to all of the files in the cbcsv - fs = f.userfiles_accessible_by_user!(self.user,nil,nil,file_access) - for i in f.ordered_raw_ids.select{ |r| (! r.nil?) && (r.to_s != '0') } - accessible = ! ( Userfile.find_accessible_by_user( i, self.user, :access_requested => file_access ) rescue nil ).nil? - params_errors.add( id, msg1.(i) ) unless accessible - errFlag = false unless accessible - end - rescue => e # Catches errors from userfiles_accessible_by_user - params_errors.add( id, msg2.(e) ) - errFlag = false - end - errFlag - end - - # Check that the validation of the other columns goes through - validateCols = lambda do |cbcsv,id| - # Error-check the remainder of the file with max_errors = 1 and non-strict (so zero rows can have anything in them) - allGood = cbcsv.validate_extra_attributes(self.user, 1, false, file_access) rescue false # returns true if no errors - allGood ||= cbcsv.errors # If there were errors, we want to look at them - params_errors.add(id, "has attributes (in cbcsv: #{cbcsv.name}) that are invalid (Received error: #{allGood.messages})") unless (allGood == true) - allGood - end - -% if single_file # Special case if there is only one file type input (to which all the userfiles are assigned) - # Get cbcsvs (note: we get files that end with cbcsv, but may not be of that class; the user is warned when this occurs, i.e. after_form fails) - files = self.params[:interface_userfile_ids].map do |f| - begin - Userfile.find_accessible_by_user( f, self.user, :access_requested => file_access ) - rescue => e - params_errors.add(<%= ":'#{single_file['id']}'" %>, "encountered an error trying to find file #{f}. Ensure the file exists and you can access it.") - return "" - end - end - cbcsvs = files.select(&:presence).map do |f| - Userfile.find_accessible_by_user( f, self.user, :access_requested => file_access ) - end.select do |f| - f.is_a?(CbrainFileList) || (f.suggested_file_type || Object) <= CbrainFileList - end - # Validate accessibility - for cbcsv in cbcsvs - # Get the id of the single_file input - id = <%= ":'#{single_file['id']}'" %> - # Also check that files ending in cbcsv are actually so - next unless checkCbcsvType.(cbcsv, id) - # Ensure user has access to the cbcsv subfiles - next unless ascertainUserAccess.(cbcsv, id) - # Validate the other columns of the file - validateCols.(cbcsv, id) - end -% else # general case: each cbcsv is in a different input - # Get all the input cbcsv files - cbcsvs = self.cbcsv_files - # If a cbcsv file is present, generate a task for each entry - # Note they should have been validate in after_form - if (cbcsvs || []).length > 0 - numRows = nil # Keep track of number of files per cbcsv - # Validate each cbcsv (all columns match per row, user has access to the file) - for id, cbcsv in cbcsvs - # Error if the type is wrong - next unless checkCbcsvType.(cbcsv, id) - # Ensure user access is correct - next unless ascertainUserAccess.(cbcsv, id) - # If the number of rows does not match, error - currNumRows = (cbcsv.ordered_raw_ids || []).length - numRows = numRows.nil? ? currNumRows : numRows - params_errors.add(id, " does not have the same number of files (#{currNumRows}) as in other present cbcsvs (#{numRows})") unless (currNumRows == numRows) - next unless (currNumRows == numRows) - # Validate the other file columns - validateCols.(cbcsv, id) - end - end -% end -%# End cbcsv validations - -% end # End unless params.empty? - "" - end - - # Add pretty parameter names for the error messages to use - # Associates the id symbol with the name field - def self.pretty_params_names - super.merge({ -% id_width = max_width.(params,"id") + "''".length -% for param in params - <%= ":%-#{id_width}s => '%s'," % ["'" + param["id"] + "'", param["name"]] %> -% end - }) - end - - # Returns all the cbcsv files present (i.e. set by the user as inputs), as tuples (id, Userfile) - def cbcsv_files - <%= "files = [%s]" % files.map { |f| ":'#{f['id']}'" }.join( ', ' ) %> - <%= "file_lists = [%s]" % file_lists.map { |f| ":'#{f['id']}'" }.join( ', ' ) %> - return [] if files.nil? || files.length == 0 - files.select { |f| self.params[f].present? && ! file_lists.include?(f) } # Prevent problems with file-type inputs with list=true - .map { |f| [f, Userfile.find_accessible_by_user(self.params[f], self.user, :access_requested => file_access)] } - .select { |f| f[1].is_a?(CbrainFileList) || (f[1].suggested_file_type || Object) <= CbrainFileList } - end - - # Final set of tasks to be launched based on this task's parameters. Only - # useful if the parameters set for this task represent a set of tasks - # instead of just one. - def final_task_list #:nodoc: - - # If we have a superclass that informs us that it knows - # how to generate a task list better than we do, we - # just delegate all the work to it. - if self.class.superclass.respond_to?(:override_final_task_list) && - self.class.superclass.override_final_task_list.present? - return super - end - -% if single_file - # Create a list of tasks out of the default input file list - # (interface_userfile_ids), each file going into parameter '<%= single_file['id'] %>' - tasklist = self.params[:interface_userfile_ids].map do |id| - task = self.dup - # Helper for filling in the changing task parameters - fillTask = lambda do |id,tsk| - tsk.params[:'<%= single_file['id']%>'] = id -% if single_file['cbrain-file-type'] - tsk.sanitize_param(:'<%= single_file['id'] %>', :file, :file_type => '<%= single_file['cbrain-file-type']%>') -% else - tsk.sanitize_param(:'<%= single_file['id'] %>', :file) -% end - tsk.description ||= '' - tsk.description += " <%= single_file['id']%>: #{Userfile.find(id).name}" - tsk.description.strip! - tsk - end - # Expand cbcsvs and generate tasks from them - f = Userfile.find_accessible_by_user( id, self.user, :access_requested => file_access ) - if f.is_a?( CbrainFileList ) - ufiles = f.userfiles_accessible_by_user!( self.user, nil, nil, file_access ) - # Skip files that are purposefully nil (e.g. given id 0 by the user) - tasks = ufiles.select { |u| ! u.nil? }.map{ |a| fillTask.( a.id, task.dup ) } - # Set and sanitize the one file parameter for each id for regular files - else - fillTask.( id, task ) - end - end.flatten - return tasklist -% else - # Grab all the cbcsv input files - cbcsvs = self.cbcsv_files - # Default case: just return self as a single task - tasklist = [ self ] - # If one or more cbcsv files is present, generate a task for each entry - # Note they should have been validated in after_form - if (cbcsvs || []).length > 0 - # Array with the actual userfiles corresponding to the cbcsv - mapCbcsvToUserfiles = cbcsvs.map { |f| f[1].ordered_raw_ids.map { |i| (i==0) ? nil : i } } - # Task list to fill and total number of tasks to output - tasklist, nTasks = [], mapCbcsvToUserfiles[0].length - # Iterate over each task that needs to be generated - for i in 0..(nTasks - 1) - # Clone this task - currTask = self.dup - # Replace each cbcsv with an entry - cbcsvs.map{ |f| f[0] }.each_with_index do |id,j| - currId = mapCbcsvToUserfiles[j][i] - #currTask.params[:interface_userfile_ids] << mapCbcsvToUserfiles unless currId.nil? - currTask.params[id] = currId # If id = 0 or nil, currId = nil - currTask.params.delete(id) if currId.nil? - end - # Add the new task to our tasklist - tasklist << currTask - end - end - # Return the final set of tasks - return tasklist -% end - end - - # Task parameters to leave untouched by the edit task mechanism. Usually - # for parameters added in after_form or final_task_list, as those wouldn't - # be present on the form and thus lost when the task is edited. - def untouchable_params_attributes #:nodoc: -% if outputs.empty? - super || { } -% else - # Output parameters will be present after the task has run and need to be - # preserved. - super.merge({ -% id_width = max_width.(outputs, 'id') + "'".length -% outputs.each do |output| - <%= ":'_cbrain_output_%-#{id_width}s => true," % (output['id'] + "'") %> -% end - }) -% end - end - - ######################## - # Generic Zenodo Support - ######################## - - # We only provide a minimal amount of base information; - # The user can fill in the details later. - def base_zenodo_deposit #:nodoc: - ZenodoClient::Deposit.new( - :metadata => ZenodoClient::DepositMetadata.new( - :title => "Outputs of #{self.pretty_name}-#{self.id}", - :description => ("Files and meta data for CBRAIN task #{self.pretty_name}@#{self.bname_tid}" + - "\n\n#{self.description}").strip, - ) - ) - end - - def zenodo_outputfile_ids #:nodoc: - params - .keys - .grep(/^_cbrain_output_/) - .select { |k| params[k].is_a?(Array) || params[k].to_s =~ /\A\d+\z/ } - .inject([]) { |union,k| union += Array(params[k]); union } - .compact - .sort - .uniq - end - - # Generic helper methods - - # Ensure that the parameter +name+ is not null and matches a generic tool - # parameter +type+ (:file, :numeric, :string or :flag) before converting the - # parameter's value to the corresponding Ruby type (if appropriate). - # For example, sanitize_param(:deviation, :numeric) would validate that - # self.params[:deviation] is a number and then convert it to a Ruby Float or - # Integer. - # - # Available +options+: - # [file_type] Userfile type to validate a parameter of +type+ :file against. - # - # If the parameter's value is an array, every value in the array is checked - # and expected to match +type+. - # - # Raises an exception for task parameter +name+ if the parameter's value - # is not adequate. - def sanitize_param(name, type, options = {}) - - # Taken userfile names. An error will be raised if two input files have the - # same name. - @taken_files ||= Set.new - - # Fetch the parameter and convert to an Enumerable if required - values = self.params[name] rescue nil - values = [values] unless values.is_a?(Enumerable) - - # Validate and convert each value - values.map! do |value| - case type - # Try to convert to integer and then float. Cant? then its not a number. - when :number - if (number = Integer(value) rescue Float(value) rescue nil) - value = number - elsif value.blank? - params_errors.add(name, ": value missing") - else - params_errors.add(name, ": not a number (#{value})") - end - - # Nothing special required for strings, bar for symbols being acceptable strings. - when :string - value = value.to_s if value.is_a?(Symbol) - params_errors.add(name, " not a string (#{value})") unless value.is_a?(String) - params_errors.add(name, " is blank") if value.blank? - # The following two checks are to prevent cases when - # a string param is used as a path - params_errors.add(name, " cannot contain newlines") if value.to_s =~ /[\n\r]/ - params_errors.add(name, " cannot start with these characters") if value.to_s =~ /^[\.\/]+/ - - # Try to match against various common representation of true and false - when :flag - if value.is_a?(String) - value = true if value =~ /\A(true|t|yes|y|on|1)\z/i - value = false if value =~ /\A(false|f|no|n|off|0|)\z/i - end - - if ! [ true, false ].include?(value) - params_errors.add(name, ": not true or false (#{value})") - end - - # Make sure the file ID is valid, accessible, not already used and - # of the correct type. - when :file - unless (id = Integer(value) rescue nil) - params_errors.add(name, ": invalid or missing userfile") - next value - end - - unless (file = Userfile.find_accessible_by_user(value, self.user, :access_requested => file_access) rescue nil) - params_errors.add(name, ": cannot find userfile (ID #{value})") - next value - end - - if @taken_files.include?(file.name) - params_errors.add(name, ": file name already in use (#{file.name})") - else - @taken_files.add(file.name) - end - - if type = options[:file_type] - type = type.constantize unless type.is_a?(Class) - params_errors.add(name, ": incorrect userfile type (#{file.name})") if - type && ! file.is_a?(type) - end - end - - value - end - - # Store the value back - self.params[name] = values.first unless self.params[name].is_a?(Enumerable) - end - -end diff --git a/BrainPortal/lib/cbrain_task_generators/templates/show_params.html.erb.erb b/BrainPortal/lib/cbrain_task_generators/templates/show_params.html.erb.erb deleted file mode 100644 index 5f641c8c1..000000000 --- a/BrainPortal/lib/cbrain_task_generators/templates/show_params.html.erb.erb +++ /dev/null @@ -1,120 +0,0 @@ - -<%%- -# -# CBRAIN Project -# -# Copyright (C) 2008-2012 -# The Royal Institution for the Advancement of Learning -# McGill University -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# --%> - -<%%- -# NOTE: This is a working template generated from a descriptor: -# [Schema] <%= schema['id'] %> -# [Schema version] <%= descriptor['schema-version'] %> -# [Tool] <%= descriptor['name'] %> -# [Version] <%= descriptor['tool-version'] || '?' %> -# See the CbrainTask Programmer Guide (CBRAIN Wiki) for a more complete picture -# of how CbrainTasks are constructed. --%> -<%- -# NOTE: This template's weird indentation is there to try and make the -# generated code as legible as possible. --%> - -<%- - # Parameter groups - params = descriptor['inputs'].select { |i| i['type'] != 'File' } - inputs = descriptor['inputs'].select { |i| i['type'] == 'File' } - outputs = descriptor['output-files'].dup --%> -<%- unless [params, inputs, outputs].all?(&:empty?) -%> -<%% - # Format a parameter (input or output) +value+ for display. Mostly for - # arrays (parameter lists). - format_param = lambda do |value| - value = [value] unless value.is_a?(Enumerable) - value.map(&:to_s).join(', ') - end - - # Format a set of one or more Userfile IDs for display. - # Similar to format_param. - format_files = lambda do |value| - value = [value] unless value.is_a?(Enumerable) - format_param.(value.map { |v| link_to_userfile_if_accessible(v) }).html_safe - end -%%> -<%- end -%> -
    -<%- if [params, inputs, outputs].all?(&:empty?) -%> - No parameters, no inputs, no ouputs. -<%- else -%> - - <%- unless params.empty? -%> - - - - - <%- params.each do |param| -%> - - - - - <%- end -%> - - <%- end -%> - <%- unless inputs.empty? -%> - - - - - <%- inputs.each do |input| -%> - - - - - <%- end -%> - - <%- end -%> - <%- unless outputs.empty? -%> - - - - - <%- outputs.each do |output| -%> - - - - - <%- end -%> - - <%- end -%> -
    - Parameters -
    <%= param['name'] %> - <%%= format_param.(params[:'<%= param['id'] %>']) %> -
    - Input files -
    <%= input['name'] %> - <%%= format_files.(params[:'<%= input['id'] %>']) %> -
    - Output files -
    <%= output['name'] %> - <%%= format_files.(params[:'<%= "_cbrain_output_" + output['id'].to_s %>']) %> -
    -<%- end -%> -
    diff --git a/BrainPortal/lib/cbrain_task_generators/templates/task_params.html.erb.erb b/BrainPortal/lib/cbrain_task_generators/templates/task_params.html.erb.erb deleted file mode 100644 index 3dbb6353e..000000000 --- a/BrainPortal/lib/cbrain_task_generators/templates/task_params.html.erb.erb +++ /dev/null @@ -1,1184 +0,0 @@ - -<%%- -# -# CBRAIN Project -# -# Copyright (C) 2008-2012 -# The Royal Institution for the Advancement of Learning -# McGill University -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# --%> - -<%% content_for :head do %> - <%%= stylesheet_link_tag 'boutiques', :media => "all" %> -<%% end %> - -<%%- -# NOTE: This is a working template generated from a descriptor: -# [Schema] <%= schema['id'] %> -# [Schema version] <%= descriptor['schema-version'] %> -# [Tool] <%= descriptor['name'] %> -# [Version] <%= descriptor['tool-version'] || '?' %> -# See the CbrainTask Programmer Guide (CBRAIN Wiki) for a more complete picture -# of how CbrainTasks are constructed. --%> -<%- -# NOTE: This template's weird indentation is there to try and make the -# generated code as legible as possible. --%> - -<%- - # Parameter types - params = descriptor['inputs'].dup - required = params.select { |i| ! i['optional'] } - optional = params.select { |i| i['optional'] } - defaults = params.select { |i| i['default-value'] } - - # Parameter groups and maps - groups = (descriptor['groups'].dup || []) rescue [] - noGroups = (groups.length == 0) - gIdToMbrs = groups.inject({}){ |m,v| m.merge( v["id"] => v["members"] ) } - gIdToPrms = gIdToMbrs.map { |k,v| [k, params.select{ |p| v.include? p["id"] }] }.to_h - - # Single input file case - files = params.select { |i| i['type'] == 'File' } - single_file = files.first if files.count == 1 && ! files.any? { |f| f['list'] } --%> -<%% input_files = Userfile.where(:id => params[:interface_userfile_ids]).all.to_a rescue [] %> - -<%% - # Handling for multi-task generation via CBCSVs - cbcsvs = input_files.select { |f| f.is_a?( CbrainFileList ) } - cbcsvToFiles = cbcsvs.inject({}) do |m,f| # Only used for preview button - f.sync_to_cache - m.merge(f.name => f.ordered_raw_ids.map{ |r| (r==0 || r.nil?) ? ' ' : # Space can't be a filename in cbrain - (Userfile.find_all_accessible_by_user(current_user).where('userfiles.id' => r).first rescue nil) } - ) - end - single_file = <%= single_file ? "\"#{single_file['id']}\"" : "nil" %> -%%> - -<%- - # Lambda for checking group membership - whichGroup = lambda { |s| gIdToMbrs.select{ |v| gIdToMbrs[v].include? s }.first[0] rescue nil } --%> - -<%%# - Generate a parameter label for HTML input with id +id+. - +name+ corresponds to the tool parameter name to display, - +optional+ indicates that the parameter is optional (or not) and - +flag+ is the parameter's command-line flag if available. -%%> -<%% label = lambda do |id, name, optional: false, flag: nil| %> - -<%% end %> - -<%% paragraph = lambda do | text | %> - - <%%= text %> - -<%% end %> - -<%%# - Generate a generic parameter input field with HTML id +id+ and name +name+. - +type+ is the kind of input field to generate (text or hidden), - +value+ is the input field's initial value, - +optional+ indicates that the parameter is optional (and the name should not - directly be placed on the tag) and - +placeholder+ is the placeholder text to fill the input with while awaiting - user input -%%> -<%% input = lambda do |id, name, type: 'text', value: nil, optional: false, placeholder: nil| %> - <%% value = @task.params[name.to_sym] if name && value.nil? %> - - id="<%%= id.to_la_id %>" - <%% end %> - class="tsk-prm-in" - type="<%%= type %>" - <%% if name %> - <%%= optional ? 'data-name' : 'name' %>="<%%= name.to_la %>" - <%% end %> - <%% if value %> - value="<%%= value %>" - <%% end %> - <%% if placeholder %> - placeholder="<%%= placeholder %>" - <%% end %> - /> -<%% end %> - -<%%# - Generate a list of inputs suitable for a list parameter with HTML id +id+ - and name +name+. +values+, +optional+ and +placeholder+ work similarly to - input's corresponding arguments. -%%> -<%% input_list = lambda do |id, name, values: nil, optional: false, placeholder: nil| %> - <%% - values = @task.params[name.to_sym] if name && values.nil? - values = values.is_a?(Enumerable) ? values.compact : [values].compact - first = values.shift - %> -
      -
    • - <%% - input.(id, name + "[]", - value: first, - optional: optional, - placeholder: placeholder - ) - %> - -
      • - <%% values.each do |value| %> -
    • - <%% - input.(nil, name + "[]", - value: value, - optional: optional, - placeholder: placeholder - ) - %> - -
      • - <%% end %> -
    -<%% end %> - -<%%# - Generate a fancy checkbox for flag or optional parameters with HTML +id+. - If +name+ is given, a checkbox suitable for flag parameters is generated - while one for optional arguments is generated otherwhise. The checkbox's - initial state (checked or not) is +check+. -%%> -<%% checkbox = lambda do |id, name: nil, check: nil| %> - <%%# - FIXME for some obscure reason, naming the 'check' parameter 'checked' - makes the first parameter (id) become nil regardless of the value passed - in. This only seems to occur in the context of Rails' template renderer. - %> - <%% - id = name ? id.to_la_id : "tsk-prm-opt-#{id}" - type = name ? 'chk' : 'opt' - %> - <%% if name %> - <%% if check.nil? %> - <%% check = @task.params[name.to_sym].presence %> - <%% check = false if check == "0" %> - <%% end %> - <%%# - As a checkbox's value is not sent if left unchecked, a hidden 'back' field - sends the unchecked state is used to send. Rails overwrites this 'back' - value with the checbox's when the checkbox is checked, as the checkbox is - after the 'back' field. - %> - - - checked="checked" - <%% end %> - /> - <%% else %> - - checked="checked" - <%% end %> - /> - <%% end %> - -<%% end %> - -<%%# - Generate a drop-down list for a set of +options+ (value, label pairs) with - HTML id +id+ and name +name+. - +nothing+ corresponds to the text displayed if +options+ is empty and - +optional+ indicates if the parameter is optional (or not) -%%> -<%% dropdown = lambda do |id, name, options = [], nothing: '(Nothing to select)', optional: false, default: nil| %> - <%% value = @task.params[name.to_sym] if name %> - <%% - input.(id, name, - type: 'hidden', - value: value, - optional: optional - ) - %> -
    - <%% if options.empty? %> - - <%%= nothing %> - - <%% else %> - - - <%% if (pair = options.detect { |o| o.first == value }) %> - <%%= pair.last %> - <%% end %> - -
      - <%% options.each do |value, label| %> - <%% if default.nil? %> -
    • <%%= label %>
      • - <%% else %> - <%% dval = (default==value) ? " data-selected=\"true\"" : "" %> -
    • ><%%= label %>
      • - <%% end %> - <%% end %> -
    - <%% end %> -
    -<%% end %> - -<%%# - Generate a drop-down list for a set of +options+ (value, label pairs) with - HTML id +id+ and name +name+. - +nothing+ corresponds to the text displayed if +options+ is empty and - +optional+ indicates if the parameter is optional (or not) -%%> -<%% html_select = lambda do |id, name, options = [], nothing: '(Nothing to select)', optional: false, default: nil| %> - <%% value = @task.params[name.to_sym] if name %> -
    - <%% if options.empty? %> - - <%%= nothing %> - - <%% else %> - - <%% end %> -
    -<%% end %> - - -<%%# Generate a checkbox for a parameter grouping %> -<%% groupCheckbox = lambda do |id, defaultChecked = false| %> - - /> - -<%% end %> - -<%%# Generate a parameter's description block for +desc+ %> -<%% description = lambda do |desc, isGroupMember = false| %> - - <%%= desc.split("\n").map { |s| h(s) }.join("
    ").html_safe if desc %><%%= "

    ".html_safe if isGroupMember %> -     -<%% end %> - -<%%# Give a short (up to +max+ characters) representation of +list+ %> -<%% - short_repr = lambda do |list, max| - str = list.map(&:to_s).join(', ') - str.length <= max ? str : (str[0, max - 3] + '...') - end -%%> - -<%# Generate a complete HTML block for a given parameter +param+ -%> -<%- parameter = lambda do |param, isGroupMember = false| -%> - <%- - param['optional'] = false if param['optional'].nil? - id = param['id'] - type = param['type'].downcase.to_sym - opt = !!(param['optional'] && type != :flag) - list = param['list'] - list = false if type == :file # override: we use CbrainFileLists - -%> - <%- - classes = [ 'tsk-prm', type.to_s ] - classes << 'list' if list - classes << 'prm-grp-mbr' if isGroupMember - -%> - <%% id = '<%= id %>' %> -
  • - <%% - <%- if type == :flag -%> - # Flag input toggle - checkbox.(id, name: id) - - <%- end -%> - # Name/Label - label.(id, %q{ <%= param['name'] %> }, - <%- if param['command-line-flag'] -%> - optional: <%= param['optional'] %>, - flag: '<%= param['command-line-flag'] %>' - <%- else -%> - optional: <%= param['optional'] %> - <%- end -%> - ) - - <%- if opt -%> - # Optional parameter enable/disable toggle - checkbox.(id) - - <%- end -%> - - <%- enum_vals = param["value-choices"] %> - <%- if not enum_vals.nil? -%> - <%- list = param["list"] -%> - <%- if list == true -%> - # Enum value html_select - name = id <%= list ? '+ "[]"' : '' %> - html_select.(id, name, - <%= enum_vals.map { |v| [v, v] } %>, - optional: <%= opt %>, - default: <%= "\"#{defaults.find { |d| d['id']==id }['default-value']}\"" rescue "nil" %> - ) - <%- else -%> - # Enum value dropdown - name = id <%= list ? '+ "[]"' : '' %> - dropdown.(id, name, - <%= enum_vals.map { |v| [v, v] } %>, - optional: <%= opt %>, - default: <%= "\"#{defaults.find { |d| d['id']==id }['default-value']}\"" rescue "nil" %> - ) - <%- end -%> - <%- elsif type == :file -%> - <%- if single_file -%> - paragraph.( - "One or multiple tasks will be launched based on your selected files." - ) - <%- elsif param['cbrain-file-type'] -%> - # File input dropdown - type = '<%= param['cbrain-file-type'] %>'.constantize rescue nil - dropdown.(id, id, - input_files - .select { |f| f.is_a?(type) if type } - .map { |f| [ f.id.to_s, f.name ] }, - optional: <%= opt %> - ) - <%- else -%> - # File input dropdown - name = id <%= list ? '+ "[]"' : '' %> - dropdown.(id, name, - input_files.map { |f| [ f.id.to_s, f.name ] }, - optional: <%= opt %> - ) - <%- end -%> - - <%- elsif [ :string, :number ].include?(type) -%> - <%- - arg_width = [ - 'optional', - ('placeholder' if type == :number), - ].compact.map(&:length).max + ':'.length - -%> - <%- if list -%> - # Input field list - input_list.(id, id, - <%- else -%> - # Input field - input.(id, id, - <%- end -%> - <%- if type == :number -%> - optional: <%= opt.to_s %>, - placeholder: '0.0' - <%- else -%> - optional: <%= opt.to_s %> - <%- end -%> - ) - <%- end -%> - <%- if param['description'] -%> - # Description - description.(<<-'DESC',<%= isGroupMember %>) - <%= param['description'] %> - DESC - <%- elsif not param['description'] and isGroupMember %> - # Blank Line for empty description (fixes spacing) - description.('',<%= isGroupMember %>) - <%- end -%> - %> -
    • -<%- end -%> -<%# Generate HTML for a group -%> -<%- writeGroup = lambda do |group| -%> - <%- - # Get group parameters - id = group['id'] - name = group['name'] - groupDescription = group['description'].presence || '' - groupParams = gIdToPrms[id] - mbrs = gIdToMbrs[id] - defaultChecked = group['one-is-required'] || required.any? { |p| mbrs.include? p["id"] } - -%> -
    - <%# Write group header %> -
    -

    - <%= name %> <%% groupCheckbox.('<%= id %>',<%= defaultChecked %>) %> -

    -
    - <%- groupDescription.split("\n").each do |line| %> - <%= line %>
    - <%- end %> -
    -
    -
    - <%# Write group parameters (required ones first) %> - <%- groupParams.each { |p| parameter.(p, true) if required.any? { |r| p["id"]==r["id"] } } -%> - <%- groupParams.each { |p| parameter.(p, true) if optional.any? { |t| p["id"]==t["id"] } } -%> -
    - -<%- end -%> -
    - <%- if params.empty? -%> - <%= name %> has no parameters. - <%- else -%> -
      -% # Write info for ungrouped parameters - <%- for param in required do parameter.(param) if whichGroup.(param["id"]).nil? end -%> - <%- for param in optional do parameter.(param) if whichGroup.(param["id"]).nil? end -%> -% # Write info for grouped parameters - <%- groups.each(&writeGroup) unless noGroups -%> -
    - <%- end -%> -
    - -
    -

    - -

    -
    - - - diff --git a/BrainPortal/lib/tasks/cbrain_plugins.rake b/BrainPortal/lib/tasks/cbrain_plugins.rake index d1b99e306..e92719221 100644 --- a/BrainPortal/lib/tasks/cbrain_plugins.rake +++ b/BrainPortal/lib/tasks/cbrain_plugins.rake @@ -16,7 +16,6 @@ namespace :cbrain do userfiles_plugins_dir = installed_plugins_dir + "userfiles" views_plugins_dir = installed_plugins_dir + "views" tasks_plugins_dir = installed_plugins_dir + "cbrain_task" - descriptors_plugins_dir = installed_plugins_dir + "cbrain_task_descriptors" boutiques_plugins_dir = installed_plugins_dir + "boutiques_descriptors" lib_plugins_dir = installed_plugins_dir + "lib" @@ -157,18 +156,6 @@ namespace :cbrain do end ) - # Setup each cbrain_task descriptor plugin - erase_dead_symlinks.('descriptor', descriptors_plugins_dir) - setup.('cbrain_task_descriptors/*', 'descriptor', descriptors_plugins_dir, - condition: lambda { |f| File.extname(f) == '.json' }, - after: lambda do |symlink_location| - dest=symlink_location.to_s.sub(/.json$/, '.rb') - if ! File.symlink?(dest) - File.symlink "cbrain_task_descriptor_loader.rb", dest - end - end - ) - # Setup each boutiques descriptor plugin (new integrator) erase_dead_symlinks.('boutiques', boutiques_plugins_dir) setup.('boutiques_descriptors/*', 'boutiques', boutiques_plugins_dir, @@ -259,33 +246,6 @@ namespace :cbrain do end end - # Generate help files for Boutiques tasks - # Note: changes here should be synced with SchemaTaskGenerator if necessary - Rake::Task["environment"].invoke - - Dir.chdir(descriptors_plugins_dir) do - helpFileDir = File.join( "cbrain_plugins", "cbrain_tasks", "help_files/" ) - basePath = Rails.root.join( File.join('public/', helpFileDir) ) - FileUtils.mkdir_p( basePath.to_s ) # creates directory if needed - schema = SchemaTaskGenerator.default_schema # read in the Boutiques schema - # For each JSON decriptor of a tool, write a help file - Dir.glob("*").select { |f| f.end_with? '.json' }.each do |f| - absFile = File.absolute_path(f.to_s) - # Generate the task from the templates and JSON descriptor - generatedTask = SchemaTaskGenerator.generate(schema, absFile) - next if generatedTask.nil? # this happens with bad descriptors - helpFileName = SchemaTaskGenerator.classify(generatedTask.name) + "_help.html" - helpFilePath = basePath.join(helpFileName).to_s - # Prevent broken symlinks from stopping the whole rake task - next unless File.exist?(File.realpath( absFile )) - # Write the help file - File.open( helpFilePath , "w" ){ |h| - h.write( generatedTask.source[:edit_help] ) - } - FileUtils.chmod(0775, helpFilePath) - end - end - end # task :public_assets end # namespace install @@ -327,7 +287,6 @@ namespace :cbrain do erase.('userfile', userfiles_plugins_dir) erase.('views', views_plugins_dir) erase.('task', tasks_plugins_dir) - erase.('descriptor', descriptors_plugins_dir) erase.('boutiques', boutiques_plugins_dir) erase.('lib', lib_plugins_dir) diff --git a/BrainPortal/spec/boutiques/boutiques_tester_spec.rb b/BrainPortal/spec/boutiques/boutiques_tester_spec.rb index 1eb4e96ac..eabc931b2 100644 --- a/BrainPortal/spec/boutiques/boutiques_tester_spec.rb +++ b/BrainPortal/spec/boutiques/boutiques_tester_spec.rb @@ -63,7 +63,7 @@ }) @dp.save! # Lambda for constructing cbcsv files - @makeCbcsv = -> (fs, name, task, mangler=nil, addToIuids=true, user: @user, group: @group, dp: @dp) { + @makeCbcsv = -> (fs, name, task, mangler=nil, addToUids=true, user: @user, group: @group, dp: @dp) { flist = CbrainFileList.new( :user => user, :user_id => user.id, @@ -75,18 +75,21 @@ text = CbrainFileList.create_csv_file_from_userfiles( fs ) text = mangler.( text ) unless mangler.nil? flist.cache_writehandle { |t| t.write( text ) } - task.params[:interface_userfile_ids] << flist.id if addToIuids + task.params[:interface_userfile_ids] ||= [] if addToUids + task.params[:interface_userfile_ids] |= [ flist.id ] if addToUids flist # return the cbcsv object } # Helper for generating and sending userfiles to tasks @addUserFile = -> (name, task, addToUids=true, user: @user, group: @group, dp: @dp) { uf = SingleFile.new({data_provider_id: dp.id, name: name, group_id: group.id, user_id: user.id}) uf.save! - task.params[:interface_userfile_ids] << uf.id if addToUids + task.params[:interface_userfile_ids] ||= [] if addToUids + task.params[:interface_userfile_ids] |= [ uf.id ] if addToUids uf } # Checks after_form output. Checks both the number or errors and ascertains at least one of their contents. @checkAfterForm = -> (task, checkVal=0, atLeastOneErrWith=nil, runBeforeForm=false) { + task.params_errors.clear task.before_form if runBeforeForm task.after_form # Run the method errMsgs = task.params_errors.full_messages # Get any error messages @@ -117,7 +120,7 @@ # Validate correctness of JSON descriptor describe "JSON descriptor" do it "validates" do - schemaLoc = File.join(SchemaTaskGenerator::SCHEMA_DIR, SchemaTaskGenerator::DEFAULT_SCHEMA_FILE) + schemaLoc = File.join(Rails.root, "lib/boutiques.schema.json") expect( runAndCheckJsonValidator(schemaLoc) ).to eql(true) end end @@ -152,15 +155,21 @@ before(:each) do # Create environment execer = FactoryBot.create(:bourreau) - schema = SchemaTaskGenerator.default_schema - descriptor = File.join(__dir__, TestScriptDescriptor) - @boutiquesTask = SchemaTaskGenerator.generate(schema, descriptor) - @boutiquesTask.integrate if File.exists?(descriptor) + schema = Rails.root + "lib/boutiques.schema.json" + desc_path = File.join(__dir__, TestScriptDescriptor) + @descriptor = BoutiquesSupport::BoutiquesDescriptor.new_from_file(desc_path) + + tool = Tool.create_from_descriptor(@descriptor) + tool_config = ToolConfig.create_from_descriptor(execer, tool, @descriptor) + if ! BoutiquesTask.const_defined?(:BoutiquesTest) + BoutiquesBootIntegrator.link_from_json_file(desc_path) rescue nil + end + # Instantiate a task object - @task = CbrainTask::BoutiquesTest.new + @task = BoutiquesTask::BoutiquesTest.new @task.bourreau = execer @task.tool_config = FactoryBot.create(:tool_config) - @task.user_id, @task.group_id, @task.params = @user.id, @group.id, {} + @task.user_id, @task.group_id, @task.params = @user.id, @group.id, {}.with_indifferent_access # Setup for holding the files the user had selected in the UI @task.params[:interface_userfile_ids] = [] # Create userfiles for C, d, j, f (used to convert the ids from strings to numbers) @@ -168,29 +177,19 @@ @file_f1, @file_f2 = @addUserFile.('f1',@task), @addUserFile.('f2',@task) # Helper for converting files in the argument dict to int ids @replaceFileIds = -> (replaceF=false) { - @task.params[:C] = @file_C.id unless @task.params[:C].nil? - @task.params[:d] = @file_d.id unless @task.params[:d].nil? - @task.params[:j] = @file_j.id unless @task.params[:j].nil? - @task.params[:f] = [ @file_f1.id, @file_f2.id ] if replaceF + @task.invoke_params[:C] = @file_C.id unless @task.invoke_params[:C].nil? + @task.invoke_params[:d] = @file_d.id unless @task.invoke_params[:d].nil? + @task.invoke_params[:j] = @file_j.id unless @task.invoke_params[:j].nil? + @task.invoke_params[:f] = [ @file_f1.id, @file_f2.id ] if replaceF } # Give access to the class version of the task - @task_const = "CbrainTask::#{SchemaTaskGenerator.classify(@task.name)}".constantize - end - - # Test the object generated by the Boutiques generator - context "Boutiques GeneratedTask Object" do - it "should have the right name" do - expect( @boutiquesTask.name ).to eq( "BoutiquesTest" ) - end - it "should have no validation errors" do - expect( @boutiquesTask.validation_errors ).to be true - end + @task_const = BoutiquesTask::BoutiquesTest end # Test the portal class automatically generated and registered in cbrain via the GeneratedTask Object context "Boutiques Generated Class Properties" do it "should have the right task class name" do - expect( @task_const.to_s ).to eq( "CbrainTask::BoutiquesTest" ) + expect( @task_const.to_s ).to eq( "BoutiquesTask::BoutiquesTest" ) end it "should have a tool" do expect( Tool.exists?(:cbrain_task_class_name => @task_const.to_s) ).to be true @@ -198,21 +197,11 @@ it "should have no public path" do # Just test the help file expect( @task_const.public_path("edit_params_help.html") ).to eq( nil ) end - it "should have access to its generated source object" do - expect( @task_const.generated_from ).to eq( @boutiquesTask ) - end - it "has all raw partials" do - expect( @task_const.raw_partial(:task_params) ).not_to eq( nil ) - expect( @task_const.raw_partial(:show_params) ).not_to eq( nil ) - expect( @task_const.raw_partial(:edit_help) ).not_to eq( nil ) - end it "has pretty param names" do - allThere = TestArgs.all? { |s| @task_const.pretty_params_names[s] == s.to_s } + @task_const.add_pretty_params_names(@descriptor.inputs || []) + allThere = TestArgs.all? { |s| @task_const.pretty_params_names[@descriptor.input_by_id(s).cb_invoke_name] == s.to_s } expect( allThere ).to be true end - it "has expected default values" do # Only -r has a default value - expect( @task_const.default_launch_args[:'r'] ).to eq( 'r' ) - end end # Test an object instantiated from the portal class generated by the Boutiques framework @@ -232,10 +221,10 @@ # Here, UserFile existence is merely simulated, and before_form is tested in isolation describe "has a before_form method that" do before(:each) do - @task.params = {} + @task.params = {}.with_indifferent_access end it "should fail when no input files are given" do - expect { @task.before_form }.to raise_error(CbrainError, /Error: this task requires/) + expect { @task.before_form }.to raise_error(CbrainError, /This task requires/i) end end @@ -254,12 +243,12 @@ it "after_form #{t[0]}" do @task.params_errors.clear # Reset to having no errors begin # Parse the input command line - @task.params = ArgumentDictionary.( t[1].dup ) + @task.params[:invoke] = ArgumentDictionary.( t[1].dup ).with_indifferent_access rescue OptionParser::MissingArgument => e next # after_form does not need to check this, since rails puts a value in the hash end - hasFileListFilled = ! @task.params[:f].nil? # Whether the file list parameter is in use - @task.params[:f] ||= [] # after_form expects [], not nil, for empty file lists + hasFileListFilled = ! @task.invoke_params[:f].nil? # Whether the file list parameter is in use + @task.invoke_params[:f] ||= [] # after_form expects [], not nil, for empty file lists @replaceFileIds.( hasFileListFilled ) # replace the file paths with IDs @task.after_form # Run the method errMsgs = @task.params_errors.full_messages @@ -267,9 +256,17 @@ errMsgs.delete_if { |m| ignoredMsgs.any? { |e| m.include?(e) } } # When there is an error, the exit code should be non-zero; no errors should be present otherwise expect( - (errMsgs.length == 0 && t[2] == 0) || (errMsgs.length > 0 && t[2] != 0) + if (errMsgs.length == 0 && t[2] == 0) || (errMsgs.length > 0 && t[2] != 0) + true + else + # Uncomment this to debug a particularly tricky situation + #puts_yellow "Got #{errMsgs.length} error messages and expected #{t[2]}" + #puts_yellow "Messages are:\n#{errMsgs.join("\n")}" if errMsgs.length > 0 + #puts_yellow "Params: #{@task.params.inspect}" + false + end ).to be true - @task.params = {} # Clean up; @task is shared between tests + @task.params = {}.with_indifferent_access # Clean up; @task is shared between tests end # it block end # all? block end # describe block generated after_form method @@ -280,16 +277,15 @@ # Setup the environment with several userfiles and cbcsv files before(:each) do # Fill in the minimal required arguments for the class (but save the mock UI chosen files) - temp = @task.params[:interface_userfile_ids] - @task.params = ArgumentDictionary.( MinArgs ) - @task.params[:interface_userfile_ids] = temp + @task.params_errors.clear + @task.params[:invoke] = ArgumentDictionary.( MinArgs ).with_indifferent_access # Create some user files @userfiles = (0..9).map { |i| @addUserFile.("f-#{i}",@task) } # Create some cbcsvs @std1, @std2 = @makeCbcsv.(@userfiles[0..3],"std2.cbcsv",@task), @makeCbcsv.(@userfiles[4..7],"std1.cbcsv",@task) # File input parameters - @task.params[:C] = @file_C.id # Replace as above, since it is a required argument - @task.params[:f] ||= [] # after_form expects [], not nil, for empty file lists + @task.invoke_params[:C] = @file_C.id # Replace as above, since it is a required argument + @task.invoke_params[:f] ||= [] # after_form expects [], not nil, for empty file lists end # Clean up after each test by removing the cbcsvs we saved (includes destroying them on the data provider) @@ -300,25 +296,25 @@ # Test the after_form error checking for multi-task launching describe "in after_form" do it "with one cbcsv file" do - @task.params[:d] = @std1.id # single cbcsv + @task.invoke_params[:d] = @std1.id # single cbcsv @checkAfterForm.( @task ) end it "with more than one cbcsv files" do - @task.params[:d], @task.params[:j] = @std1.id, @std2.id + @task.invoke_params[:d], @task.invoke_params[:j] = @std1.id, @std2.id @checkAfterForm.( @task ) end it "with a cbcsv that does not have the cbcsv extension" do - @task.params[:d] = @makeCbcsv.(@userfiles[0..3],"misname.m",@task).id + @task.invoke_params[:d] = @makeCbcsv.(@userfiles[0..3],"misname.m",@task).id @checkAfterForm.( @task ) end it "with a cbcsv with nil entries" do nilEntries = @makeCbcsv.(@userfiles[3..6],"hasNils.cbcsv",@task,@nilRowAdder) - @task.params[:d] = nilEntries.id + @task.invoke_params[:d] = nilEntries.id @checkAfterForm.( @task ) end it "to detect errors when lengths don't match" do smaller = @makeCbcsv.(@userfiles[8..9], "small.cbcsv", @task) - @task.params[:d], @task.params[:j] = @std1.id, smaller.id + @task.invoke_params[:d], @task.invoke_params[:j] = @std1.id, smaller.id @checkAfterForm.( @task, 1, "number of files" ) end it "to detect errors when a file does not exist" do @@ -329,7 +325,7 @@ v.join( CbrainFileList::FIELD_SEPARATOR ) } ) - @task.params[:d] = noFile.id + @task.invoke_params[:d] = noFile.id @checkAfterForm.( @task, 1, "unable to find file" ) end it "to detect errors when a file is inaccessible" do @@ -338,13 +334,13 @@ file2 = @addUserFile.("file2.c", @task, user: user2, group: grp2) # Put the file in a cbcsv and check after_form cbcsvTest = @makeCbcsv.([@userfiles[0],file2,@userfiles[1]],"cbcsvWithOthersFiles.cbcsv", @task) - @task.params[:d] = cbcsvTest.id + @task.invoke_params[:d] = cbcsvTest.id # Make sure after_form catches the problem @checkAfterForm.( @task, 1, "unable to find file" ) end # This assumes the user made a mistake, e.g. forgot to convert the file, in this case it "to detect errors when a file is not a cbcsv but has the cbcsv extension" do - @task.params[:d] = @addUserFile.('fake.cbcsv',@task).id + @task.invoke_params[:d] = @addUserFile.('fake.cbcsv',@task).id @checkAfterForm.( @task, 1, "not of type" ) end it "to detect errors when a row has invalid attributes" do @@ -355,7 +351,7 @@ v.join( CbrainFileList::FIELD_SEPARATOR ) } ) - @task.params[:d] = invalidFile.id + @task.invoke_params[:d] = invalidFile.id @checkAfterForm.( @task, 1, "are invalid" ) # Two errors: as the misnamed file is missing and the row is invalid end end @@ -365,38 +361,27 @@ describe "in final_task_list" do # Normal case (no cbcsv files) it "with no cbcsvs" do - @task.params[:d] = @userfiles[0].id + @task.invoke_params[:C] = @userfiles[0].id expect( @task.final_task_list.length ).to eq( 1 ) end # Standard single cbcsv case it "with a single cbcsv" do - @task.params[:d] = @std1.id + @task.invoke_params[:C] = @std1.id expect( @task.final_task_list.length ).to eq( 4 ) end - # Try with multiple cbcsvs - it "with multiple cbcsvs" do - # First, test that the number of tasks is good - @task.params[:d], @task.params[:j] = @std1.id, @std2.id #0-3, 4-7 - taskList = @task.final_task_list - expect( taskList.length ).to eq( 4 ) - # Second, test that the parameters of those tasks are good (i.e. correct userfiles in the right places) - taskList.each_with_index do |task, i| - expect( task.params[:d] ).to eq( @userfiles[i].id ) # cbcsv expansion of std1 - expect( task.params[:j] ).to eq( @userfiles[i+4].id ) # cbcsv expansion of std2 - end - end # The presence of null entries should give tasks with empty parameters when reached it "with nil entries in cbcsvs" do c1 = @makeCbcsv.(@userfiles[3..6],"hasNils.cbcsv",@task,@nilRowAdder) - c2 = @makeCbcsv.(@userfiles[1..5],"noNils.cbcsv",@task) - @task.params[:d], @task.params[:j] = c1.id, c2.id + #c2 = @makeCbcsv.(@userfiles[1..5],"noNils.cbcsv",@task) + @task.invoke_params[:C] = c1.id + #@task.invoke_params[:j] = c2.id taskList = @task.final_task_list # Should be 5 tasks in total (the nil row should count) - expect( taskList.length ).to eq( 5 ) + expect( taskList.length ).to eq( 4 ) # Should be nothing for d when it's the nil row's turn taskList.each_with_index do |task, i| - expect( task.params[:d] ).to eq( (i==4) ? nil : @userfiles[i+3].id ) - expect( task.params[:j] ).to eq( @userfiles[i+1].id ) + expect( task.invoke_params[:C] ).to eq( @userfiles[i+3].id ) + #expect( task.invoke_params[:j] ).to eq( @userfiles[i+1].id ) end end end @@ -412,23 +397,35 @@ # Run before block to create Minimal task, added to by specific tests before(:each) do # Generate a descriptor - @descriptor = NewMinimalTask.() + @descriptor = BoutiquesSupport::BoutiquesDescriptor.new(NewMinimalTask.()) + execer = FactoryBot.create(:bourreau) + tool = Tool.create_from_descriptor(@descriptor) + tool_config = ToolConfig.create_from_descriptor(execer, tool, @descriptor) + BoutiquesTask.const_set(:MinimalTest, Class.new(BoutiquesPortalTask)) if ! BoutiquesTask.const_defined?(:MinimalTest) + ToolConfig.register_descriptor(@descriptor, tool.name, tool_config.version_name) rescue nil + # Generates a task object from the minimal mock app - @generateTask = -> params { + key = [ tool.name, tool_config.version_name ] + @generateTask = ->(params,reset_desc = nil) do useDefaults = (params.is_a? String) && (params == 'defaults') - genTask = SchemaTaskGenerator.generate(SchemaTaskGenerator.default_schema, @descriptor, false).integrate - task = CbrainTask::MinimalTest.new - task.params = useDefaults ? task.class.default_launch_args : params + task = BoutiquesTask::MinimalTest.new(:tool_config_id => tool_config.id, :bourreau_id => execer.id) + + # reset_desc is a kludge + if reset_desc + ToolConfig.instance_eval { + @_descriptors_[key] = reset_desc + } + end + + task.params = useDefaults ? task.default_launch_args : { :invoke => params.with_indifferent_access } task - } + end end # Test the object generated by the Boutiques generator context "Boutiques GeneratedTask Object" do it "should have the right names" do - genTask = SchemaTaskGenerator.generate(SchemaTaskGenerator.default_schema, @descriptor, false).integrate - expect( (CbrainTask::MinimalTest.new).name ).to eq( "MinimalTest" ) # Check for task instance - expect( genTask.name ).to eq( "CbrainTask::MinimalTest" ) # Check for generated task class instance + expect( (BoutiquesTask::MinimalTest.new).name ).to eq( "MinimalTest" ) # Check for task instance end end @@ -440,10 +437,10 @@ @descriptor['command-line'] += '[F] ' @descriptor['inputs'] << GenerateJsonInputDefault.("f", 'File', 'File arg') # Instantiate a task object from the descriptor - @task = @generateTask.( 'defaults' ) + @task = @generateTask.( 'defaults', @descriptor ) # Add metadata to the task @task.bourreau = FactoryBot.create(:bourreau) - @task.user_id, @task.group_id, @task.params = @user.id, @group.id, {} + @task.user_id, @task.group_id, @task.params = @user.id, @group.id, {}.with_indifferent_access @task.params[:interface_userfile_ids] = [] @task.tool_config = FactoryBot.create(:tool_config) # Generate some userfiles for testing @@ -454,30 +451,28 @@ Userfile.all.select { |f| f.is_a?(CbrainFileList) }.each { |uf| uf.destroy } end describe "has after_form that" do - it "works without any cbcsvs" do - @addUserFile.('t.txt',@task); @addUserFile.('r.txt',@task) - @checkAfterForm.( @task, 0, nil, true ) - end it "works with one cbcsv" do - @makeCbcsv.([@f1,@f2], 'test.cbcsv', @task) - @checkAfterForm.( @task, 0, nil, true ) - end - it "works with two cbcsvs" do - @makeCbcsv.([@f1,@f2], 't1.cbcsv', @task); @makeCbcsv.([@f1,@f3], 't2.cbcsv', @task) - @checkAfterForm.( @task, 0, nil, true ) - end - it "works with two cbcsvs of different lengths" do - @makeCbcsv.([@f1,@f2], 't1.cbcsv', @task); @makeCbcsv.([@f1,@f2,@f3], 't2.cbcsv', @task) + cb1 = @makeCbcsv.([@f1,@f2], 'test.cbcsv', @task) + @task.invoke_params['f'] = cb1.id @checkAfterForm.( @task, 0, nil, true ) end + #it "works with two cbcsvs" do + # @makeCbcsv.([@f1,@f2], 't1.cbcsv', @task); @makeCbcsv.([@f1,@f3], 't2.cbcsv', @task) + # @checkAfterForm.( @task, 0, nil, true ) + #end + #it "works with two cbcsvs of different lengths" do + # @makeCbcsv.([@f1,@f2], 't1.cbcsv', @task); @makeCbcsv.([@f1,@f2,@f3], 't2.cbcsv', @task) + # @checkAfterForm.( @task, 0, nil, true ) + #end it "fails when a subfile is non-existent" do - @makeCbcsv.( [@f1,@f2,@f3], 'missing.cbcsv', @task, + cbl = @makeCbcsv.( [@f1,@f2,@f3], 'missing.cbcsv', @task, -> (text) { # Lambda for mangling the input text so the first number becomes invalid (choose max + 1) v = text.split( CbrainFileList::FIELD_SEPARATOR ) - v[0] = Userfile.all.map { |f| f.id }.max + 1 + v[0] = Userfile.pluck(:id).max + 1 v.join( CbrainFileList::FIELD_SEPARATOR ) } ) + @task.invoke_params['f'] = cbl.id @checkAfterForm.( @task, 1, "unable to find file", true ) end it "fails gracefully when a file is inaccessible" do @@ -485,7 +480,8 @@ user2, grp2 = FactoryBot.create( :user ), FactoryBot.create( :group ) file2 = @addUserFile.("f2.tex", @task, user: user2, group: grp2) # Make sure after_form catches the problem - @checkAfterForm.( @task, 1, "trying to find file", true ) + @task.invoke_params['f'] = file2.id + @checkAfterForm.( @task, 1, "cannot find userfile", true ) end it "fails gracefully when a cbcsv subfile is inaccessible" do # Create a new user and file for him/her @@ -494,6 +490,7 @@ # Put the file in a cbcsv and check after_form (only add the prohibited file to a cbcsv) cbcsvTest = @makeCbcsv.( [@f1,file2,@f3], "cbcsvWithOthersFiles.cbcsv", @task) # Make sure after_form catches the problem + @task.invoke_params['f'] = cbcsvTest.id @checkAfterForm.( @task, 1, "unable to find file", true ) end it "fails when there are non-matching attributes" do @@ -505,6 +502,7 @@ v.join( CbrainFileList::FIELD_SEPARATOR ) } ) + @task.invoke_params['f'] = invalidFile.id @checkAfterForm.( @task, 1, "are invalid" ) end end @@ -541,19 +539,19 @@ end it "should work a regular type" do @descriptor['inputs'] << GenerateJsonInputDefault.('b','Number','A number arg',{'default-value' => 9}) - task = @generateTask.( 'defaults' ) + task = @generateTask.( 'defaults', @descriptor ) task.before_form @checkAfterForm.( task ) end it "should work with appropriate enums" do @descriptor['inputs'] << GenerateJsonInputDefault.('b','String','An enum arg',{'value-choices' => ['a','b','c'], 'default-value' => 'b'}) - task = @generateTask.( 'defaults' ) + task = @generateTask.( 'defaults', @descriptor ) task.before_form @checkAfterForm.( task ) end it "should fail with an inappropriate enum value" do @descriptor['inputs'] << GenerateJsonInputDefault.('b','String','An enum arg',{'value-choices' => ['a','b','c'], 'default-value' => 'd'}) - task = @generateTask.( 'defaults' ) + task = @generateTask.( 'defaults', @descriptor ) task.before_form @checkAfterForm.( task, 1, "acceptable value" ) # Should give an error relating to the enum having an unacceptable value end @@ -569,22 +567,22 @@ end it "is satisfied when neither is present" do - task = @generateTask.( { a: 'val' , b: '1'} ) + task = @generateTask.( { a: 'val' , b: '1'}, @descriptor ) @checkAfterForm.( task ) end it "is satisfied when only groups are present" do - @descriptor['groups'] = [{'id' => 'G', 'name' => 'G', 'members' => ['a','b'], 'mutually-exclusive' => true}] - task = @generateTask.( { a: 'val1' } ) + @descriptor['groups'] = [BoutiquesSupport::Group.new({'id' => 'G', 'name' => 'G', 'members' => ['a','b'], 'mutually-exclusive' => true})] + task = @generateTask.( { a: 'val1' }, @descriptor ) @checkAfterForm.( task ) end it "is satisfied when only disables is present" do @descriptor['inputs'][0]['disables-inputs'] = ['b'] - task = @generateTask.( { a: 'val1' } ) + task = @generateTask.( { a: 'val1' }, @descriptor ) @checkAfterForm.( task ) end it "is satisfied when only requires is present" do @descriptor['inputs'][0]['requires-inputs'] = ['b'] - task = @generateTask.( { a: 'val1', b: 9 } ) + task = @generateTask.( { a: 'val1', b: 9 }, @descriptor ) @checkAfterForm.( task ) end end # after_form independence diff --git a/BrainPortal/spec/boutiques/descriptor_test.json b/BrainPortal/spec/boutiques/descriptor_test.json index e22121278..193a2883a 100644 --- a/BrainPortal/spec/boutiques/descriptor_test.json +++ b/BrainPortal/spec/boutiques/descriptor_test.json @@ -38,8 +38,7 @@ "optional" : false, "list" : false, "value-key" : "[C]", - "command-line-flag" : "-C", - "uses-absolute-path" : true + "command-line-flag" : "-C" }, { "id" : "a", "name" : "a", @@ -112,6 +111,7 @@ "description" : "A number disabler", "optional" : true, "list" : false, + "integer": true, "disables-inputs" : [ "j", "k", "y" ], "value-key" : "[i]", "command-line-flag" : "-i", @@ -150,7 +150,7 @@ "type" : "String", "description" : "A string list (mutually requires k)", "optional" : true, - "list" : false, + "list" : true, "requires-inputs" : [ "k" ], "value-key" : "[m]", "command-line-flag" : "-m" @@ -299,7 +299,7 @@ "description" : "Optional output filename for required output", "optional" : true, "list" : false, - "default-value" : "r", + "default-value" : "r.txt", "value-key" : "[r]", "command-line-flag" : "-r" }], diff --git a/BrainPortal/spec/boutiques/test_helpers.rb b/BrainPortal/spec/boutiques/test_helpers.rb index d4f33f17c..e517f1c5e 100644 --- a/BrainPortal/spec/boutiques/test_helpers.rb +++ b/BrainPortal/spec/boutiques/test_helpers.rb @@ -38,7 +38,8 @@ module TestHelpers TempStore = File.join('spec','fixtures') # Site for temp file creation, as in other specs ### Helper script argument-specific constants ### - TmpJoin = lambda { |fname| File.join(TempStore, fname) } + #TmpJoin = lambda { |fname| File.join(TempStore, fname) } + TmpJoin = lambda { |fname| fname } # Local name variables for outfile arguments DefReqOutName = TmpJoin.('r.txt') # Default name for required output AltReqOutName = TmpJoin.('r.csv') # Alternate name for required output @@ -71,8 +72,16 @@ module TestHelpers # Execute program with given options def runTestScript(cmdOptions, outfileNamesToCheckFor = []) system( File.join(__dir__, TestScriptName + (Verbose ? ' --verbose ' : ' ') + cmdOptions) ) + status=$? outfileNamesToCheckFor.each { |n| return 11 unless File.exist?(n) } - return $?.exitstatus + return status.exitstatus + end + + def extractOptions(runscript, command_name) + myline = Array(runscript).join("").split(/\n/) + .detect { |line| line.include? command_name } + opts = myline.sub("./#{command_name}","").sub(command_name,"").strip + opts end # JSON validation @@ -97,6 +106,12 @@ def destroyInputFiles ['c','f','jf','f1','f2'].map{|f| TmpJoin.(f) }.each { |f| File.delete(f) if File.exist?(f) } end + def destroyTaskSupportFiles + %w( .qsub* .invoke* .science* .boutiques ).each do |pat| + Dir.glob(pat).to_a.each { |file| File.delete(file) } + end + end + # Destroy output files of the mock program def destroyOutputFiles # Send the deletion request per output file that exists @@ -177,52 +192,52 @@ def destroyOutputFiles ["has mutable required output file name", baseArgs + r_arg, 0, [AltReqOutName] ], ["should not find the default required file when renamed", baseArgs + r_arg, 11, [DefReqOutName] ], ["outputs optional file", baseArgs + o_arg, 0, [DefReqOutName, OptOutName] ], - ["works with a correctly specified enum", baseArgs + '-E c', 0], + ["works with a correctly specified enum", baseArgs + '-E b', 0], ### Tests that should result in the program failing ### # Argument requirement failures - ["fails when a required argument is missing (A: flag + value)", "-n 7 -B 7 -C #{C_file} -v s", 9], - ["fails when a required argument is missing (A: value)", "-n 7 -B 7 -C #{C_file} -v s -A", 1], + ["fails when a required argument is missing (A: flag + value)", "-n 7 -B 7 -C #{C_file} -v s", 9, nil, "'A' is a required property"], + ["fails when a required argument is missing (A: value)", "-n 7 -B 7 -C #{C_file} -v s -A", 1, nil, "'A' is a required property"], # Argument type failures - ["fails when number (-B) is non-numeric (required)", "-n 7 -A a -B q -C #{C_file} -v s -b 7", 3], - ["fails when number (-b) is non-numeric (optional)", baseArgs + "-b u", 3], - ["fails when number in list (-l) is non-numeric (optional, pos 2)", baseArgs + "-l 2 u 2", 4], - ["fails when number in list (-l) is non-numeric (optional, pos 1)", baseArgs + "-l u 1 2", 4], - ["fails when enum is not given a reasonable value", baseArgs + '-E d', 11], + ["fails when number (-B) is non-numeric (required)", "-n 7 -A a -B q -C #{C_file} -v s -b 7", 3, nil, "'q' is not of type 'number'"], + ["fails when number (-b) is non-numeric (optional)", baseArgs + "-b u", 3, nil, "'u' is not of type 'number'"], + ["fails when number in list (-l) is non-numeric (optional, pos 2)", baseArgs + "-l 2 u 2", 4, nil, "'u' is not of type 'number'"], + ["fails when number in list (-l) is non-numeric (optional, pos 1)", baseArgs + "-l u 1 2", 4, nil, "'u' is not of type 'number'"], + ["fails when enum is not given a reasonable value", baseArgs + '-E d', 11, nil, "'d' is not one of"], ["fails when numeric enum is not given a proper value", baseArgs + '-i 7', 11], # Special separator failures ["fails when special separator is missing", baseArgs + "-x 7", 5], ["fails when special separator is wrong", baseArgs + "-x~7", 5], # Number constraints on lists - ["fails when int list contains non-int", baseArgs + "-L 9 9.1 12", 12], - ["fails when list entry is too low", baseArgs + "-L 9 6 12", 12], - ["fails when list entry is too high", baseArgs + "-L 9 15 12", 12], - ["fails when list entry is on excluded boundary", baseArgs + "-L 9 13 12", 12], + ["fails when int list contains non-int", baseArgs + "-L 9 9.1 12", 12, nil, "is not of type 'integer'"], + ["fails when list entry is too low", baseArgs + "-L 9 6 12", 12, nil, "6 is less than the minimum"], + ["fails when list entry is too high", baseArgs + "-L 9 15 12", 12, nil, "15 is greater than or equal to the maximum"], + ["fails when list entry is on excluded boundary", baseArgs + "-L 9 13 12", 12, nil, "13 is greater than or equal to the maximum"], # List constraint failures - ["fails when string list entries are too few", baseArgs + "-e hi", 13], - ["fails when string list entries are too many", baseArgs + "-e a b c d", 13], - ["fails when number list entries are too few", baseArgs + "-L 11 12", 13], - ["fails when number list entries are too many", baseArgs + "-L 7 8 9 10 11 12", 13], + ["fails when string list entries are too few", baseArgs + "-e hi", 13, nil, "is too short"], + ["fails when string list entries are too many", baseArgs + "-e a b c d", 13, nil, "is too long"], + ["fails when number list entries are too few", baseArgs + "-L 11 12", 13, nil, "is too short"], + ["fails when number list entries are too many", baseArgs + "-L 7 8 9 10 11 12", 13, nil, "is too long"], # Numeric constraints failures - ["fails when float is under min", baseArgs + "-N 7", 12], - ["fails when float is over max", baseArgs + "-N 13", 12], - ["fails when float is on prohibited boundary", baseArgs + "-N 7.7", 12], - ["fails when int is not an int", baseArgs + "-I 7.9", 12], - ["fails when int is under min", baseArgs + "-I -9", 12], - ["fails when int is over max", baseArgs + "-I 13", 12], - ["fails when int is on prohibited boundary", baseArgs + "-I 9", 12], + ["fails when float is under min", baseArgs + "-N 7", 12, nil, "is less than or equal to the minimum"], + ["fails when float is over max", baseArgs + "-N 13", 12, nil, "is greater than the maximum"], + ["fails when float is on prohibited boundary", baseArgs + "-N 7.7", 12, nil, "is less than or equal to the minimum"], + ["fails when int is not an int", baseArgs + "-I 7.9", 12, nil, "is not of type 'integer'"], + ["fails when int is under min", baseArgs + "-I -9", 12, nil, "is less than the minimum"], + ["fails when int is over max", baseArgs + "-I 13", 12, nil, "is greater than or equal to the maximum"], + ["fails when int is on prohibited boundary", baseArgs + "-I 9", 12, nil, '9 is greater than or equal to the maximum of 9'], # Disables/requires failures - ["fails if both disabler and target present (k)", baseArgs + "-i 9 -k s", 6], - ["fails if both disabler and target present (k+s)", baseArgs + "-i 9 -j #{J_file} -k s", 6], - ["fails if both disabler and target present (flag: y)", baseArgs + "-i 9 -y", 6], - ["fails when requirement missing (k: no m [number])", baseArgs + "-k s -l 1 2", 7], - ["fails when requirement missing (k: no l [string])", baseArgs + "-k s -m s1 s2", 7], - ["fails when requirement missing (k: no y [flag])", baseArgs + "-k s -m s1 s2 -l 1 2", 7], - ["fails when requirement missing (flag case: y - no l)", baseArgs + "-y", 7], + ["fails if both disabler and target present (k)", baseArgs + "-i 9 -k s", 6, nil, "'l' is a required property"], + ["fails if both disabler and target present (k+s)", baseArgs + "-i 9 -j #{J_file} -k s", 6, nil, "'l' is a required property"], + ["fails if both disabler and target present (flag: y)", baseArgs + "-i 9 -y", 6, nil, "True is not one of"], + ["fails when requirement missing (k: no m [number])", baseArgs + "-k s -l 1 2", 7, nil, "'m' is a required property"], + ["fails when requirement missing (k: no l [string])", baseArgs + "-k s -m s1 s2", 7, nil, "'l' is a required property"], + ["fails when requirement missing (k: no y [flag])", baseArgs + "-k s -m s1 s2 -l 1 2", 7, nil, "'y' is a required property"], + ["fails when requirement missing (flag case: y - no l)", baseArgs + "-y", 7, nil, "True is not one of"], # Group tests - ["fails if group one-is-required is violated (group 1)", reqArgs + "-w", 8], - ["fails if group mutex is violated (group 2)", baseArgs + "-q s -u", 8], - ["fails if group one-is-required is violated (group 3)", reqArgs + "-n 7", 8], - ["fails if group mutex is violated (group 3)", reqArgs + "-n 7 -v s -w", 8], + ["fails if group one-is-required is violated (group 1)", reqArgs + "-w", 8, nil, "is not valid under any of the given schemas"], + ["fails if group mutex is violated (group 2)", baseArgs + "-q s -u", 8, nil, "command failed with return code 1"], # bug in bosh!! + ["fails if group one-is-required is violated (group 3)", reqArgs + "-n 7", 8, nil, "is not valid under any of the given schemas"], + ["fails if group mutex is violated (group 3)", reqArgs + "-n 7 -v s -w", 8, nil, "command failed with return code 1"], # bug in bosh!! # Superfluous argument failures ["fails with unrecognized flagged arguments", baseArgs + "-z", 1], ["fails with unrecognized non-flagged arguments", baseArgs + "z", 1], @@ -305,7 +320,7 @@ def destroyOutputFiles # e.g. {:a => val_a, :l => [1,2], :v => true, ...} when "-a val_a -l 1 2 -v" appears in the string ArgumentDictionary = lambda do |argsIn, idsForFiles=nil| # This will hold the output hash arguments - hash, copy = {}, argsIn.dup + hash, copy = {}.with_indifferent_access, argsIn.dup # Shellify the input string args = Shellwords.shellwords(argsIn) # Must handle space-separated lists separately @@ -315,12 +330,12 @@ def destroyOutputFiles # Fix issues with special separator argument -x # We put a boolean there if the wrong separator is used, so the after_form test fails properly xarg = copy.split.find { |a| a.start_with? "-x=" } - hash.keys.each { |k| hash[k] = xarg[2..-1] if k==:x } unless xarg.nil? - hash[:x] = false if hash.keys.include?(:x) and xarg.nil? + hash.keys.each { |k| hash[k] = xarg[3..-1] if k.to_s=="x" } unless xarg.nil? + hash['x'] = false if hash.keys.include?('x') and xarg.nil? # Replace file paths with ids - unless idsForFiles.nil? + if idsForFiles.present? hash.each do |k,v| - if FileLists.include? k + if FileLists.include? k.to_sym hash[k] = v.map { |filepath| idsForFiles[ InputFilesList.find_index(filepath) ] } else hash[k] = idsForFiles[ InputFilesList.find_index(v) ] if InputFilesList.include? v @@ -352,13 +367,17 @@ def destroyOutputFiles # Helper for cleaning spaces after key substitution, to make it easier to write the correct test result # Ignores the export commands and assumes the final command is the log writer NormedTaskCmd = lambda do |task| - task.cluster_commands[-2].split.join(' ') + coms = task.cluster_commands + coms = coms.join("") if coms.is_a?(Array) + comline = coms.split(/\n/).select { |line| line.match(/^\s*\.\//) }.join("\n").strip + comline end # A mock json task object, to test possible problems that the full mock app cannot be used to reproduce # e.g. a bug incurred when group constraints were present without any disables-inputs/requires-inputs being so # Note the method generates a new task each time NewMinimalTask = -> { + BoutiquesSupport::BoutiquesDescriptor.new( { 'name' => "MinimalTest", 'tool-version' => "9.7.13", @@ -368,11 +387,12 @@ def destroyOutputFiles 'inputs' => [GenerateJsonInputDefault.('a','String','A String arg')], 'output-files' => [{'id' => 'u', 'name' => 'U', 'path-template' => '[A]'}], } + ) } # Helper to generate simple json inputs with default values GenerateJsonInputDefault = lambda do |id,type,desc,otherParams = {}| - return { + return BoutiquesSupport::Input.new({ 'id' => id, 'name' => id.upcase, 'type' => type, @@ -380,7 +400,7 @@ def destroyOutputFiles 'command-line-flag' => "-#{id}", 'value-key' => "[#{id.upcase}]", 'optional' => true - }.merge( otherParams ) + }.merge( otherParams )) end end diff --git a/BrainPortal/spec/models/bourreau_spec.rb b/BrainPortal/spec/models/bourreau_spec.rb index 0f6ce2bcc..c2b85d60e 100644 --- a/BrainPortal/spec/models/bourreau_spec.rb +++ b/BrainPortal/spec/models/bourreau_spec.rb @@ -56,7 +56,6 @@ describe "#start" do before(:each) do allow(bourreau).to receive(:has_remote_control_info?).and_return(true) - allow(RemoteResource).to receive_message_chain(:current_resource, :is_a?).and_return(true) allow(bourreau).to receive(:start_tunnels).and_return(true) allow(bourreau).to receive(:write_to_remote_shell_command) allow(File).to receive(:read).and_return("Bourreau Started")