generates a dependency graph that can run tasks in parallel

dep_gen.rb

require 'rubygems'
require 'json'
require 'pp'
require 'rake'

class MAS
  attr_reader :scripts, :parallel_tasks
  def initialize
    @scripts = JSON.parse(File.read("mas.json"))
    @to_run = []
    @depth_counter = 0
    @parallel_tasks = []
    puts "NOTE: you should have run prepcompile first"
  end

  def do_menu
    puts
    r = []
    @scripts.keys.each_with_index {|script, i|
      puts "#{i+1}: #{script}"
      r[i+1] = script
    }
    puts "Enter script(s) to run (numbers separated by spaces as in: 1 3 or 'all' ):"
    puts
    user_input = gets.chomp
    if user_input == 'all'
      (1..(@scripts.length)).each {|c| @to_run << r[c] }
    else
      choices = user_input.split.map {|s| s.to_i } 
      puts "Choice(s) : #{choices.join(', ')}"
      choices.each{|c| @to_run << r[c] } 
    end
    complete_deps_tree
    pp @to_run
  end

  def get_dependencies n, deplist
    puts "  get_dependencies #{n}, #{deplist}"
    if !@scripts[n]['depends'].empty?
      @scripts[n]['depends'].each {|s|
        deplist << s
        reslst = get_dependencies( s, deplist)
      }
    end
    deplist
  end

  def complete_deps_tree
    new_to_run = @to_run.clone
    puts "Before complete_deps_tree: #{new_to_run}"
    @to_run.each { |s|
      otherdeps = get_dependencies(s, [])
      puts "otherdeps: #{otherdeps}"
      new_to_run << otherdeps unless otherdeps.empty?
    }
    puts "new_to_run: #{new_to_run}"
    new_to_run.flatten!
    new_to_run.uniq!
    puts "After complete_deps_tree: #{new_to_run}"
    @to_run = new_to_run
  end

  def build_parallel_tasks_list [email protected]
    if without_deps_num == 0 && @depth_counter > @scripts.length
      raise "Dependency cycle detected!"
    end
    if @to_run.length > 0       
      without_deps, with_deps =  @to_run.partition{|s| @scripts[s]['depends'].empty? }

      puts "run in parallel: #{without_deps.join(',')}"
      puts "still with deps: #{with_deps.join(',')}"
      @parallel_tasks << without_deps.clone
      if without_deps.empty? && !with_deps.empty?
        #try to find deps
        with_deps.each {|dn|
          @to_run << get_dependencies( dn, [])
        }
        @to_run.flatten
        puts "Now to run is: #@to_run"
      else
        @scripts.each {|s,shsh|
          shsh['depends'] = shsh['depends'] - without_deps
        }

        @to_run = with_deps
        @to_run.each {|s| 
          print "#{s} : "
          pp @scripts[s]
        } 
        puts
        @depth_counter += 1
        build_parallel_tasks_list without_deps.length
      end
    end
  end

  def run(test=false)
    do_menu
    build_parallel_tasks_list
    @parallel_tasks.each_with_index {|tasklst,i|
      puts "level #{i} tasks:"
      threads = []
      tasklst.each {|task|
        #script = @scripts[sn] 
        cmd_str = "sleep #{rand 5}; echo #{task}; "
        
        if test 
          puts "   ---> #{task} (in thread)" 
          threads << Thread.new {
            system(cmd_str)
          }
        else
        #  sytem( cmd_str )
        end
      }
      puts "   Thread.join (await completion)"
      threads.each {|t| t.join }
    }

  end

end

mas = MAS.new
mas.run(true)

pp mas.parallel_tasks