Testing Fog in EventMachine eventmachine_httpserver

em_server.rb

require 'eventmachine'
require 'evma_httpserver'
require 'fog'

Ropt = {
  :provider            => 'Rackspace',
  :rackspace_username  => "yourusernmae",
  :rackspace_api_key   => "yourpassword",      
  :connection_options  => {}
}

Ropt[:connection_options][:family] = ::Socket::Constants::AF_INET 
Ropt[:connection_options][:chunk_size] = 102_400
Ropt[:rackspace_region] = :ord
Ropt[:rackspace_servicenet] = false

Dirname = "Your container name"

Service = Fog::Storage.new(Ropt) 

class MyHttpServer < EM::Connection
  include EM::HttpServer

  def mem str=''    
    res = `ps -o rss -p #{$$}`.strip.split.last.to_i 
    puts "mem:#{str}=#{res}"
    res
  end  
  
   def post_init
     super
     no_environment_strings
   end

  def service
    res = ::Service
    puts res.inspect
    res    
  end
  
  def dirname
    ::Dirname
  end
   
  def process_http_request

    puts "Handling request"
    
    response = EM::DelegatedHttpResponse.new(self)
    c = "<center><h1>Getting #{dirname}</h1></center>"
    work  = proc do
      response.status = 200
      response.content_type 'text/html'
      
      c << "<br/>Mem before connection: #{mem}"    

      c << "<br/>Mem after connection: #{mem}"
      res = service.directories #.get(dirname)
      c << "<br/>Mem after directories: #{mem}"
      c << "<br/>Returned <textarea>#{res.inspect}</textarea>"      
    end
    callback = proc do
      c << "<br/>Mem before sending response: #{mem}"
      response.content = c
      response.send_response
    end
    
    EM.defer work, callback
    mem
  end
end

EM.run{
  puts "Starting server"
  EM.start_server '0.0.0.0', 9999, MyHttpServer
}

em_server_excon.rb

#### Simplifying it down to make just an excon request ####
#
# Note: in this example, the memory leak appears to be due to doing a call to .inspect on the Excon result. 
# Not surprisingly on a large object this causes all kinds of data to be produced which just jams things up.
# No memory leak here that I can tell of


require 'eventmachine'
require 'excon'


Dirname = "re_file_uploads:[enc]workspace--AUsIyapJH0WlUiyxklaT"


Service = Excon.new('http://www.adobe.com')


#class MySimpleServer < EM::Connection
 module MySimpleServer 

  def mem str=''    
    res = `ps -o rss -p #{$$}`.strip.split.last.to_i 
    puts "mem:#{str}=#{res}"
    res
  end  
  
   
  def service
    res = Service
    puts res.inspect
    res    
  end
  
  def dirname
    Dirname
  end
   
  def receive_data data


    puts "Handling request"
    
    #response = EM::DelegatedHttpResponse.new(self)
    c = "<center><h1>Getting #{dirname}</h1></center>"
    work  = proc do      
      c << "<br/>Mem before connection: #{mem}"    

      c << "<br/>Mem after connection: #{mem}"

      res = service.get(:path=>'/content/dam/Adobe/en/devnet/acrobat/pdfs/pdf_reference_1-7.pdf')

      c << "<br/>Mem after directories: #{mem}"
      c << "<br/>Returned <textarea>#{res}</textarea>"      
    end
    callback = proc do
      c << "<br/>Mem before sending response: #{mem}"
      send_data c
      close_connection
            
    end
    
    EM.defer work, callback
    puts "Connections: #{EM.connection_count}"

    mem
  end
end



EM.run do 

  puts "Starting server in pid #{$$}"
  EM.threadpool_size = 20
  EM.start_server '0.0.0.0', 9999, MySimpleServer

end

em_server_simple.rb

#### Alternative to em_server.rb - removing the evma_http_server dependency #####
#
# Note: playing with the EM.threadpool_size in EM.run has a profound effect
# Using one thread at a time seems to leak memory heavily
# Hammering the server, so that all threads are active seems to cause the memory to top out after a while
# easy to demonstrate: browse to http://localhost:9999/
# wait for it to fail then repeat, etc, a few times
# then hold F5 to force tons of refreshes. Memory limits (drops even)
# Then repeat by some slow refreshes, one at a time and the memory grows again



require 'eventmachine'
require 'fog'


Ropt = {
  :provider            => 'Rackspace',
  :rackspace_username  => "yourusernmae",
  :rackspace_api_key   => "yourpassword",      
  :connection_options  => {}
}

Ropt[:connection_options][:family] = ::Socket::Constants::AF_INET 
Ropt[:connection_options][:chunk_size] = 102_400
Ropt[:rackspace_region] = :ord
Ropt[:rackspace_servicenet] = false


Dirname = "re_file_uploads:[enc]workspace--AUsIyapJH0WlUiyxklaT"
Objname = "20150217172608462--8bcb5ffcecd2c7e4fdd63c1bf8abfbf6"
Service = Fog::Storage.new(Ropt) 

#class MySimpleServer < EM::Connection
 module MySimpleServer 

  def mem str=''    
    res = `ps -o rss -p #{$$}`.strip.split.last.to_i 
    puts "mem:#{str}=#{res}"
    res
  end  
  
   
  def service
    res = Service
    puts res.inspect
    res    
  end
  
  def dirname
    Dirname
  end
  
  def object_name
    Objname
  end
   
  def receive_data data


    puts "Handling request"
    
    #response = EM::DelegatedHttpResponse.new(self)
    c = "<center><h1>Getting #{dirname}</h1></center>"
    work  = proc do      
      c << "<br/>Mem before connection: #{mem}"    

      c << "<br/>Mem after connection: #{mem}"
      directory = service.directories.get(dirname)
      data = nil
      file_data = directory.files.get(object_name) do | d, remaining, content_length |
        if data
          data << d 
        else
          data = d 
        end      
      end
      
      c << "<br/>Mem after directories: #{mem}"
      c << "<br/>Returned <textarea>#{data.length}</textarea>"      
    end
    callback = proc do
      c << "<br/>Mem before sending response: #{mem}"
      send_data c
      close_connection
            
    end
    
    EM.defer work, callback
    puts "Connections: #{EM.connection_count}"

    mem
  end
end



EM.run do 

  puts "Starting server in pid #{$$}"
  EM.threadpool_size = 20
  EM.start_server '0.0.0.0', 9999, MySimpleServer

end

Gemfile

source 'https://rubygems.org'

gem 'eventmachine'
gem 'eventmachine_httpserver', :require => 'evma_httpserver'
gem 'mysql2'
gem 'httpclient', :git=>'https://github.com/philayres/httpclient.git'
gem 'hashie'
gem 'connection_pool'
gem 'logger'
gem 'excon'
gem 'fog', '1.27'
gem 'nokogiri'

threadtest_excon.rb

require 'excon'


Dirname = "re_file_uploads:[enc]workspace--AUsIyapJH0WlUiyxklaT"

Service = Excon.new('http://www.adobe.com')

#class MySimpleServer < EM::Connection
 module MySimpleServer 

  def mem str=''    
    res = `ps -o rss -p #{$$}`.strip.split.last.to_i 
    puts "mem:#{str}=#{res}"
    res
  end  
  
   
  def service
    res = Service
    puts res.inspect
    res    
  end
  
  def dirname
    Dirname
  end
   
  def receive_data data


    puts "Handling request"
    
    #response = EM::DelegatedHttpResponse.new(self)
    c = "<center><h1>Getting #{dirname}</h1></center>"
    work  = proc do      
      c << "<br/>Mem before connection: #{mem}"    

      c << "<br/>Mem after connection: #{mem}"
      res = service.get(:path=>'/content/dam/Adobe/en/devnet/acrobat/pdfs/pdf_reference_1-7.pdf') #.get(dirname)
      c << "<br/>Mem after directories: #{mem}"
      c << "<br/>Returned <textarea>#{res}</textarea>"      
    end
    callback = proc do
      c << "<br/>Mem before sending response: #{mem "done"}"
    end
    
    
    mem
    
    return work, callback
    
  end
end


include MySimpleServer

ts = []

5.times do |i|
  ts << Thread.new {
    work, callback = MySimpleServer.receive_data nil
    Thread.current["callback"] = callback  
    work.call
  }
  sleep(rand(2))      
end

ts.each {|t| t.join; t['callback'].call }

puts "!!!!!!!!!!!!!! Done"
mem
GC.start
mem
sleep 2

ts = []

5.times do |i|
  ts << Thread.new {
    work, callback = MySimpleServer.receive_data nil
    Thread.current["callback"] = callback 
    work.call  
  }
  sleep(rand(2))    
end

ts.each {|t| t.join; t['callback'].call }

puts "!!!!!!!!!!!!!!!! Done again"
mem
GC.start
mem
sleep 2

threadtest_fog.rb

require 'fog'

Ropt = {
  :provider            => 'Rackspace',
  :rackspace_username  => "yourusernmae",
  :rackspace_api_key   => "yourpassword",      
  :connection_options  => {}
}

Ropt[:connection_options][:family] = ::Socket::Constants::AF_INET 
Ropt[:connection_options][:chunk_size] = 102_400
Ropt[:rackspace_region] = :ord
Ropt[:rackspace_servicenet] = false

Dirname = "re_file_uploads:[enc]workspace--AUsIyapJH0WlUiyxklaT"
Objname = "20150217172608462--8bcb5ffcecd2c7e4fdd63c1bf8abfbf6"
Service = Fog::Storage.new(Ropt) 
puts Service

#class MySimpleServer < EM::Connection
 module MySimpleServer 

  def mem str=''    
    res = `ps -o rss -p #{$$}`.strip.split.last.to_i 
    puts "mem:#{str}=#{res}"
    res
  end  
  
   
  def service
    Service
    
  end
  
  def dirname
    Dirname
  end

  def object_name
    Objname
  end  
  
  def receive_data data


    puts "Handling request"
    
    #response = EM::DelegatedHttpResponse.new(self)
    c = "<center><h1>Getting #{dirname}</h1></center>"
    work  = proc do      
      c << "<br/>Mem before connection: #{mem "Start"}"    
      
      directory = service.directories.get(dirname)
      data = nil
      file_data = directory.files.get(object_name) do | d, remaining, content_length |
        if data
          data << d 
        else
          data = d 
        end      
      end
      
      c << "<br/>Mem after get: #{mem "after get"}"
      c << "<br/>Returned <textarea>#{data.length}</textarea>" 
      
    end
    callback = proc do
      c << "<br/>Mem before sending response: #{mem "done"}"
      
    end
    
    
    
    
    return work, callback
    
  end
end


include MySimpleServer

ts = []

20.times do |i|
  ts << Thread.new {
    work, callback = MySimpleServer.receive_data nil
    Thread.current["callback"] = callback    
    work.call
  }
  sleep(rand(2))  
end

ts.each {|t| t.join; t['callback'].call }

puts "!!!!!!!!!!!!!! Done"
mem
GC.start
mem
sleep 5

ts = []

5.times do |i|
  ts << Thread.new {
    work, callback = MySimpleServer.receive_data nil    
    Thread.current["callback"] = callback
    work.call
  }
  sleep(rand(2))  
end

ts.each {|t| t.join; t['callback'].call }

puts "!!!!!!!!!!!!!!!! Done again"
mem
GC.start
mem
sleep 2