Author: sitharus
Date: 2006-06-26 07:43:39 +0000 (Mon, 26 Jun 2006)
New Revision: 9387
Modified:
trunk/apps/rubyFreenet/freenet/fcp/client.rb
trunk/apps/rubyFreenet/rqueue/queue_server.rb
trunk/apps/rubyFreenet/rqueue/rqueue
trunk/apps/rubyFreenet/rqueue/templates/index.rhtml
trunk/apps/rubyFreenet/rqueue/web_server.rb
Log:
Changes to FCP::Client to improve performance.
RQueue now has a nicer interface and is more reliable
Modified: trunk/apps/rubyFreenet/freenet/fcp/client.rb
===================================================================
--- trunk/apps/rubyFreenet/freenet/fcp/client.rb 2006-06-25 19:51:15 UTC (rev 9386)
+++ trunk/apps/rubyFreenet/freenet/fcp/client.rb 2006-06-26 07:43:39 UTC (rev 9387)
@@ -434,30 +434,30 @@
status = message.status
log(DEBUG, "Message found: #{original_message.type}. #{original_message.callback?} :#{status}")
original_message.reply = message
+ case status
+ when :found
+ log(INFO, 'Data found, requesting status')
+ original_message.content_type = message.items['Metadata.ContentType']
+ if original_message.items['Persistence'] != 'connection'
+ original_message.data_found = true
+ request_status(original_message.identifier, original_message.items['Global'] || false, false, false)
+ end
+ when :failed
+ if message.items['Fatal'] == 'false'
+ if original_message.retries < 5
+ original_message.retries += 1
+ log(DEBUG, "Retrying #{original_message.items['URI']}")
+ remove_request(original_message)
+ send(original_message)
+ end
+ end
+ end
+
if original_message.callback?
thread = Thread.new do
original_message.lock
message.lock
original_message.callback(status)
- case status
- when :found
- log(INFO, 'Data found, requesting status')
- original_message.content_type = message.items['Metadata.ContentType']
- if original_message.items['Persistence'] != 'connection' and not original_message.data_found
- original_message.data_found = true
- request_status(original_message.identifier, original_message.items['Global'] || false, true, false)
- end
- when :failed
- if message.items['Fatal'] == 'false'
- if message.items['Code'] == 15
- if original_message.retries < 5
- original_message.retries += 1
- log(DEBUG, "Retrying #{original_message.items['URI']}")
- get(original_message.items['URI'], true, original_message.items)
- end
- end
- end
- end
message.unlock
original_message.unlock
end
Modified: trunk/apps/rubyFreenet/rqueue/queue_server.rb
===================================================================
--- trunk/apps/rubyFreenet/rqueue/queue_server.rb 2006-06-25 19:51:15 UTC (rev 9386)
+++ trunk/apps/rubyFreenet/rqueue/queue_server.rb 2006-06-26 07:43:39 UTC (rev 9387)
@@ -3,6 +3,8 @@
# DRuby server
class QueueServer
+ attr_accessor :temp_dir
+
def self.start_server(bind_to)
server = QueueServer.new
DRb.start_service("druby://#{bind_to}", server)
@@ -13,6 +15,7 @@
@client = Freenet::FCP::Client.new
@client.watch_global
@status = {}
+ @messages = {}
@status_mutex = Mutex.new
end
@@ -29,11 +32,14 @@
puts 'bad URI'
next
end
- @client.get(uri, false, 'Persistence'=>'reboot', 'Global'=>'true', 'Verbosity'=>'1') do |status, request, response|
+ @messages[uri] = @client.get(uri, false, 'Persistence'=>'reboot',
+ 'Global'=>'true', 'Verbosity'=>'1', 'MaxRetries'=>'30') do |status, request, response|
case status
when :failed
@status_mutex.lock
@status[request.items['URI']][:status] = 'Failed'
+ @client.remove_request(request)
+ p response.items
@status_mutex.unlock
when :progress
@status_mutex.lock
@@ -45,6 +51,7 @@
data[:succeeded] = response.items['Succeeded']
data[:finalised] = response.items['Finalized']
data[:content_type] = response.content_type
+ data[:progress] = ((response.items['Suceeded'].to_f/response.items['Required'].to_f)*100).ceil
@status_mutex.unlock
when :found
@status_mutex.lock
@@ -53,19 +60,24 @@
when :finished
@status_mutex.lock
@status[request.items['URI']][:status] = 'Finished'
- @status[request.items['URI']][:data] = response.data
- @status[request.items['URI']][:content_type] = response.content_type
+ @status[request.items['URI']][:progress] = 100
+ if @temp_dir
+ File.open(File.join(@temp_dir, @status[request.items['URI']][:filename]), 'w') do |file|
+ file.write(response.data)
+ end
+ else
+ @status[request.items['URI']][:data] = response.data
+ end
+ @status[request.items['URI']][:content_type] = response.items['Metadata.ContentType']
@status_mutex.unlock
@client.remove_request(request)
end
end
queued << uri
@status_mutex.lock
- puts 'here'
filename = new_uri.path.gsub('/','_')
- puts filename
- @status[uri] = {:status=>:started, :data=>nil, :total=>0, :required=>0, :failed=>0, :fatally_failed=>0,
- :succeeded=>0, :finalised=>false, :content_type=>'', :filename=>filename}
+ @status[uri] = {:uri=>uri, :status=>:started, :data=>nil, :total=>0, :required=>0, :failed=>0, :fatally_failed=>0,
+ :succeeded=>0, :finalised=>false, :progress=>0, :content_type=>'', :filename=>filename, :added=>Time.now}
@status_mutex.unlock
rescue Freenet::URIError
end
@@ -87,9 +99,17 @@
# Remove the URI from the queue.
def remove(uri)
@status_mutex.lock
- if @status[uri]
- @client.remove_request(@status[uri]) if @status[uri][:status] != 'Finished'
+ if @messages[uri]
+ @client.remove_request(@messages[uri]) if @status[uri][:status] != 'Finished'
+ if @temp_dir
+ status = @status[uri]
+ begin
+ File.delete(File.join(@temp_dir, status[:filename]))
+ rescue
+ end
+ end
@status.delete(uri)
+ @messages.delete(uri)
end
@status_mutex.unlock
true
@@ -97,8 +117,25 @@
def details_for(uri)
@status_mutex.lock
- details = @status[uri].dup
+ details = nil
+ details = @status[uri].dup unless @status[uri].nil?
@status_mutex.unlock
details
end
+
+ def data_for(uri)
+ data = nil
+ @status_mutex.lock
+ if @status[uri]
+ if @temp_dir
+ File.open(File.join(@temp_dir, @status[uri][:filename])) do |file|
+ data = file.read
+ end
+ else
+ data = @status[uri].data
+ end
+ end
+ @status_mutex.unlock
+ data
+ end
end
\ No newline at end of file
Modified: trunk/apps/rubyFreenet/rqueue/rqueue
===================================================================
--- trunk/apps/rubyFreenet/rqueue/rqueue 2006-06-25 19:51:15 UTC (rev 9386)
+++ trunk/apps/rubyFreenet/rqueue/rqueue 2006-06-26 07:43:39 UTC (rev 9387)
@@ -7,8 +7,8 @@
# ./rqueue [options]
#
# == Options
-# [--drubybind, -d] The IP address and port to bind to, eg 0.0.0.0:9876 for all hosts, default is 127.0.0.1:9876
-# [--web, -w] Start the web server, optionally takes an argument for the port to listen on, default 9877
+# [--drubybind, -d ] The IP address and port to bind to, eg 0.0.0.0:9876 for all hosts, default is 127.0.0.1:9876
+# [--web, -w ] Start the web server, optionally takes an argument for the port to listen on, default 9877
require 'getoptlong'
require 'drb'
require '../freenet'
@@ -19,25 +19,33 @@
opts = GetoptLong.new(
['--web', '-w', GetoptLong::OPTIONAL_ARGUMENT],
- ['--drubybind', '-d', GetoptLong::NO_ARGUMENT],
- ['--help', '-h', GetoptLong::NO_ARGUMENT]
+ ['--drubybind', '-d', GetoptLong::OPTIONAL_ARGUMENT],
+ ['--help', '-h', GetoptLong::NO_ARGUMENT],
+ ['--tempdir', '-t', GetoptLong::REQUIRED_ARGUMENT]
)
druby_bind = '127.0.0.1:9876'
web = nil
+temp_dir = File.join(File.dirname(__FILE__), 'tmp')
+unless File.exists?(temp_dir)
+ Dir.mkdir(temp_dir)
+end
+
opts.each do |type, value|
case type
when '--drubybind': druby_bind = value
when '--web'
web = value || 9877
web = 9877 if value == ''
+ when '--tempdir': temp_dir = value
when '--help'
RDoc::usage
exit 0
end
end
-QueueServer.start_server(druby_bind)
+server = QueueServer.start_server(druby_bind)
+server.temp_dir = temp_dir
QueueWeb.start_server(web) if web
DRb.thread.join if DRb.thread
\ No newline at end of file
Modified: trunk/apps/rubyFreenet/rqueue/templates/index.rhtml
===================================================================
--- trunk/apps/rubyFreenet/rqueue/templates/index.rhtml 2006-06-25 19:51:15 UTC (rev 9386)
+++ trunk/apps/rubyFreenet/rqueue/templates/index.rhtml 2006-06-26 07:43:39 UTC (rev 9387)
@@ -3,6 +3,53 @@
<html>
<head>
<title>QueueWeb</title>
+ <style type="text/css" media="screen">
+ body {
+ font-family: 'Verdana','Arial','Geneva',sans-serif;
+ background: rgb(220, 220, 200);
+ padding: 0px;
+ margin: 0px;
+ }
+
+ h1 {
+ padding: 5px 20px 5px 20px;
+ margin: 0;
+ background: rgb(100, 100, 50);
+ color: rgb(255, 255, 255);
+ border-bottom: 1px solid black;
+ }
+
+ h2 {
+ margin: 0;
+ padding: 8px 15px;
+ }
+ table {
+ width: 800px;
+ margin: auto;
+ border: 1px solid black;
+ border-collapse: collapse;
+ }
+
+ td, th {
+ border: 1px solid rgb(150, 150, 100);
+ padding: 2px;
+ }
+
+ td.progress, th.progress {
+ text-align: center;
+ padding: 3px;
+ width: 105px;
+ }
+
+ .uri {
+ width: 400px;
+ }
+
+ form {
+ margin: 10px 5px;
+ }
+ </style>
+
</head>
<body>
<h1>QueueWeb</h1>
@@ -10,17 +57,22 @@
<table>
<thead>
<tr>
- <th>URI</th>
+ <th class="uri">URI</th>
<th>Status</th>
+ <th class="progress">Progress</th>
<th>Download</th>
<th>Remove</th>
</tr>
</thead>
<tbody>
- <% @status.each do |uri, status| %>
+ <% @status.values.sort{|a,b| a[:added] <=> b[:added]}.each do |status|
+ uri = status[:uri]%>
<tr>
- <td><%=uri%></td>
+ <td><%=status[:uri].gsub(/@[^\/]+/,'@...')%></td>
<td><%=status[:status]%></td>
+ <td class="progress">
+ <div style="width:100px; border: 1px solid rgb(50, 50, 100); height: 10px"><div style="width: <%=status[:progress]%>px; background: rgb(50, 50, 100); height: 10px;"></div></div>
+ </td>
<td><%if status[:status] == 'Finished'%>
<a href="/download?uri=<%=CGI::escape(uri)%>">Download</a>
<%end%></td>
Modified: trunk/apps/rubyFreenet/rqueue/web_server.rb
===================================================================
--- trunk/apps/rubyFreenet/rqueue/web_server.rb 2006-06-25 19:51:15 UTC (rev 9386)
+++ trunk/apps/rubyFreenet/rqueue/web_server.rb 2006-06-26 07:43:39 UTC (rev 9387)
@@ -27,10 +27,12 @@
case req.path
when '/download'
params = CGI::parse(req.query_string)
- status = @drb.status
- item = status[params['uri'][0]]
- res['Content-Type'] = item[:content_type] if item[:content_type] != ''
- res.body = item[:data]
+ item = @drb.details_for(params['uri'][0])
+ if item
+ res['Content-Type'] = item[:content_type] if item[:content_type] != ''
+ end
+ res['Content-Disposition'] = 'inline; filename="'+item[:filename].to_s+'"'
+ res.body = @drb.data_for(params['uri'][0])
when '/remove'
params = CGI::parse(req.query_string)
@drb.remove(params['uri'][0])
