forked from bartvdbraak/blender
netrender fixes:
slave use output log option correctly use os.path.join more consistently (reported by Dalai Felinto) unify frame range (was 4 digits on master, 6 elsewhere, now 6 everywhere). Fix occasional broken pipe between master and slave (didn't affect result, just polluted log).
This commit is contained in:
parent
04f619d8af
commit
bd7cfbc3cc
@ -171,6 +171,7 @@ def clientSendJob(conn, scene, anim = False):
|
||||
# try to send path first
|
||||
conn.request("POST", "/job", repr(job.serialize()))
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
|
||||
job_id = response.getheader("job-id")
|
||||
|
||||
@ -181,6 +182,7 @@ def clientSendJob(conn, scene, anim = False):
|
||||
conn.request("PUT", fileURL(job_id, rfile.index), f)
|
||||
f.close()
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
|
||||
# server will reply with ACCEPTED until all files are found
|
||||
|
||||
@ -236,6 +238,7 @@ class NetworkRenderEngine(bpy.types.RenderEngine):
|
||||
|
||||
requestResult(conn, job_id, scene.frame_current)
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
|
||||
if response.status == http.client.NO_CONTENT:
|
||||
new_job = True
|
||||
@ -244,16 +247,19 @@ class NetworkRenderEngine(bpy.types.RenderEngine):
|
||||
|
||||
requestResult(conn, job_id, scene.frame_current)
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
|
||||
while response.status == http.client.ACCEPTED and not self.test_break():
|
||||
time.sleep(1)
|
||||
requestResult(conn, job_id, scene.frame_current)
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
|
||||
# cancel new jobs (animate on network) on break
|
||||
if self.test_break() and new_job:
|
||||
conn.request("POST", cancelURL(job_id))
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
print( response.status, response.reason )
|
||||
netsettings.job_id = 0
|
||||
|
||||
@ -265,7 +271,7 @@ class NetworkRenderEngine(bpy.types.RenderEngine):
|
||||
x= int(r.resolution_x*r.resolution_percentage*0.01)
|
||||
y= int(r.resolution_y*r.resolution_percentage*0.01)
|
||||
|
||||
f = open(netsettings.path + "output.exr", "wb")
|
||||
f = open(os.path.join(netsettings.path, "output.exr"), "wb")
|
||||
buf = response.read(1024)
|
||||
|
||||
while buf:
|
||||
@ -275,7 +281,7 @@ class NetworkRenderEngine(bpy.types.RenderEngine):
|
||||
f.close()
|
||||
|
||||
result = self.begin_result(0, 0, x, y)
|
||||
result.load_from_file(netsettings.path + "output.exr")
|
||||
result.load_from_file(os.path.join(netsettings.path, "output.exr"))
|
||||
self.end_result(result)
|
||||
|
||||
conn.close()
|
||||
|
@ -89,7 +89,7 @@ class MRenderJob(netrender.model.RenderJob):
|
||||
|
||||
def save(self):
|
||||
if self.save_path:
|
||||
f = open(self.save_path + "job.txt", "w")
|
||||
f = open(os.path.join(self.save_path, "job.txt"), "w")
|
||||
f.write(repr(self.serialize()))
|
||||
f.close()
|
||||
|
||||
@ -134,8 +134,8 @@ class MRenderJob(netrender.model.RenderJob):
|
||||
self.status = JOB_QUEUED
|
||||
|
||||
def addLog(self, frames):
|
||||
log_name = "_".join(("%04d" % f for f in frames)) + ".log"
|
||||
log_path = self.save_path + log_name
|
||||
log_name = "_".join(("%06d" % f for f in frames)) + ".log"
|
||||
log_path = os.path.join(self.save_path, log_name)
|
||||
|
||||
for number in frames:
|
||||
frame = self[number]
|
||||
@ -260,7 +260,7 @@ class RenderHandler(http.server.BaseHTTPRequestHandler):
|
||||
elif frame.status == DONE:
|
||||
self.server.stats("", "Sending result to client")
|
||||
|
||||
filename = job.save_path + "%04d" % frame_number + ".exr"
|
||||
filename = os.path.join(job.save_path, "%06d.exr" % frame_number)
|
||||
|
||||
f = open(filename, 'rb')
|
||||
self.send_head(content = "image/x-exr")
|
||||
@ -294,7 +294,7 @@ class RenderHandler(http.server.BaseHTTPRequestHandler):
|
||||
if frame.status in (QUEUED, DISPATCHED):
|
||||
self.send_head(http.client.ACCEPTED)
|
||||
elif frame.status == DONE:
|
||||
filename = job.save_path + "%04d" % frame_number + ".exr"
|
||||
filename = os.path.join(job.save_path, "%06d.exr" % frame_number)
|
||||
|
||||
thumbname = thumbnail(filename)
|
||||
|
||||
@ -716,7 +716,7 @@ class RenderHandler(http.server.BaseHTTPRequestHandler):
|
||||
if file_index > 0:
|
||||
file_path = prefixPath(job.save_path, render_file.filepath, main_path)
|
||||
else:
|
||||
file_path = job.save_path + main_name
|
||||
file_path = os.path.join(job.save_path, main_name)
|
||||
|
||||
buf = self.rfile.read(length)
|
||||
|
||||
@ -772,7 +772,7 @@ class RenderHandler(http.server.BaseHTTPRequestHandler):
|
||||
if job_result == DONE:
|
||||
length = int(self.headers['content-length'])
|
||||
buf = self.rfile.read(length)
|
||||
f = open(job.save_path + "%04d" % job_frame + ".exr", 'wb')
|
||||
f = open(os.path.join(job.save_path, "%06d.exr" % job_frame), 'wb')
|
||||
f.write(buf)
|
||||
f.close()
|
||||
|
||||
@ -822,13 +822,12 @@ class RenderHandler(http.server.BaseHTTPRequestHandler):
|
||||
if job.type == netrender.model.JOB_BLENDER:
|
||||
length = int(self.headers['content-length'])
|
||||
buf = self.rfile.read(length)
|
||||
f = open(job.save_path + "%04d" % job_frame + ".jpg", 'wb')
|
||||
f = open(os.path.join(job.save_path, "%06d.jpg" % job_frame), 'wb')
|
||||
f.write(buf)
|
||||
f.close()
|
||||
|
||||
del buf
|
||||
|
||||
self.send_head()
|
||||
else: # frame not found
|
||||
self.send_head(http.client.NO_CONTENT)
|
||||
else: # job not found
|
||||
@ -880,7 +879,7 @@ class RenderMasterServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
||||
self.job_id = 0
|
||||
|
||||
if subdir:
|
||||
self.path = path + "master_" + str(os.getpid()) + os.sep
|
||||
self.path = os.path.join(path, "master_" + str(os.getpid()))
|
||||
else:
|
||||
self.path = path
|
||||
|
||||
@ -1007,7 +1006,7 @@ class RenderMasterServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
||||
self.jobs_map[job.id] = job
|
||||
|
||||
# create job directory
|
||||
job.save_path = self.path + "job_" + job.id + os.sep
|
||||
job.save_path = os.path.join(self.path, "job_" + job.id)
|
||||
if not os.path.exists(job.save_path):
|
||||
os.mkdir(job.save_path)
|
||||
|
||||
|
@ -205,6 +205,7 @@ class RENDER_OT_netclientstatus(bpy.types.Operator):
|
||||
conn.request("GET", "/status")
|
||||
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
print( response.status, response.reason )
|
||||
|
||||
jobs = (netrender.model.RenderJob.materialize(j) for j in eval(str(response.read(), encoding='utf8')))
|
||||
@ -306,6 +307,7 @@ class RENDER_OT_netclientslaves(bpy.types.Operator):
|
||||
conn.request("GET", "/slaves")
|
||||
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
print( response.status, response.reason )
|
||||
|
||||
slaves = (netrender.model.RenderSlave.materialize(s) for s in eval(str(response.read(), encoding='utf8')))
|
||||
@ -354,6 +356,7 @@ class RENDER_OT_netclientcancel(bpy.types.Operator):
|
||||
conn.request("POST", cancelURL(job.id))
|
||||
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
print( response.status, response.reason )
|
||||
|
||||
netsettings.jobs.remove(netsettings.active_job_index)
|
||||
@ -380,6 +383,7 @@ class RENDER_OT_netclientcancelall(bpy.types.Operator):
|
||||
conn.request("POST", "/clear")
|
||||
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
print( response.status, response.reason )
|
||||
|
||||
while(len(netsettings.jobs) > 0):
|
||||
@ -412,6 +416,7 @@ class netclientdownload(bpy.types.Operator):
|
||||
for frame in job.frames:
|
||||
client.requestResult(conn, job.id, frame.number)
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
|
||||
if response.status != http.client.OK:
|
||||
print("missing", frame.number)
|
||||
@ -419,7 +424,7 @@ class netclientdownload(bpy.types.Operator):
|
||||
|
||||
print("got back", frame.number)
|
||||
|
||||
f = open(netsettings.path + "%06d" % frame.number + ".exr", "wb")
|
||||
f = open(os.path.join(netsettings.path, "%06d.exr" % frame.number), "wb")
|
||||
buf = response.read(1024)
|
||||
|
||||
while buf:
|
||||
|
@ -59,8 +59,8 @@ def slave_Info():
|
||||
def testCancel(conn, job_id, frame_number):
|
||||
conn.request("HEAD", "/status", headers={"job-id":job_id, "job-frame": str(frame_number)})
|
||||
|
||||
# cancelled if job isn't found anymore
|
||||
if conn.getresponse().status == http.client.NO_CONTENT:
|
||||
# canceled if job isn't found anymore
|
||||
if responseStatus(conn) == http.client.NO_CONTENT:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
@ -81,7 +81,7 @@ def testFile(conn, job_id, slave_id, rfile, JOB_PREFIX, main_path = None):
|
||||
if not found:
|
||||
# Force prefix path if not found
|
||||
job_full_path = prefixPath(JOB_PREFIX, rfile.filepath, main_path, force = True)
|
||||
temp_path = JOB_PREFIX + "slave.temp"
|
||||
temp_path = os.path.join(JOB_PREFIX, "slave.temp")
|
||||
conn.request("GET", fileURL(job_id, rfile.index), headers={"slave-id":slave_id})
|
||||
response = conn.getresponse()
|
||||
|
||||
@ -113,10 +113,11 @@ def render_slave(engine, netsettings, threads):
|
||||
if conn:
|
||||
conn.request("POST", "/slave", repr(slave_Info().serialize()))
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
|
||||
slave_id = response.getheader("slave-id")
|
||||
|
||||
NODE_PREFIX = netsettings.path + "slave_" + slave_id + os.sep
|
||||
NODE_PREFIX = os.path.join(netsettings.path, "slave_" + slave_id)
|
||||
if not os.path.exists(NODE_PREFIX):
|
||||
os.mkdir(NODE_PREFIX)
|
||||
|
||||
@ -132,7 +133,7 @@ def render_slave(engine, netsettings, threads):
|
||||
job = netrender.model.RenderJob.materialize(eval(str(response.read(), encoding='utf8')))
|
||||
engine.update_stats("", "Network render processing job from master")
|
||||
|
||||
JOB_PREFIX = NODE_PREFIX + "job_" + job.id + os.sep
|
||||
JOB_PREFIX = os.path.join(NODE_PREFIX, "job_" + job.id)
|
||||
if not os.path.exists(JOB_PREFIX):
|
||||
os.mkdir(JOB_PREFIX)
|
||||
|
||||
@ -157,6 +158,7 @@ def render_slave(engine, netsettings, threads):
|
||||
logfile = netrender.model.LogFile(job.id, slave_id, [frame.number for frame in job.frames])
|
||||
conn.request("POST", "/log", bytes(repr(logfile.serialize()), encoding='utf8'))
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
|
||||
|
||||
first_frame = job.frames[0].number
|
||||
@ -172,7 +174,7 @@ def render_slave(engine, netsettings, threads):
|
||||
frame_args += ["-f", str(frame.number)]
|
||||
|
||||
val = SetErrorMode()
|
||||
process = subprocess.Popen([BLENDER_PATH, "-b", "-noaudio", job_full_path, "-t", str(threads), "-o", JOB_PREFIX + "######", "-E", "BLENDER_RENDER", "-F", "MULTILAYER"] + frame_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
process = subprocess.Popen([BLENDER_PATH, "-b", "-noaudio", job_full_path, "-t", str(threads), "-o", os.path.join(JOB_PREFIX, "######"), "-E", "BLENDER_RENDER", "-F", "MULTILAYER"] + frame_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
RestoreErrorMode(val)
|
||||
elif job.type == netrender.model.JOB_PROCESS:
|
||||
command = job.frames[0].command
|
||||
@ -196,9 +198,10 @@ def render_slave(engine, netsettings, threads):
|
||||
# (only need to update on one frame, they are linked
|
||||
conn.request("PUT", logURL(job.id, first_frame), stdout, headers=headers)
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
|
||||
# Also output on console
|
||||
if netsettings.slave_thumb:
|
||||
if netsettings.slave_outputlog:
|
||||
print(str(stdout, encoding='utf8'), end="")
|
||||
|
||||
stdout = bytes()
|
||||
@ -227,7 +230,7 @@ def render_slave(engine, netsettings, threads):
|
||||
|
||||
# (only need to update on one frame, they are linked
|
||||
conn.request("PUT", logURL(job.id, first_frame), stdout, headers=headers)
|
||||
if conn.getresponse().status == http.client.NO_CONTENT:
|
||||
if responseStatus(conn) == http.client.NO_CONTENT:
|
||||
continue
|
||||
|
||||
total_t = time.time() - start_t
|
||||
@ -248,7 +251,7 @@ def render_slave(engine, netsettings, threads):
|
||||
if job.type == netrender.model.JOB_BLENDER:
|
||||
# send image back to server
|
||||
|
||||
filename = JOB_PREFIX + "%06d" % frame.number + ".exr"
|
||||
filename = os.path.join(JOB_PREFIX, "%06d.exr" % frame.number)
|
||||
|
||||
# thumbnail first
|
||||
if netsettings.slave_thumb:
|
||||
@ -257,17 +260,18 @@ def render_slave(engine, netsettings, threads):
|
||||
f = open(thumbname, 'rb')
|
||||
conn.request("PUT", "/thumb", f, headers=headers)
|
||||
f.close()
|
||||
conn.getresponse()
|
||||
responseStatus(conn)
|
||||
|
||||
|
||||
f = open(filename, 'rb')
|
||||
conn.request("PUT", "/render", f, headers=headers)
|
||||
f.close()
|
||||
if conn.getresponse().status == http.client.NO_CONTENT:
|
||||
if responseStatus(conn) == http.client.NO_CONTENT:
|
||||
continue
|
||||
|
||||
elif job.type == netrender.model.JOB_PROCESS:
|
||||
conn.request("PUT", "/render", headers=headers)
|
||||
if conn.getresponse().status == http.client.NO_CONTENT:
|
||||
if responseStatus(conn) == http.client.NO_CONTENT:
|
||||
continue
|
||||
else:
|
||||
headers["job-result"] = str(ERROR)
|
||||
@ -275,7 +279,7 @@ def render_slave(engine, netsettings, threads):
|
||||
headers["job-frame"] = str(frame.number)
|
||||
# send error result back to server
|
||||
conn.request("PUT", "/render", headers=headers)
|
||||
if conn.getresponse().status == http.client.NO_CONTENT:
|
||||
if responseStatus(conn) == http.client.NO_CONTENT:
|
||||
continue
|
||||
|
||||
engine.update_stats("", "Network render connected to master, waiting for jobs")
|
||||
|
@ -57,6 +57,11 @@ FRAME_STATUS_TEXT = {
|
||||
ERROR: "Error"
|
||||
}
|
||||
|
||||
def responseStatus(conn):
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
return response.status
|
||||
|
||||
def reporting(report, message, errorType = None):
|
||||
if errorType:
|
||||
t = 'ERROR'
|
||||
@ -171,16 +176,16 @@ def prefixPath(prefix_directory, file_path, prefix_path, force = False):
|
||||
|
||||
if prefix_path and p.startswith(prefix_path):
|
||||
if len(prefix_path) < len(p):
|
||||
directory = prefix_directory + p[len(prefix_path)+1:] + os.sep # +1 to remove separator
|
||||
directory = os.path.join(prefix_directory, p[len(prefix_path)+1:]) # +1 to remove separator
|
||||
if not os.path.exists(directory):
|
||||
os.mkdir(directory)
|
||||
else:
|
||||
directory = prefix_directory
|
||||
full_path = directory + n
|
||||
full_path = os.path.join(directory, n)
|
||||
else:
|
||||
full_path = prefix_directory + n
|
||||
full_path = os.path.join(prefix_directory, n)
|
||||
else:
|
||||
full_path = prefix_directory + file_path
|
||||
full_path = (prefix_directory, file_path)
|
||||
|
||||
return full_path
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user