We Keep Coding

Microservice communication with python in Asynchronous manner

I want to trigger Python script from my Spring boot microservices im Asynchronous manner, SO that my Microservice will be notified once the execution of python script completes.Can any one suggest the best approach for this? appreciated if any one provide some reference to sample code.
Thanks in advance!!!
Thanks,
Sudheer

here is very good Example for something like that:
Source: http://code.activestate.com/recipes/511454-an-asynchronous-http-server-with-cgi-support/
import SimpleAsyncServer
# =============================================================
# An implementation of the HTTP protocol, supporting persistent
# connections and CGI
# =============================================================
import sys
import os
import traceback
import datetime
import mimetypes
import urlparse
import urllib
import cStringIO
class HTTP(SimpleAsyncServer.ClientHandler):
# parameters to override if necessary
root = os.getcwd() # the directory to serve files from
cgi_directories = ['/cgi-bin'] # subdirectories for cgi scripts
logging = True # print logging info for each request ?
blocksize = 2 << 16 # size of blocks to read from files and send
def request_complete(self):
"""In the HTTP protocol, a request is complete if the "end of headers"
sequence ('\r\n\r\n') has been received
If the request is POST, stores the request body in a StringIO before
returning True"""
terminator = self.incoming.find('\r\n\r\n')
if terminator == -1:
return False
lines = self.incoming[:terminator].split('\r\n')
self.requestline = lines[0]
try:
self.method,self.url,self.protocol = lines[0].strip().split()
except:
self.method = None # indicates bad request
return True
# put request headers in a dictionary
self.headers = {}
for line in lines[1:]:
k,v = line.split(':',1)
self.headers[k.lower().strip()] = v.strip()
# persistent connection
close_conn = self.headers.get("connection","")
if (self.protocol == "HTTP/1.1"
and close_conn.lower() == "keep-alive"):
self.close_when_done = False
# parse the url
scheme,netloc,path,params,query,fragment = urlparse.urlparse(self.url)
self.path,self.rest = path,(params,query,fragment)
if self.method == 'POST':
# for POST requests, read the request body
# its length must be specified in the content-length header
content_length = int(self.headers.get('content-length',0))
body = self.incoming[terminator+4:]
# request is incomplete if not all message body received
if len(body)<content_length:
return False
f_body = cStringIO.StringIO(body)
f_body.seek(0)
sys.stdin = f_body # compatibility with CGI
return True
def make_response(self):
"""Build the response : a list of strings or files"""
if self.method is None: # bad request
return self.error_resp(400,'Bad request : %s' %self.requestline)
resp_headers, resp_body, resp_file = '','',None
if not self.method in ['GET','POST','HEAD']:
return self.err_resp(501,'Unsupported method (%s)' %self.method)
else:
file_name = self.file_name = self.translate_path()
if not os.path.exists(file_name):
return self.err_resp(404,'File not found')
elif self.managed():
response = self.mngt_method()
else:
ext = os.path.splitext(file_name)[1]
c_type = mimetypes.types_map.get(ext,'text/plain')
resp_line = "%s 200 Ok\r\n" %self.protocol
size = os.stat(file_name).st_size
resp_headers = "Content-Type: %s\r\n" %c_type
resp_headers += "Content-Length: %s\r\n" %size
resp_headers += '\r\n'
if self.method == "HEAD":
resp_string = resp_line + resp_headers
elif size > HTTP.blocksize:
resp_string = resp_line + resp_headers
resp_file = open(file_name,'rb')
else:
resp_string = resp_line + resp_headers + \
open(file_name,'rb').read()
response = [resp_string]
if resp_file:
response.append(resp_file)
self.log(200)
return response
def translate_path(self):
"""Translate URL path into a path in the file system"""
return os.path.join(HTTP.root,*self.path.split('/'))
def managed(self):
"""Test if the request can be processed by a specific method
If so, set self.mngt_method to the method used
This implementation tests if the script is in a cgi directory"""
if self.is_cgi():
self.mngt_method = self.run_cgi
return True
return False
def is_cgi(self):
"""Test if url in a cgi directory"""
for path in self.cgi_directories:
if self.path.startswith(path):
rest = self.path[len(path):]
if not rest or rest.startswith('/'):
return True
return False
def run_cgi(self):
# set CGI environment variables
self.make_cgi_env()
# redirect print statements to a cStringIO
save_stdout = sys.stdout
sys.stdout = cStringIO.StringIO()
# run the script
try:
execfile(self.file_name)
except:
sys.stdout = cStringIO.StringIO()
sys.stdout.write("Content-type:text/plain\r\n\r\n")
traceback.print_exc(file=sys.stdout)
response = sys.stdout.getvalue()
if self.method == "HEAD":
# for HEAD request, don't send message body even if the script
# returns one (RFC 3875)
head_lines = []
for line in response.split('\n'):
if not line:
break
head_lines.append(line)
response = '\n'.join(head_lines)
sys.stdout = save_stdout # restore sys.stdout
# close connection in case there is no content-length header
self.close_when_done = True
resp_line = "%s 200 Ok\r\n" %self.protocol
return [resp_line + response]
def make_cgi_env(self):
"""Set CGI environment variables"""
env = {}
env['SERVER_SOFTWARE'] = "AsyncServer"
env['SERVER_NAME'] = "AsyncServer"
env['GATEWAY_INTERFACE'] = 'CGI/1.1'
env['DOCUMENT_ROOT'] = HTTP.root
env['SERVER_PROTOCOL'] = "HTTP/1.1"
env['SERVER_PORT'] = str(self.server.port)
env['REQUEST_METHOD'] = self.method
env['REQUEST_URI'] = self.url
env['PATH_TRANSLATED'] = self.translate_path()
env['SCRIPT_NAME'] = self.path
env['PATH_INFO'] = urlparse.urlunparse(("","","",self.rest[0],"",""))
env['QUERY_STRING'] = self.rest[1]
if not self.host == self.client_address[0]:
env['REMOTE_HOST'] = self.host
env['REMOTE_ADDR'] = self.client_address[0]
env['CONTENT_LENGTH'] = str(self.headers.get('content-length',''))
for k in ['USER_AGENT','COOKIE','ACCEPT','ACCEPT_CHARSET',
'ACCEPT_ENCODING','ACCEPT_LANGUAGE','CONNECTION']:
hdr = k.lower().replace("_","-")
env['HTTP_%s' %k.upper()] = str(self.headers.get(hdr,''))
os.environ.update(env)
def err_resp(self,code,msg):
"""Return an error message"""
resp_line = "%s %s %s\r\n" %(self.protocol,code,msg)
self.close_when_done = True
self.log(code)
return [resp_line]
def log(self,code):
"""Write a trace of the request on stderr"""
if HTTP.logging:
date_str = datetime.datetime.now().strftime('[%d/%b/%Y %H:%M:%S]')
sys.stderr.write('%s - - %s "%s" %s\n' %(self.host,
date_str,self.requestline,code))
if __name__=="__main__":
# launch the server on the specified port
port = 8081
print "Asynchronous HTTP server running on port %s" %port
print "Press Ctrl+C to stop"
server = SimpleAsyncServer.Server('', port)
HTTP.logging = False
try:
SimpleAsyncServer.loop(server,HTTP)
except KeyboardInterrupt:
for s in server.client_handlers:
server.close_client(s)
print 'Ctrl+C pressed. Closing'

One way of doing asynchronous execution in Python would be to use the celery framework. It's really simple to install and setup (basically just pip install), and the semantic for calling a method asynchronously is super clean.
If you are not too bound to Spring you could even switch to the pymacaron microservice framework (basically a flask app that uses swagger to spawn a REST API). Pymacaron (http://pymacaron.com/) supports asynchronous calls by default, via the pymacaron-async module. See the examples here: http://pymacaron.com/async.html

Too Many Open Files (Selenium + PhantomJSDriver)

In my embedded Selenium/PhantomJSDriver driver it seems resources are not being cleaned up. Running the client synchronously causes millions of open files and eventually throws a "Too many files open" type exception.
Here is some output I gathered from lsof while the program is running for ~1 minute
$ lsof | awk '{ print $2; }' | uniq -c | sort -rn | head
1221966 12180
34790 29773
31260 12138
20955 8414
17940 10343
16665 32332
9512 27713
7275 19226
5496 7153
5040 14065
$ lsof -p 12180 | awk '{ print $2; }' | uniq -c | sort -rn | head
2859 12180
1 PID
$ lsof -p 12180 -Fn | sort -rn | uniq -c | sort -rn | head
1124 npipe
536 nanon_inode
4 nsocket
3 n/opt/jdk/jdk1.8.0_60/jre/lib/jce.jar
3 n/opt/jdk/jdk1.8.0_60/jre/lib/charsets.jar
3 n/dev/urandom
3 n/dev/random
3 n/dev/pts/20
2 n/usr/share/sbt-launcher-packaging/bin/sbt-launch.jar
2 n/usr/share/java/jayatana.jar
I don't understand why using the -p flag on lsof has a smaller result set. But it appears most of the entries are pipe and anon_inode.
The client is very simple at ~100 lines, and at the end of usage calls driver.close() and driver.quit(). I experimented with caching and reusing clients but it did not alleviate the open files
case class HeadlessClient(
country: String,
userAgent: String,
inheritSessionId: Option[Int] = None
) {
protected var numberOfRequests: Int = 0
protected val proxySessionId: Int = inheritSessionId.getOrElse(new Random().nextInt(Integer.MAX_VALUE))
protected val address = InetAddress.getByName("proxy.domain.com")
protected val host = address.getHostAddress
protected val login: String = HeadlessClient.username + proxySessionId
protected val windowSize = new org.openqa.selenium.Dimension(375, 667)
protected val (mobProxy, seleniumProxy) = {
val proxy = new BrowserMobProxyServer()
proxy.setTrustAllServers(true)
proxy.setChainedProxy(new InetSocketAddress(host, HeadlessClient.port))
proxy.chainedProxyAuthorization(login, HeadlessClient.password, AuthType.BASIC)
proxy.addLastHttpFilterFactory(new HttpFiltersSourceAdapter() {
override def filterRequest(originalRequest: HttpRequest): HttpFilters = {
new HttpFiltersAdapter(originalRequest) {
override def proxyToServerRequest(httpObject: HttpObject): io.netty.handler.codec.http.HttpResponse = {
httpObject match {
case req: HttpRequest => req.headers().remove(HttpHeaders.Names.VIA)
case _ =>
}
null
}
}
}
})
proxy.enableHarCaptureTypes(CaptureType.REQUEST_CONTENT, CaptureType.RESPONSE_CONTENT)
proxy.start(0)
val seleniumProxy = ClientUtil.createSeleniumProxy(proxy)
(proxy, seleniumProxy)
}
protected val driver: PhantomJSDriver = {
val capabilities: DesiredCapabilities = DesiredCapabilities.chrome()
val cliArgsCap = new util.ArrayList[String]
cliArgsCap.add("--webdriver-loglevel=NONE")
cliArgsCap.add("--ignore-ssl-errors=yes")
cliArgsCap.add("--load-images=no")
capabilities.setCapability(CapabilityType.PROXY, seleniumProxy)
capabilities.setCapability("phantomjs.page.customHeaders.Referer", "")
capabilities.setCapability("phantomjs.page.settings.userAgent", userAgent)
capabilities.setCapability(PhantomJSDriverService.PHANTOMJS_CLI_ARGS, cliArgsCap)
new PhantomJSDriver(capabilities)
}
driver.executePhantomJS(
"""
|var navigation = [];
|
|this.onNavigationRequested = function(url, type, willNavigate, main) {
| navigation.push(url)
| console.log('Trying to navigate to: ' + url);
|}
|
|this.onResourceRequested = function(request, net) {
| console.log("Requesting " + request.url);
| if (! (navigation.indexOf(request.url) > -1)) {
| console.log("Aborting " + request.url)
| net.abort();
| }
|};
""".stripMargin
)
driver.manage().window().setSize(windowSize)
def follow(url: String)(implicit ec: ExecutionContext): List[HarEntry] = {
try{
Await.result(Future{
mobProxy.newHar(url)
driver.get(url)
val entries = mobProxy.getHar.getLog.getEntries.asScala.toList
shutdown()
entries
}, 45.seconds)
} catch {
case e: Exception =>
try {
shutdown()
} catch {
case shutdown: Exception =>
throw new Exception(s"Error ${shutdown.getMessage} cleaning up after Exception: ${e.getMessage}")
}
throw e
}
}
def shutdown() = {
driver.close()
driver.quit()
}
}
I tried several versions of Selenium in case there was a bugfix. The build.sbt:
libraryDependencies += "org.seleniumhq.selenium" % "selenium-java" % "3.0.1"
libraryDependencies += "net.lightbody.bmp" % "browsermob-core" % "2.1.2"
Also, I tried PhantomJS 2.0.1, and 2.1.1:
$ phantomjs --version
2.0.1-development
$ phantomjs --version
2.1.1
Is this a PhantomJS or Selenium problem? Is my client using the API improperly?

The resource usage is caused by BrowserMob. To close the proxy and clean-up its resources, one must call stop().
For this client that means modifying the shutdown method
def shutdown() = {
mobProxy.stop()
driver.close()
driver.quit()
}
Another method, abort, offers immediate termination of the proxy server and does not wait for traffic to cease.

In my opinion it seems a problem of PhantomJS. You can try the following alternatives:
Use phantomjs 2.5.0-beta. It has been recently released. I'm not sure if this upgrade solves your problem, but at least it is worth to give a try. According to the changelog, the new features of this version are:
Upgrade QtWebKit to QtWebKitNG
Upgraded Qt to 5.7.1
Clean the phantomjs processes after closing webdriver. You can implement your own cleaner to force that phantomjs is actually closed after driver.close() (invoking killall -9 phantomjs or similar).

Create external process and wait 5 seconds kill it in php

I want to kill a external process after 5 seconds it created
That process is javac Hello.java & java Hello
I figured out a method that is kill process's pid, but samples below cannot get pid.
<?php
system('javac Hello.java & java Hello', $retval);
popen('javac Hello.java & java Hello', 'w');
?>
I don't understand proc_open() and proc_get_status example in PHP manual.
In my case, how to do that?

Here is a very clear explanation for how to use proc_open()
https://www.sitepoint.com/proc-open-communicate-with-the-outside-world/
I've done my job for creating an external process to compile , run java program, wait seconds, kill java running process in PHP script.
Here is that script:
<?php
/**
This is a judge script for verifying lab assignment
of NTU Civil Engineering Computer Programing with no testdata.
You can copy, redistribute, or modify it freely.
Tips:
It is not necessary to use file already provided.
Any related file can be uploaded via testdata field.
Then how to apply specific file uploaded to ./problem/testdata/[item]/[subitem]
completely depends on by how the judge file is defined.
*/
// Error report mechanism of this script
ini_set('display_errors', '1');
ERROR_REPORTING(E_ALL);
// Auto load class definition file that this script will be using.
function __autoload($class_name) {
include_once($class_name . '.php');
}
class Java_No_Input {
private $stu_account;
private $item;
private $subitem;
private $main;
private $dir_name;
private $status;
private $solution_output;
private $student_output;
private $hookup;
public function __construct () {
// Arguments: student account, item, subitem
$this->stu_account = $_SERVER['argv'][1];
$this->item = $_SERVER['argv'][2];
$this->subitem = $_SERVER['argv'][3];
try {
// Connect to MySQL database TAFreeDB
$this->hookup = UniversalConnect::doConnect();
// Create directory to put source codes temporarily
$this->createDir();
// Fetch student and solution source from table [item]_[subitem]
$this->fetchSource();
// Start judge
$this->startJudge();
// Update judge status
$this->updateStatus();
// Remove directory
$this->removeDir();
$this->hookup = null;
exit();
}
catch (PDOException $e) {
echo 'Error: ' . $e->getMessage() . '<br>';
}
}
public function createDir () {
$this->dir_name = './process/' . uniqid(time(), true);
mkdir($this->dir_name);
mkdir($this->dir_name . '/student');
mkdir($this->dir_name . '/solution');
}
public function removeDir () {
system('rm -rf ' . $this->dir_name, $retval);
if ($retval !== 0 ) {
echo 'Directory can not be removed...';
exit();
}
}
public function fetchSource () {
$stmt = $this->hookup->prepare('SELECT main, classname, original_source, ' . $this->stu_account . ' FROM ' . $this->item . '_' . $this->subitem);
$stmt->execute();
while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
if ($row['main'] === 'V') {
$this->main = $row['classname'];
}
$student = fopen($this->dir_name . '/student/' . $row['classname'], 'w');
fwrite($student, $row[$this->stu_account]);
fclose($student);
$solution = fopen($this->dir_name . '/solution/' . $row['classname'], 'w');
fwrite($solution, $row['original_source']);
fclose($solution);
}
}
public function startJudge () {
// Solution and student directory whose source is in
$solution_dir = $this->dir_name . '/solution';
$student_dir = $this->dir_name . '/student';
// Compile source code from both solution and student
$solution_CE = $this->compile($solution_dir);
if (!empty($solution_CE)) {
// Configure result that will response to client side
$error_msg = '<h1>Solution has compiler error</h1>' . '<pre><code>' . $solution_CE . '</code></pre>';
$this->configureView($error_msg);
// System error
$this->status = 'SE';
return;
}
$student_CE = $this->compile($student_dir);
if (!empty($student_CE)) {
// Configure result that will response to client side
$error_msg = '<h1>Your source code has compiler error</h1>' . '<pre><code>' . $student_CE . '</code></pre>';
$this->configureView($error_msg);
// Compiler error
$this->status = 'CE';
return;
}
// Execute source code from both solution and student
$solution_RE = $this->execute($solution_dir, 2);
if (!empty($solution_RE)) {
// Configure result that will response to client side
$error_msg = '<h1>Solution has runtime error</h1>' . '<pre><code>' . $solution_RE . '</code></pre>';
$this->configureView($error_msg);
// System error
$this->status = 'SE';
return;
}
$student_RE = $this->execute($student_dir, 2);
if (!empty($student_RE)) {
// Configure result that will response to client side
$error_msg = '<h1>Your source code has runtime error</h1>' . '<pre><code>' . $student_RE . '</code></pre>';
$this->configureView($error_msg);
// Runtime error
$this->status = 'RE';
return;
}
// Compare output from both solution and student
$this->solution_output = $this->execute($solution_dir, 1);
$this->student_output = $this->execute($student_dir, 1);
$retval = strcmp($this->solution_output, $this->student_output);
if ($retval === 0) {
// Accept
$this->status = 'AC';
}
else {
// Wrong Answer
$this->status = 'WA';
}
// Configure result that will response to client side
$error_msg = null;
$this->configureView($error_msg);
return;
}
public function compile ($dir) {
// Configure descriptor array
$desc = array (
0 => array ('pipe', 'r'), // STDIN for process
1 => array ('pipe', 'w'), // STDOUT for process
2 => array ('pipe', 'w') // STDERR for process
);
// Configure compilation command
$cmd = 'javac -d ' . $dir . ' ';
$source = glob($dir . '/*');
foreach ($source as $key => $value) {
$cmd .= $value . ' ';
}
// Create compilation process
$process = proc_open($cmd, $desc, $pipes);
// Close STDIN pipe
fclose($pipes[0]);
// Get output of STDERR pipe
$error = stream_get_contents($pipes[2]);
// Close STDOUT and STDERR pipe
fclose($pipes[1]);
fclose($pipes[2]);
// Close process
proc_close($process);
return $error;
}
public function execute ($dir, $pipe_id) {
// Configure descriptor array
$desc = array (
0 => array ('pipe', 'r'), // STDIN for process
1 => array ('pipe', 'w'), // STDOUT for process
2 => array ('pipe', 'w') // STDERR for process
);
// Configure execution command
$cmd = 'exec java -classpath ' . $dir . ' ';
$last_pos = strrpos($this->main, '.java');
$classname = substr($this->main, 0, $last_pos);
$cmd .= $classname;
// Create execution process
$process = proc_open($cmd, $desc, $pipes);
// Get pid of execution process
$process_status = proc_get_status($process);
$pid = $process_status['pid'];
// Close STDIN pipe
fclose($pipes[0]);
// Wait seconds
sleep(1);
// Kill execution process
posix_kill($pid, SIGTERM);
// Get output of STDOUT or STDERR pipe
$output = stream_get_contents($pipes[$pipe_id]);
// Close STDOUT and STDERR pipe
fclose($pipes[1]);
fclose($pipes[2]);
return $output;
}
public function updateStatus () {
$stmt = $this->hookup->prepare('UPDATE ' . $this->item . ' SET ' . $this->stu_account . '=\'' . $this->status . '\' WHERE subitem=\'' . $this->subitem . '\'');
$stmt->execute();
}
public function configureView ($error_msg) {
if (!is_null($error_msg)) {
echo $error_msg;
}
else {
$result = '';
if ($this->status === 'WA') {
$result = 'Wrong Answer';
}
if ($this->status === 'AC') {
$result = 'Accept';
}
echo<<<EOF
<h1>$result</h1>
<div class='WHOSE_DIV'>
<img class='UP_DOWN_IMG' src='./tafree-svg/attention.svg'>
<div class='RES_DIV'>
<div class='SOL_DIV'>{$this->solution_output}</div>
<div class='STU_DIV'>{$this->student_output}</div>
</div>
</div>
EOF;
}
return;
}
}
$judger = new Java_No_Input();
?>

Java maximum heap size on SGE cluster

I have a pipeline written in python calling some processes in Java. The pipeline runs with two possible modes, on local mode (on a single node) or on SGE cluster.
When I set the option to cluster mode, the error message in the logs are such this
Invalid maximum heap size: -Xmx4g -jar
Error: Could not create the Java Virtual Machine.
Error: A fatal exception has occurred. Program will exit.
In local mode, there is no error and no problems.
My question is what can cause such an error ?
My class to run jobs either locally or on cluster is as follow
class LocalJobManager(JobManager):
def __init__(self):
self.cmd_strs = []
def add_job(self, cmd, cmd_args, **kwargs):
cmd_str = ' '.join([cmd, ] + [str(x) for x in cmd_args])
self.cmd_strs.append(cmd_str)
def run_job(self, cmd, cmd_args, **kwargs):
cmd_str = ' '.join([cmd, ] + [str(x) for x in cmd_args])
self._run_cmd(cmd_str)
def wait(self):
for cmd_str in self.cmd_strs:
self._run_cmd(cmd_str)
def _run_cmd(self, cmd_str):
'''
Throw exception if run command fails
'''
process = subprocess.Popen(cmd_str, stdin=subprocess.PIPE, shell=True)
process.stdin.close()
sts = os.waitpid(process.pid, 0)
if sts[1] != 0:
raise Exception('Failed to run {0}\n'.format(cmd_str))
class ClusterJobManager(JobManager):
def __init__(self, log_dir=None):
import drmaa
self._drmaa = drmaa
self.log_dir = log_dir
if self.log_dir is not None:
make_directory(self.log_dir)
self.session = self._drmaa.Session()
self.session.initialize()
self.job_ids = Queue()
self._lock = threading.Lock()
def add_job(self, cmd, cmd_args, mem=4, max_mem=10, num_cpus=1):
job_id = self._run_job(cmd, cmd_args, mem, max_mem, num_cpus)
self.job_ids.put(job_id)
def run_job(self, cmd, cmd_args, mem=4, max_mem=10, num_cpus=1):
job_id = self._run_job(cmd, cmd_args, mem, max_mem, num_cpus)
self._check_exit_status(job_id)
def wait(self):
self._lock.acquire()
job_ids = []
while not self.job_ids.empty():
job_ids.append(self.job_ids.get())
self.session.synchronize(job_ids, self._drmaa.Session.TIMEOUT_WAIT_FOREVER, False)
self._lock.release()
for job_id in job_ids:
self._check_exit_status(job_id)
def close(self):
self.session.control(self._drmaa.Session.JOB_IDS_SESSION_ALL, self._drmaa.JobControlAction.TERMINATE)
self.session.exit()
def _run_job(self, cmd, cmd_args, mem, max_mem, num_cpus):
job_template = self._init_job_template(cmd, cmd_args, mem, max_mem, num_cpus)
job_id = self.session.runJob(job_template)
self.session.deleteJobTemplate(job_template)
return job_id
def _init_job_template(self, cmd, cmd_args, mem, max_mem, num_cpus):
job_template = self.session.createJobTemplate()
job_template.remoteCommand = cmd
job_template.args = [str(x) for x in cmd_args]
job_template.workingDirectory = os.getcwd()
if self.log_dir is not None:
job_template.errorPath = ':' + self.log_dir
job_template.outputPath = ':' + self.log_dir
job_template.nativeSpecification = '-l mem_free={mem}G,mem_token={mem}G,h_vmem={max_mem}G -V -w n -pe ncpus {num_cpus}'.format(**locals())
return job_template
def _check_exit_status(self, job_id):
return_value = self.session.wait(job_id, self._drmaa.Session.TIMEOUT_WAIT_FOREVER)
if return_value.exitStatus != 0:
raise Exception('Job {0} failed with exit status {1}.'.format(return_value.jobId,
return_value.exitStatus))
Usually the Could not create the Java Virtual Machine (as I am reading though some forum) is caused by syntax error, even though the command called is correct and works locally, besides the class to run jobs on the cluster show above, runs for everything except Java
Thanks

I've run into this on SGE. You may have a default hard memory limit set around 4GB and Java seems to use a bit more than that 4GB you are setting in the -Xmx4g parameter during initialization. Can you see if you administrator has set a hard memory limit? Typically you will set or override the default limit using:
qsub -l h_vmem=16G
Try giving far more memory than needed via that parameter and see if that solves it and then ratchet down the h_vmem as far down as it goes without crashing.

Powershell: Capturing standard out and error with Process object

I want to start a Java program from PowerShell and get the results printed on the console.
I have followed the instructions of this question:
Capturing standard out and error with Start-Process
But for me, this is not working as I expected. What I'm doing wrong?
This is the script:
$psi = New-object System.Diagnostics.ProcessStartInfo
$psi.CreateNoWindow = $true
$psi.UseShellExecute = $false
$psi.RedirectStandardOutput = $true
$psi.RedirectStandardError = $true
$psi.FileName = 'java.exe'
$psi.Arguments = #("-jar","tools\compiler.jar","--compilation_level", "ADVANCED_OPTIMIZATIONS", "--js", $BuildFile, "--js_output_file", $BuildMinFile)
$process = New-Object System.Diagnostics.Process
$process.StartInfo = $psi
$process.Start() | Out-Null
$process.WaitForExit()
$output = $process.StandardOutput.ReadToEnd()
$output
The $output variable is always empty (and nothing is printed on the console of course).

The docs on the RedirectStandardError property suggests that it is better to put the WaitForExit() call after the ReadToEnd() call. The following works correctly for me:
$psi = New-object System.Diagnostics.ProcessStartInfo
$psi.CreateNoWindow = $true
$psi.UseShellExecute = $false
$psi.RedirectStandardOutput = $true
$psi.RedirectStandardError = $true
$psi.FileName = 'ipconfig.exe'
$psi.Arguments = #("/a")
$process = New-Object System.Diagnostics.Process
$process.StartInfo = $psi
[void]$process.Start()
$output = $process.StandardOutput.ReadToEnd()
$process.WaitForExit()
$output

Small variation so that you can selectively print the output if needed. As in if your looking just for error or warning messages and by the way Keith you saved my bacon with your response...
$psi = New-object System.Diagnostics.ProcessStartInfo
$psi.CreateNoWindow = $true
$psi.UseShellExecute = $false
$psi.RedirectStandardOutput = $true
$psi.RedirectStandardError = $true
$psi.FileName = 'robocopy'
$psi.Arguments = #("$HomeDirectory $NewHomeDirectory /MIR /XF desktop.ini /XD VDI /R:0 /W:0 /s /v /np")
$process = New-Object System.Diagnostics.Process
$process.StartInfo = $psi
[void]$process.Start()
do
{
$process.StandardOutput.ReadLine()
}
while (!$process.HasExited)

Here is a modification to paul's answer, hopefully it addresses the truncated output. i did a test using a failure and did not see truncation.
function Start-ProcessWithOutput
{
param ([string]$Path,[string[]]$ArgumentList)
$Output = New-Object -TypeName System.Text.StringBuilder
$Error = New-Object -TypeName System.Text.StringBuilder
$psi = New-object System.Diagnostics.ProcessStartInfo
$psi.CreateNoWindow = $true
$psi.UseShellExecute = $false
$psi.RedirectStandardOutput = $true
$psi.RedirectStandardError = $true
$psi.FileName = $Path
if ($ArgumentList.Count -gt 0)
{
$psi.Arguments = $ArgumentList
}
$process = New-Object System.Diagnostics.Process
$process.StartInfo = $psi
[void]$process.Start()
do
{
if (!$process.StandardOutput.EndOfStream)
{
[void]$Output.AppendLine($process.StandardOutput.ReadLine())
}
if (!$process.StandardError.EndOfStream)
{
[void]$Error.AppendLine($process.StandardError.ReadLine())
}
Start-Sleep -Milliseconds 10
} while (!$process.HasExited)
#read remainder
while (!$process.StandardOutput.EndOfStream)
{
#write-verbose 'read remaining output'
[void]$Output.AppendLine($process.StandardOutput.ReadLine())
}
while (!$process.StandardError.EndOfStream)
{
#write-verbose 'read remaining error'
[void]$Error.AppendLine($process.StandardError.ReadLine())
}
return #{ExitCode = $process.ExitCode; Output = $Output.ToString(); Error = $Error.ToString(); ExitTime=$process.ExitTime}
}
$p = Start-ProcessWithOutput "C:\Program Files\7-Zip\7z.exe" -ArgumentList "x","-y","-oE:\PowershellModules",$NewModules.FullName -verbose
$p.ExitCode
$p.Output
$p.Error
the 10ms sleep is to avoid spinning cpu when nothing to read.

I was getting the deadlock scenario mentioned by Ash using Justin's solution. Modified the script accordingly to subscribe to async event handlers to get the output and error text which accomplishes the same thing but avoids the deadlock condition.
Seemed to resolve the deadlock issue in my testing without altering the return data.
# Define global variables used in the Start-ProcessWithOutput function.
$global:processOutputStringGlobal = ""
$global:processErrorStringGlobal = ""
# Launch an executable and return the exitcode, output text, and error text of the process.
function Start-ProcessWithOutput
{
# Function requires a path to an executable and an optional list of arguments
param (
[Parameter(Mandatory=$true)] [string]$ExecutablePath,
[Parameter(Mandatory=$false)] [string[]]$ArgumentList
)
# Reset our global variables to an empty string in the event this process is called multiple times.
$global:processOutputStringGlobal = ""
$global:processErrorStringGlobal = ""
# Create the Process Info object which contains details about the process. We tell it to
# redirect standard output and error output which will be collected and stored in a variable.
$ProcessStartInfoObject = New-object System.Diagnostics.ProcessStartInfo
$ProcessStartInfoObject.FileName = $ExecutablePath
$ProcessStartInfoObject.CreateNoWindow = $true
$ProcessStartInfoObject.UseShellExecute = $false
$ProcessStartInfoObject.RedirectStandardOutput = $true
$ProcessStartInfoObject.RedirectStandardError = $true
# Add the arguments to the process info object if any were provided
if ($ArgumentList.Count -gt 0)
{
$ProcessStartInfoObject.Arguments = $ArgumentList
}
# Create the object that will represent the process
$Process = New-Object System.Diagnostics.Process
$Process.StartInfo = $ProcessStartInfoObject
# Define actions for the event handlers we will subscribe to in a moment. These are checking whether
# any data was sent by the event handler and updating the global variable if it is not null or empty.
$ProcessOutputEventAction = {
if ($null -ne $EventArgs.Data -and $EventArgs.Data -ne ""){
$global:processOutputStringGlobal += "$($EventArgs.Data)`r`n"
}
}
$ProcessErrorEventAction = {
if ($null -ne $EventArgs.Data -and $EventArgs.Data -ne ""){
$global:processErrorStringGlobal += "$($EventArgs.Data)`r`n"
}
}
# We need to create an event handler for the Process object. This will call the action defined above
# anytime that event is triggered. We are looking for output and error data received by the process
# and appending the global variables with those values.
Register-ObjectEvent -InputObject $Process -EventName "OutputDataReceived" -Action $ProcessOutputEventAction
Register-ObjectEvent -InputObject $Process -EventName "ErrorDataReceived" -Action $ProcessErrorEventAction
# Process starts here
[void]$Process.Start()
# This sets up an asyncronous task to read the console output from the process, which triggers the appropriate
# event, which we setup handlers for just above.
$Process.BeginErrorReadLine()
$Process.BeginOutputReadLine()
# Wait for the process to exit.
$Process.WaitForExit()
# We need to wait just a moment so the async tasks that are reading the output of the process can catch
# up. Not having this sleep here can cause the return values to be empty or incomplete. In my testing,
# it seemed like half a second was enough time to always get the data, but you may need to adjust accordingly.
Start-Sleep -Milliseconds 500
# Return an object that contains the exit code, output text, and error text.
return #{
ExitCode = $Process.ExitCode;
OutputString = $global:processOutputStringGlobal;
ErrorString = $global:processErrorStringGlobal;
ExitTime = $Process.ExitTime
}
}