We Keep Coding

Microservice communication with python in Asynchronous manner

I want to trigger Python script from my Spring boot microservices im Asynchronous manner, SO that my Microservice will be notified once the execution of python script completes.Can any one suggest the best approach for this? appreciated if any one provide some reference to sample code.
Thanks in advance!!!
Thanks,
Sudheer

here is very good Example for something like that:
Source: http://code.activestate.com/recipes/511454-an-asynchronous-http-server-with-cgi-support/
import SimpleAsyncServer
# =============================================================
# An implementation of the HTTP protocol, supporting persistent
# connections and CGI
# =============================================================
import sys
import os
import traceback
import datetime
import mimetypes
import urlparse
import urllib
import cStringIO
class HTTP(SimpleAsyncServer.ClientHandler):
# parameters to override if necessary
root = os.getcwd() # the directory to serve files from
cgi_directories = ['/cgi-bin'] # subdirectories for cgi scripts
logging = True # print logging info for each request ?
blocksize = 2 << 16 # size of blocks to read from files and send
def request_complete(self):
"""In the HTTP protocol, a request is complete if the "end of headers"
sequence ('\r\n\r\n') has been received
If the request is POST, stores the request body in a StringIO before
returning True"""
terminator = self.incoming.find('\r\n\r\n')
if terminator == -1:
return False
lines = self.incoming[:terminator].split('\r\n')
self.requestline = lines[0]
try:
self.method,self.url,self.protocol = lines[0].strip().split()
except:
self.method = None # indicates bad request
return True
# put request headers in a dictionary
self.headers = {}
for line in lines[1:]:
k,v = line.split(':',1)
self.headers[k.lower().strip()] = v.strip()
# persistent connection
close_conn = self.headers.get("connection","")
if (self.protocol == "HTTP/1.1"
and close_conn.lower() == "keep-alive"):
self.close_when_done = False
# parse the url
scheme,netloc,path,params,query,fragment = urlparse.urlparse(self.url)
self.path,self.rest = path,(params,query,fragment)
if self.method == 'POST':
# for POST requests, read the request body
# its length must be specified in the content-length header
content_length = int(self.headers.get('content-length',0))
body = self.incoming[terminator+4:]
# request is incomplete if not all message body received
if len(body)<content_length:
return False
f_body = cStringIO.StringIO(body)
f_body.seek(0)
sys.stdin = f_body # compatibility with CGI
return True
def make_response(self):
"""Build the response : a list of strings or files"""
if self.method is None: # bad request
return self.error_resp(400,'Bad request : %s' %self.requestline)
resp_headers, resp_body, resp_file = '','',None
if not self.method in ['GET','POST','HEAD']:
return self.err_resp(501,'Unsupported method (%s)' %self.method)
else:
file_name = self.file_name = self.translate_path()
if not os.path.exists(file_name):
return self.err_resp(404,'File not found')
elif self.managed():
response = self.mngt_method()
else:
ext = os.path.splitext(file_name)[1]
c_type = mimetypes.types_map.get(ext,'text/plain')
resp_line = "%s 200 Ok\r\n" %self.protocol
size = os.stat(file_name).st_size
resp_headers = "Content-Type: %s\r\n" %c_type
resp_headers += "Content-Length: %s\r\n" %size
resp_headers += '\r\n'
if self.method == "HEAD":
resp_string = resp_line + resp_headers
elif size > HTTP.blocksize:
resp_string = resp_line + resp_headers
resp_file = open(file_name,'rb')
else:
resp_string = resp_line + resp_headers + \
open(file_name,'rb').read()
response = [resp_string]
if resp_file:
response.append(resp_file)
self.log(200)
return response
def translate_path(self):
"""Translate URL path into a path in the file system"""
return os.path.join(HTTP.root,*self.path.split('/'))
def managed(self):
"""Test if the request can be processed by a specific method
If so, set self.mngt_method to the method used
This implementation tests if the script is in a cgi directory"""
if self.is_cgi():
self.mngt_method = self.run_cgi
return True
return False
def is_cgi(self):
"""Test if url in a cgi directory"""
for path in self.cgi_directories:
if self.path.startswith(path):
rest = self.path[len(path):]
if not rest or rest.startswith('/'):
return True
return False
def run_cgi(self):
# set CGI environment variables
self.make_cgi_env()
# redirect print statements to a cStringIO
save_stdout = sys.stdout
sys.stdout = cStringIO.StringIO()
# run the script
try:
execfile(self.file_name)
except:
sys.stdout = cStringIO.StringIO()
sys.stdout.write("Content-type:text/plain\r\n\r\n")
traceback.print_exc(file=sys.stdout)
response = sys.stdout.getvalue()
if self.method == "HEAD":
# for HEAD request, don't send message body even if the script
# returns one (RFC 3875)
head_lines = []
for line in response.split('\n'):
if not line:
break
head_lines.append(line)
response = '\n'.join(head_lines)
sys.stdout = save_stdout # restore sys.stdout
# close connection in case there is no content-length header
self.close_when_done = True
resp_line = "%s 200 Ok\r\n" %self.protocol
return [resp_line + response]
def make_cgi_env(self):
"""Set CGI environment variables"""
env = {}
env['SERVER_SOFTWARE'] = "AsyncServer"
env['SERVER_NAME'] = "AsyncServer"
env['GATEWAY_INTERFACE'] = 'CGI/1.1'
env['DOCUMENT_ROOT'] = HTTP.root
env['SERVER_PROTOCOL'] = "HTTP/1.1"
env['SERVER_PORT'] = str(self.server.port)
env['REQUEST_METHOD'] = self.method
env['REQUEST_URI'] = self.url
env['PATH_TRANSLATED'] = self.translate_path()
env['SCRIPT_NAME'] = self.path
env['PATH_INFO'] = urlparse.urlunparse(("","","",self.rest[0],"",""))
env['QUERY_STRING'] = self.rest[1]
if not self.host == self.client_address[0]:
env['REMOTE_HOST'] = self.host
env['REMOTE_ADDR'] = self.client_address[0]
env['CONTENT_LENGTH'] = str(self.headers.get('content-length',''))
for k in ['USER_AGENT','COOKIE','ACCEPT','ACCEPT_CHARSET',
'ACCEPT_ENCODING','ACCEPT_LANGUAGE','CONNECTION']:
hdr = k.lower().replace("_","-")
env['HTTP_%s' %k.upper()] = str(self.headers.get(hdr,''))
os.environ.update(env)
def err_resp(self,code,msg):
"""Return an error message"""
resp_line = "%s %s %s\r\n" %(self.protocol,code,msg)
self.close_when_done = True
self.log(code)
return [resp_line]
def log(self,code):
"""Write a trace of the request on stderr"""
if HTTP.logging:
date_str = datetime.datetime.now().strftime('[%d/%b/%Y %H:%M:%S]')
sys.stderr.write('%s - - %s "%s" %s\n' %(self.host,
date_str,self.requestline,code))
if __name__=="__main__":
# launch the server on the specified port
port = 8081
print "Asynchronous HTTP server running on port %s" %port
print "Press Ctrl+C to stop"
server = SimpleAsyncServer.Server('', port)
HTTP.logging = False
try:
SimpleAsyncServer.loop(server,HTTP)
except KeyboardInterrupt:
for s in server.client_handlers:
server.close_client(s)
print 'Ctrl+C pressed. Closing'

One way of doing asynchronous execution in Python would be to use the celery framework. It's really simple to install and setup (basically just pip install), and the semantic for calling a method asynchronously is super clean.
If you are not too bound to Spring you could even switch to the pymacaron microservice framework (basically a flask app that uses swagger to spawn a REST API). Pymacaron (http://pymacaron.com/) supports asynchronous calls by default, via the pymacaron-async module. See the examples here: http://pymacaron.com/async.html

How to do "git filter-branch --subdirectory <folderName>" with JGit

I have a lot of maven multi-module projects (around 50). Each of them with 1 to 4 sub-modules. Each multi-module project lies in a separate git repository. Due to an issue I need to split the modules into separate repositories.
I have a shell script to move a folder (a maven module) from Repo A to Repo B preserving its git history.
#!/bin/sh
# moves a folder from one git repository to another
# moveFolder <absolute repository one path> <repository one folder> <absolute repository two path>
echo "Moving "$2" from Repository: "$1" to Repository:"$3
# prepare repository one
cd $1
git clean -f -d -x
git checkout -b tmpBranch
git filter-branch -f --subdirectory-filter $2 HEAD
mkdir $2
mv * $2
git add .
git commit -a -m "CIRCLE-524: Moved "$2" into new repository"
#import in repository two
cd $3
git remote add repositoryOne $1
git pull repositoryOne tmpBranch
git remote rm repositoryOne
#cleanup
cd $1
git checkout master
git branch -D tmpBranch
When I have to do this for only a few projects i could do it manually. But because I have like 50 multi-module projects I like to automate this.
I could use a ProcessBuilder to execute this script. But the process of splitting all modules will be done on a windows machine.
(When I do this manually I'm using the Git Bash)
So I found JGit to automate my whole process with java. My problem is with git filter-branch -f --subdirectory-filder $2 HEAD
So far i have the following.
public static void revWalk(final Git repo) throws NoWorkTreeException, GitAPIException, MissingObjectException, IncorrectObjectTypeException, IOException {
final Ref tmpBranch = checkOutBranch(repo, "tmpBranch");
final Repository repository = repo.getRepository();
try (RevWalk revWalk = new RevWalk(repository)) {
final ObjectId commitId = repository.resolve(tmpBranch.getName());
revWalk.markStart(revWalk.parseCommit(commitId));
for (final RevCommit commit : revWalk) {
System.out.println("Time: " + commit.getAuthorIdent().getWhen() + " Message: " + commit.getShortMessage());
}
}
}
AND
public static void fileWalk(final Git repo, final String modulePath) throws IOException {
final Ref head = repo.getRepository().findRef("HEAD");
final RevWalk walk = new RevWalk(repo.getRepository());
final RevCommit commit = walk.parseCommit(head.getObjectId());
final RevTree tree = commit.getTree();
System.out.println("Having tree: " + tree);
final TreeWalk treeWalk = new TreeWalk(repo.getRepository());
treeWalk.addTree(tree);
treeWalk.setRecursive(true);
treeWalk.setFilter(PathFilterGroup.createFromStrings(modulePath));
while (treeWalk.next()) {
System.out.println("found: " + treeWalk.getPathString());
}
walk.close();
treeWalk.close();
}
revWalk checks out a new tmpBranch to work on and then lists ALL Commits with Time and Message.
fileWalk takes the HEAD and lists ALL files that match the given filter by the modulePath
I guess to achieve the git filter-branch -f --subdirectory-filder $2 HEAD I have to combine both ways but to be honest I have no clue how to do that.
Could you please point me into the right direction and give me some code examples?

To answer my own question:
I used the ProcessBuilder to execute my given script and I did the following.
private void executeScript(final String pathToGitBash, final File scriptFileToExecute, final String... arguments) throws IOException, InterruptedException {
final int offset = 4;
final String[] command = new String[arguments.length + offset];
command[0] = pathToGitBash;
command[1] = "--login";
command[2] = "-i";
command[3] = scriptFileToExecute.getAbsolutePath();
for (int i = 0; i < arguments.length; i++) {
command[i + offset] = arguments[i];
}
final ProcessBuilder builder = new ProcessBuilder(command);
builder.redirectErrorStream(true);
final Process process = builder.start();
IOUtils.copy(process.getInputStream(), System.out);
IOUtils.copy(process.getErrorStream(), System.err);
switch (process.waitFor()) {
case 1:
LOGGER.error("Parameters for scripts are not correct.");
break;
case 0:
LOGGER.info("Script seemed to execute properly");
break;
default:
LOGGER.info("Unknown returnCode. Script finished with: {}", process.exitValue());
}
}
To execute it withing Gitbash I needed to provide the gitbash/sh.exe and the parameters "--login -i"

What is configuartion required to get data from object storage by SWIFT in Spark

I go through document but still it is very much confusing how to get data from swift.
I configured swift in my one linux machine. By using below command I am able to get container list,
swift -A https://acc.objectstorage.softlayer.net/auth/v1.0/ -U
username -K passwordkey list
I seen many blog for blumix(https://console.ng.bluemix.net/docs/services/AnalyticsforApacheSpark/index-gentopic1.html#genTopProcId2) and written the below code
sc.textFile("swift://container.myacct/file.xml")
I am looking to integrate in java spark. Where need to configure object storage credential in java code. Is there any sample code or blog?

This notebook illustrates a number of ways to load data using the Scala language. Scala runs on the JVM. Java and Scala classes can be freely mixed, no matter whether they reside in different projects or in the same. Looking at the mechanics of how Scala code interacts with Openstack Swift object storage should help guide you to craft a Java equivalent.
From the above notebook, here are some steps illustrating how to configure and extract data from an Openstack Swift Object Storage instance using the Stocator library using the Scala language. The swift url decomposes into:
swift2d :// container . myacct / filename.extension
^ ^ ^ ^
stocator name of namespace object storage
protocol container filename
Imports
import org.apache.spark.SparkContext
import scala.util.control.NonFatal
import play.api.libs.json.Json
val sqlctx = new SQLContext(sc)
val scplain = sqlctx.sparkContext
Sample Creds
// #hidden_cell
var credentials = scala.collection.mutable.HashMap[String, String](
"auth_url"->"https://identity.open.softlayer.com",
"project"->"object_storage_3xxxxxx3_xxxx_xxxx_xxxx_xxxxxxxxxxxx",
"project_id"->"6xxxxxxxxxx04fxxxxxxxxxx6xxxxxx7",
"region"->"dallas",
"user_id"->"cxxxxxxxxxxaxxxxxxxxxx1xxxxxxxxx",
"domain_id"->"cxxxxxxxxxxaxxyyyyyyxx1xxxxxxxxx",
"domain_name"->"853255",
"username"->"Admin_cxxxxxxxxxxaxxxxxxxxxx1xxxxxxxxx",
"password"->"""&M7372!FAKE""",
"container"->"notebooks",
"tenantId"->"undefined",
"filename"->"file.xml"
)
Helper Method
def setRemoteObjectStorageConfig(name:String, sc: SparkContext, dsConfiguration:String) : Boolean = {
try {
val result = scala.util.parsing.json.JSON.parseFull(dsConfiguration)
result match {
case Some(e:Map[String,String]) => {
val prefix = "fs.swift2d.service." + name
val hconf = sc.hadoopConfiguration
hconf.set("fs.swift2d.impl","com.ibm.stocator.fs.ObjectStoreFileSystem")
hconf.set(prefix + ".auth.url", e("auth_url") + "/v3/auth/tokens")
hconf.set(prefix + ".tenant", e("project_id"))
hconf.set(prefix + ".username", e("user_id"))
hconf.set(prefix + ".password", e("password"))
hconf.set(prefix + "auth.method", "keystoneV3")
hconf.set(prefix + ".region", e("region"))
hconf.setBoolean(prefix + ".public", true)
println("Successfully modified sparkcontext object with remote Object Storage Credentials using datasource name " + name)
println("")
return true
}
case None => println("Failed.")
return false
}
}
catch {
case NonFatal(exc) => println(exc)
return false
}
}
Load the Data
val setObjStor = setRemoteObjectStorageConfig("sparksql", scplain, Json.toJson(credentials.toMap).toString)
val data_rdd = scplain.textFile("swift2d://notebooks.sparksql/" + credentials("filename"))
data_rdd.take(5)

how to use property=value in CLI commons Library

I am trying to use the OptionBuilder.withArgName( "property=value" )
If my Option is called status and my command line was:
--status p=11 s=22
It only succeeds to identify the first argument which is 11 and it fails to identify the second argument...
Option status = OptionBuilder.withLongOpt("status")
.withArgName( "property=value" )
.hasArgs(2)
.withValueSeparator()
.withDescription("Get the status")
.create('s');
options.addOption(status);
Thanks for help in advance

You can access to passed properties using simple modification of passed command line options
--status p=11 --status s=22
or with your short syntax
-s p=11 -s s=22
In this case you can access to your properties simply with code
if (cmd.hasOption("status")) {
Properties props = cmd.getOptionProperties("status");
System.out.println(props.getProperty("p"));
System.out.println(props.getProperty("t"));
}
If you need to use your syntax strictly, you can manually parse your property=value pairs.
In this case you should remove .withValueSeparator() call, and then use
String [] propvalues = cmd.getOptionValues("status");
for (String propvalue : propvalues) {
String [] values = propvalue.split("=");
System.out.println(values[0] + " : " + values[1]);
}

Scala - Fails to execute a process through terminal in a particular scenario

I'm using graphviz to generate graphs based on the messages passed in a scala program.
To invoke the graphviz application from inside the scala program, I'm using the exec() method (similar to Java). It successfully executed the command and created the graph when I used the below code snippet:
var cmd: String = "dot -Tpng Graph.dot -o Graph.png"
var run: Runtime = Runtime.getRuntime() ;
var pr: Process = run.exec(cmd) ;
However It fails to execute after changing the path of the input and output files (I just included a directory inside which the input file and output file resides as shown below)
def main(args: Array[String]): Unit = {
var DirectoryName: String = "Logs"
var GraphFileName: String = DirectoryName + File.separator + "Graph.dot"
val GraphFileObj: File = new File(GraphFileName)
// var cmd: String = "dot -Tpng Graph.dot -o Graph.png"
var cmd: String = "dot -Tpng \"" + GraphFileObj.getAbsolutePath + "\" -o \"" + DirectoryName + File.separator + "Graph.png\"" ;
println(cmd)
var run: Runtime = Runtime.getRuntime() ;
var pr: Process = run.exec(cmd) ;
}
The same command when executed through terminal gives proper output. Can you please help me to find what I'm missing?

exec is not a shell...e.g. quoting won't work as you expect, and thus your path (which may contain spaces, etc) will not be processed as you expect. The command will be broken apart using StringTokenizer, and your literal quotes will be...well..literal.
Use the form of exec that takes an array instead, so you can tokenize the command correctly.
val args = Array[String]("dot", "-Tpng", GraphFileObj.getAbsolutePath, ...);
run.exec(args)