We Keep Coding

Spring WebFlux consumer to sink

Here is a simple spring boot application:
#SpringBootApplication
#RestController
public class ReactiveApplication {
static Flux<String> fluxString;
static volatile Queue<String> queue = new ConcurrentLinkedQueueProxy();
private static class ConcurrentLinkedQueueProxy extends ConcurrentLinkedQueue<String> {
private static final long serialVersionUID = 1L;
#Override
public boolean add(String e) {
synchronized (this) {
notify();
}
return super.add(e);
}
#Override
public String poll() {
synchronized (this) {
if(isEmpty()) {
try {
wait();
} catch (InterruptedException ex) {}
}
}
return super.peek() == null ? "" : super.poll();
}
}
static Consumer<String> consumer = str -> queue.add(str);
public static void main(String[] args) throws InterruptedException {
SpringApplication.run(ReactiveApplication.class, args);
}
static {
for(int i = 0; i < 10; i++)
queue.add("testData " + i + " ");
}
#GetMapping(value = "/", produces = MediaType.APPLICATION_STREAM_JSON_VALUE)
public Flux<String> home() {
Scheduler sch = Schedulers.newParallel("parallel-sch", 1);
List<String> list = new ArrayList<>(queue);
queue.removeAll(queue);
fluxString = Flux.<String>create(sink -> {
sink.onRequest(n -> {
for(int i = 0; i < n; i++) {
sink.next(queue.poll());
}
}).onCancel(() -> sch.dispose());
}).log().subscribeOn(sch).mergeWith(Flux.<String>fromIterable(list));
return fluxString;
}
#GetMapping("/add")
public String add( #RequestParam String s) {
consumer.accept(s);
return s;
}
}
So basically this application creates a String stream. Visiting / will grab all the string present queue and then merge anything that is added from /add resource(ignore the "Safe Methods Must be Idempotent" thing).
What I feel is strange is that when I move public static void main(...) to line 1, the application starts to misbehave and adding new values to /add doesn't have any effect. I think there must be something interesting going on that is making application misbehave. Any explaination?

I ended up using this which works great:
#SpringBootApplication
#RestController
public class ReactiveApplication {
private static BlockingQueue<String> queue = new ArrayBlockingQueue<>(1000);
private static Consumer<String> consumer = str -> {
try { queue.put(str); }
catch (InterruptedException e) {}
};
static {
for (int i = 0; i < 10; i++) queue.add("testData " + i + " ");
}
public static void main(String[] args) {
SpringApplication.run(ReactiveApplication.class, args);
}
#GetMapping(value = "/", produces = MediaType.APPLICATION_STREAM_JSON_VALUE)
public Flux<String> home() {
final Scheduler sch = Schedulers.newSingle("async-flux");
return Flux.<String>generate(sink -> {
try { sink.next(queue.take()); }
catch (InterruptedException e) { }
}).log().subscribeOn(sch);
}
#GetMapping("/add")
public String add(#RequestParam String s) {
consumer.accept(s);
return s;
}
}

Stop record thread witn commit results in mongodb

There is a Put method:
#RequestMapping(value = "/api/v1/fias/{fileName}", method = PUT). This method parses files. Parsing occurs in a separate thread. Therefore, for the user, the method works instantly and returns the ID of the created entity in the Mongo database.
There is a Post method:
#RequestMapping (value = "/ api / v1 / fias / interrupt / {objectId}", method = POST). This method should suspend the thread that parses the file from the POST method. Can this be implemented?
I try like this:
#Bean(name = "threadPoolTaskExecutor")
public ThreadPoolTaskExecutor executorService() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(2);
executor.setMaxPoolSize(2);
executor.setQueueCapacity(500);
executor.setThreadNamePrefix("codeinside-");
executor.initialize();
return executor;
}
#RequestMapping(value = "/api/v1/fias/{fileName}", method = PUT)
public ResponseEntity<Document> update(#PathVariable(value="fileName") String fileName) throws BadParamException, NotFinishProcessException {
return ResponseEntity.status(HttpStatus.CREATED).body(fiasQueryService.updateFiasByFileName(fileName));
}
#RequestMapping(value = "/api/v1/fias/interrupt/{objectId}", method = POST)
public ResponseEntity<Document> interrupt(#PathVariable(value="objectId") String objectId) {
return ResponseEntity.status(HttpStatus.OK).body(fiasQueryService.interrupt(objectId));
}
#Service
public class FiasQueryServiceImpl implements FiasQueryService {
#Autowired
private AsyncFias asyncFias;
#Autowired
private ThreadPoolTaskExecutor executorService;
private CompletableFuture<Integer> asyncResult;
#Override
public Document updateFiasByFileName(String fileName) throws NotFinishProcessException {
String settingsPath = settingsService.getStringParam(FIAS_FILE_PATH);
File file = new File(settingsPath + "/" + fileName);
ObjectId objectId = checkAndInsertStatus(file.getName().toLowerCase());
asyncResult = asyncFias.startUpdate(file, objectId);
return new Document("_id", objectId.toString()).append("success", true);
}
#Override
public Document interrupt(String objectIdString) {
setStatus(new ObjectId(objectIdString), INTERRUPT);
asyncResult.cancel(true);
Integer cnt = null;
if (asyncResult.isCancelled()) {
ObjectId objectId = new ObjectId(objectIdString);
try {
cnt = asyncResult.get();
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
setStatus(objectId, INTERRUPT);
return new Document("success", true).append("count", cnt);
} else {
return new Document("success", false);
}
}
}
#Service
public class AsyncFias {
#Async("threadPoolTaskExecutor")
#Scope(BeanDefinition.SCOPE_PROTOTYPE)
public CompletableFuture<Integer> startUpdate(File file, ObjectId objectId) {
// a lot of code.......
ClientSession session = mongo.startSession(ClientSessionOptions.builder().causallyConsistent(true).build());
MongoDatabase db = getDb(collectionInfo);
MongoCollection<Document> collection =
db.getCollection(collectionInfo.getCollectionName());
collection.insertMany(session, dbObjects);
session.close();
// a lot of code.......
return CompletableFuture.completedFuture(count);
}
}
But I get NPE in the line asyncResult.cancel (true);
I also tried to stop the workflow in this way: executorService.shutdown ();
But in this case, the records that should have been recorded by the time the stream stopped were rolled back. How can I stop the recording stream so that the currently recorded recordings are saved?

I changed the startUpdate method:
#Service
public class AsyncFias {
private static final Logger log = LoggerFactory.getLogger(AsyncFias.class);
private final FiasFileService fiasFileService;
private final MongoDBService mongoDBService;
private AtomicBoolean inProgress = new AtomicBoolean(false);
private AtomicInteger count = new AtomicInteger(0);
AsyncFias(FiasFileService fiasFileService, MongoDBService mongoDBService) {
this.fiasFileService = fiasFileService;
this.mongoDBService = mongoDBService;
}
public Integer getIncrement(){
return count.get();
}
#Async("threadPoolTaskExecutor")
#Scope(BeanDefinition.SCOPE_PROTOTYPE)
public Future<Void> startUpdate(File file) throws InterruptedException {
DbfUtilEnum utilEnum = DbfUtilEnum.fromFileName(file.getName().toLowerCase());
DbfMapper<Document> objMapper = utilEnum.getDbfMapper();
List<Document> dbObjects = fiasFileService.processFile(file, objMapper);
String collectionName = utilEnum.getCollectionName();
EntryMetaInfo metaInfo = new EntryMetaInfo(collectionName, collectionName, null, false, null);
List<List<Document>> lists = ListUtils.partition(dbObjects, 1000);
if (inProgress.compareAndSet(false, true)) {
for (List<Document> it : lists) {
//Thread.sleep(2000);
if (Thread.currentThread().isInterrupted()) {
System.out.println("Cancelled");
inProgress.set(false);
break;
}
mongoDBService.insertBulk(metaInfo, it);
count.getAndIncrement();
try {
TimeUnit.MILLISECONDS.sleep(1);
} catch (InterruptedException e) {
AsyncResult<Void> result = new AsyncResult<>(null);
result.cancel(true);
return result;
}
}
}
lists.clear();
count.set(0);
AsyncResult<Void> result = new AsyncResult<>(null);
result.cancel(true);
return result;
}
}
#Service
public class FiasQueryServiceImpl implements FiasQueryService {
private Future<Void> asyncResult;
// a lot of code
#Override
public Document updateFiasByFileName(String fileName) throws NotFinishProcessException, InterruptedException {
String settingsPath = settingsService.getStringParam(FIAS_FILE_PATH);
File file = new File(settingsPath + "/" + fileName);
ObjectId objectId = checkAndInsertStatus(file.getName().toLowerCase());
asyncResult = asyncFias.process(file);
return new Document("_id", objectId.toString()).append("success", true);
}
#Override
public Document interrupt(String objectIdString) {
asyncResult.cancel(true);
if (asyncResult.isCancelled()) {
log.info("asyncResult.isCancelled()");
ObjectId objectId = new ObjectId(objectIdString);
setStatus(objectId, INTERRUPT);
return new Document("success", true).append("count", asyncFias.getIncrement());
} else {
return new Document("success", false);
}
}
}
General conclusion: for objects of the Future type, the cancel (true) method must be called 2 times: at the time of creation of Future and at the moment of stopping the working thread.

Spring MVC blocking queue with initializing of huge library

I am using a huge library in my application. I cannot initialize the library every time a request come so I put the initialization into the configuration. I have implemented a queue and a service for managing the queue. After application is loaded I have two instances running. Problem is that it stucks and no response is getting back.
Application context xml:
<context:component-scan base-package="com.metadata.tripletws.model" />
<context:component-scan base-package="com.metadata.tripletws.service" />
<bean id="wsdService" class="com.metadata.tripletws.service.WsdService" init-method="init"></bean>
<context:component-scan base-package="com.metadata.tripletws.controller" />
Service
#Service
public class WsdService
{
public final static int MAX_THREADS = 2;
private WsdQueue queue;
public void init()
{
try {
queue = new WsdQueue(MAX_THREADS);
queue.initQueue();
} catch (InterruptedException ex) {
Logger.getLogger(WsdService.class.getName()).log(Level.SEVERE, null, ex);
}
}
public WSD getInstance() throws InterruptedException
{
return queue.dequeue();
}
public void releaseInstance(WSD instance) throws InterruptedException
{
queue.enqueue(instance);
}
}
Queue
public class WsdQueue
{
private List<WSD> queue = new LinkedList();
private int limit = 5;
public WsdQueue()
{
}
public WsdQueue(int limit)
{
this.limit = limit;
}
public void initQueue() throws InterruptedException
{
for (int i = 0; i < limit; i++) {
System.out.println("Initializing WSD nr." + i);
WSD wsd = new WSD();
wsd.Initialize();
this.enqueue(wsd);
}
}
public synchronized void enqueue(WSD item) throws InterruptedException
{
while (this.queue.size() == this.limit) {
wait();
}
if (this.queue.size() == 0) {
notifyAll();
}
System.out.println("Adding instance");
this.queue.add(item);
}
public synchronized WSD dequeue() throws InterruptedException
{
while (this.queue.size() == 0) {
wait();
}
if (this.queue.size() == this.limit) {
notifyAll();
}
System.out.println("Taking instance");
return this.queue.remove(0);
}
}
and here is the controller
Controller
#RequestMapping(value = "/api")
public class ApiController
{
#Autowired
private WsdService wsdService;
#RequestMapping(value = "/wordnet", method = RequestMethod.POST, produces = MediaType.APPLICATION_JSON_VALUE)
public #ResponseBody
ArrayList<LexicalWord> getWordsFromSentence(#RequestBody String sentence)
{
ArrayList<LexicalWord> results = new ArrayList<>();
try {
WSD wsdInstance = wsdService.getInstance();
List<CWSDResult> list = wsdInstance.AnanlyseSentenceToList(sentence);
for (CWSDResult res : list) {
if (!res.hasLegalSense) {
continue;
}
String key = getHighestProbableSynsetKey(res);
if (key != null && res.senses.containsKey(key)) {
Synset synset = res.senses.get(key);
if (synset != null) {
results.add(new LexicalWord(res.token, synset.getLexFileName()));
}
}
}
wsdService.releaseInstance(wsdInstance);
} catch (InterruptedException ex) {
Logger.getLogger(ApiController.class.getName()).log(Level.SEVERE, null, ex);
}
return results;
}
}
This is the first time I am making a queue and working with threads and I appriciate any kind of help or example. There is no error, it just stucks, does nothing and no response is send back.
EDIT1:
after answers I changed the code, but no luck. Still stucks. Maybe the problem is somewhere else, but when I send a single request it works and I get the response.
#Component
public class WsdService
{
public final static int MAX_THREADS = 2;
private BlockingQueue<WSD> queue;
public WsdService()
{
queue = new ArrayBlockingQueue(MAX_THREADS);
}
#PostConstruct
public void init()
{
try {
for (int i = 0; i < MAX_THREADS; i++) {
WSD wsd = new WSD();
wsd.Initialize();
System.out.println("WSD nr. " + i + " initialized!");
queue.put(wsd);
}
} catch (InterruptedException ex) {
Logger.getLogger(WsdService.class.getName()).log(Level.SEVERE, null, ex);
}
}
public WSD getInstance() throws InterruptedException
{
return queue.take();
}
public void releaseInstance(WSD instance) throws InterruptedException
{
queue.put(instance);
}
}
and here is new applicationContext
<context:component-scan base-package="com.metadata.tripletws.model" />
<context:component-scan base-package="com.metadata.tripletws.service" />
<context:component-scan base-package="com.metadata.tripletws.controller" />
SOLUTION
So ..after lot of digging of the library I assumed it is not possible to use multithreading there. Anyway I used the blockingQueue in other service (NLP) and it is working nicely (Same as EDIT1 just the variable is different). Thanks everybody for your help.
Thanks,
Michal

So you're essentially trying to make a pool of WSDs?. You're certain that WSD isn't thread-safe?
You can safely throw your queue in the garbage, and use a correct ready made version of BlockingQueue instead. It might not necessarily solve your actual problem, but it'll be a working version of what you're attempting here.

Thread does not run

I have some places in a excel file, each of the point have a lng and lat coordinate.
Now I try to create a static Map for each point using the google map static map api.
And I have Two component, a parser and a loader.
The Parser is used to read the excel file while the loaded is used to load tiles.
And I make the loader run in a seprate Thread.
public class Parser {
private static Parser instance;
private StaticMapLoader loader;
private Parser(StaticMapLoader loader) {
this.loader = loader;
}
public synchronized static Parser getInstance(StaticMapLoader loader) {
if (instance == null) {
instance = new Parser(loader);
}
return instance;
}
public void parse(String path) {
List<Branch> result = new ArrayList<Branch>();
InputStream inp;
try {
inp = new FileInputStream(path);
Workbook wb = WorkbookFactory.create(inp);
Sheet sheet = wb.getSheetAt(0);
int rows = sheet.getLastRowNum();
for(Row r : sheet.getRows){
loader.addTask(r.type,r.name,r.x,r.y);
}
} catch (InvalidFormatException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
// Branch bc = new Branch("网点1", null, null);
return result;
}
}
Loader:
public class StaticMapLoader extends Thread {
private final static Logger log = Logger.getLogger(StaticMapLoader.class);
private List<Task> tasks = new ArrayList<Task>();
private String tilePath;
private boolean running = false;
public StaticMapLoader(String saveDir) {
this.tilePath = saveDir;
}
#Override
public void run() {
while (running) {
log.debug("run " + tasks.size());
if (tasks.size() > 0) {
Task t = tasks.get(0);
if (t != null && t.status == Status.waiting) {
tasks.remove(0);
t.status = Status.running;
downLoad(t);
}
}
}
}
private void downLoad(Task t) {
log.debug(String.format("load data for " + t.toString()));
//down tiles and save
t.status=Status.success;
}
public void addTask(String type, String name, double x, double y) {
log.debug(String.format("add task of :%s,%s", type, name));
tasks.add(new Task(type,name,x,y));
}
public void startRunning() {
running = true;
this.start();
}
public void stopRunning() {
running = false;
this.interrupt();
}
class Task {
Status status = Status.waiting;
String type, name;
double x,y;
Task(String type, String name, double x,double y) {
this.type = type;
this.name = name;
this.xian = xian;
this.x = x;
this.y = y;
}
}
enum Status {
waiting, running, fail, success
}
}
The process is rather simple, the StaticMapLoader have a field of ArrayList. While the Parser parse a record(place), it will be thrown to the list.
And the loader will iterator the list and download the data.
However I meet a strange problem here:
#Override
public void run() {
while (running) {
log.debug("run " + tasks.size());
if (tasks.size() > 0) {
Task t = tasks.get(0);
if (t != null && t.status == Status.waiting) {
tasks.remove(0);
t.status = Status.running;
downLoad(t);
}
}
}
}
The above codes runs, and I will get the logs like this:
run 1
add task of ..
run 2
add task of ...
However , if I comment the log line, the downLoad will be never called, I will get:
run 1
run 2
......
It seems that this may be caused by the Thread , do I miss anything?
BTW, the above codes ran inside the HttpServlet context, and I start them like this:
#Override
public void init() throws ServletException {
super.init();
try {
URL fileUrl = getServletContext().getResource(getInitParameter("xlsxFile"));
URL tilePath = getServletContext().getResource(getInitParameter("tilePath"));
StaticMapLoader loader = new StaticMapLoader(tilePath.getPath());
loader.startRunning();
Parser.getInstance(loader).parse(fileUrl.getPath());
} catch (MalformedURLException e) {
}
}

How to access Hibernate session from src folder?

I would like to know how to access the Service and Domains properly in this sample class placed in src/java folder
public class NewsIngestion implements Runnable {
private String str;
private int num;
private Logger log = Logger.getLogger("grails.app");
private static boolean isRunning;
private Thread t;
private WorkerJobService jobService;
private NewsService newsService;
public NewsIngestion(String s, int n)
{
jobService = new WorkerJobService();
newsService = new NewsService();
str = s;
num = n;
isRunning = false;
t = new Thread (this, "NewsIngestion");
}
public void run ()
{
while(isRunning){
try{
if(jobService.isJobEnabled("ConsumeFeedsJob") && jobService.lockJob("ConsumeFeedsJob")){
log.info("${this.class.name}: ConsumeFeedsJob started");
try{
// get all sources
List sources = (List) InvokerHelper.invokeMethod(RSSFeed.class, "list", null);
for(int i = 0; i < sources.size(); i++) {
RSSFeed s = (RSSFeed) sources.get(i);
// check if it's time to read the source
int diff = DateTimeUtil.getSecondsDateDiff(s.getLastChecked(), new Date());
if(s.getLastChecked() == null || diff >= s.getCheckInterval()){
List keyword_list = (List) InvokerHelper.invokeMethod(Keyword.class, "list", null);
for(int j = 0; j < keyword_list.size(); j++) {
String keyword = (String) keyword_list.get(j);
try{
newsService.ingestNewsFromSources(keyword, s);
}catch(Exception e){
log.error("${this.class.name}: ${e}");
}
log.debug("Completed reading feeds for ${keyword}.");
log.info("${this.class.name}: Reading feeds for '${keyword}' (${s.feedName}) took ${Float.toString(st2.getDuration())} second(s).");
}
s.setLastChecked(new Date());
InvokerHelper.invokeMethod(RSSFeed.class, "save", null);
}
log.info("${this.class.name}: Reading feeds for '${s.feedName}' for all keywords took ${Float.toString(st.getDuration())} second(s).");
}
}catch(Exception e){
log.error("${this.class.name}: Exception: ${e}");
}
log.info("${this.class.name}: ConsumeFeedsJob ended.");
// unlock job
jobService.unlockJob("ConsumeFeedsJob");
}
log.info("alfred: success");
}
catch (Exception e){
log.info("alfred exception: " + e.getMessage());
}
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
log.info(e.getMessage());
}
}
}
public void start() {
if(t == null){
t = new Thread (this, "NewsIngestion");
}
if(!isRunning){
isRunning = true;
t.start();
}
}
public void stop() {
isRunning = false;
}
public boolean isRunning() {
return isRunning;
}
}
I'm encountering this error message:
No Hibernate Session bound to thread,
and configuration does not allow
creation of non-transactional one here
Thanks.

You shouldn't instantiate the service class by yourself, but instead take the class instance from the main context
import org.codehaus.groovy.grails.commons.ApplicationHolder
def ctx = ApplicationHolder.application.mainContext
def newsService = ctx.newsService
If you're using Java
import org.codehaus.groovy.grails.commons.ApplicationHolder
public class SomeClass {
SomeService someService;
public SomeClass() {
someService = (SomeService) ApplicationHolder.getApplication().getMainContext().getBean("someService");
}
}

Consider using Spring and #Transactional annotation or AOP.