So I have 3 threads,
A thread which downloads audio and the audio schedule object from the internet.
A thread which plays the audio according to the audio schedule.
And a web socket "notification" listener that listens for messages that say we have to download new audio and schedule as our current one is outdated.
The program flow is as follows:
On application startup: The ScheduleDownloader starts,downloads audio and schedule file.Once completed it needs to tell the audio player "hey the files are ready and here is the schedule" and it doesnt need to do anything for now
The audio player starts and continuously loops with no exit condition.
The web socket listener starts,when it gets a message.It should tell the schedule downloader "You need to start again as there is new files you need to download",it doesnt need to send any data to the schedule downloaded,just start it up again.The music should remain playing.Once it is done it should now restart the audio player thread with the new schedule.
Here is what I have so far,I am not sure how to get ScheduleDownloader to tell AudioPlayer "the files are ready and you need to start,here is the schedule" or "you need to restart with the new schedule,here it is" or how to get the listener to say "ScheduleDownloader you need to start again"
public class ScheduleDownloader extends Thread {
private Thread t;
private String threadName;
String username;
String password;
public ScheduleDownloader(String username,String password,String threadName){
this.username = username;
this.password = password;
this.threadName= threadName;
}
public void start () {
System.out.println("Starting " + threadName );
if (t == null) {
t = new Thread (this, threadName);
t.start ();
}}
public void run() {
try {
Schedule schedule= null;
while(schedule == null){
System.out.println("Searching for schedule");
schedule= getTodaysSchedule();
}
System.out.println("Schedule Found");
boolean result = false;
while(result == false){
result = downloadFiles(schedule);
}
System.out.println("Files Downloaded");
} catch (IOException e) {
e.printStackTrace();
}
}
public Schedule getTodaysSchedule() throws IOException {
Schedule schedule = null;
CredentialsProvider provider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials
= new UsernamePasswordCredentials(username,password);
provider.setCredentials(AuthScope.ANY, credentials);
String url = "http://localhost:5000/api/schedule/today";
HttpClient httpClient = HttpClientBuilder.create().setDefaultCredentialsProvider(provider).build(); //Use this instead
HttpGet request = new HttpGet(url);
HttpResponse response = httpClient.execute(request);
//read content response body
if (response.getStatusLine().getStatusCode() != 200) {
System.out.println("sorry error:" + response.getStatusLine().getStatusCode());
} else {
BufferedReader rd = new BufferedReader(
new InputStreamReader(response.getEntity().getContent()));
StringBuffer result = new StringBuffer();
String line = "";
while ((line = rd.readLine()) != null) {
result.append(line);
}
//change json response to java objects
Gson gson = new Gson();
schedule = gson.fromJson(String.valueOf(result),Schedule.class);
}
return schedule;
}
public static boolean downloadFiles(Schedule schedule) {
//get the music
for(int i =0;i<schedule.getMusicScheduleItems().size();i++){
downloadOneFile("shoutloudaudio","music/" +
schedule.getMusicScheduleItems().get(i).getMusic().getId()+
"-music.wav");
}
//get the advertisements
for(int i =0;i<schedule.getAdvertisementScheduleItems().size();i++){
downloadOneFile("shoutloudaudio","advertisements/" +
schedule.getAdvertisementScheduleItems().get(i).getAdvertisement().getId()+
"-advertisement.wav");
}
return true;
}
public static boolean downloadOneFile(String bucketName,String key) {
if( new File(key.split("/")[1]).isFile()){
//check if we have it already and dont need to download it
System.out.println(key + " alraeady exits");
return true;
}
AWSCredentials awsCredentials = new BasicAWSCredentials(
"removed",
"removed"
);
AmazonS3 s3client = AmazonS3ClientBuilder
.standard()
.withCredentials(new AWSStaticCredentialsProvider(awsCredentials))
.withRegion(Regions.EU_WEST_1)
.build();
S3Object s3object = s3client.getObject(bucketName, key);
S3ObjectInputStream inputStream = s3object.getObjectContent();
InputStream reader = new BufferedInputStream(
inputStream);
File file = new File(key.split("/")[1]);//save the file as whats after the / in key
OutputStream writer = null;
try {
writer = new BufferedOutputStream(new FileOutputStream(file));
} catch (FileNotFoundException e) {
e.printStackTrace();
return false;
}
int read = -1;
try {
while ((read = reader.read()) != -1) {
writer.write(read);
}
writer.flush();
writer.close();
}catch(IOException e){
e.printStackTrace();
return false;
}
return true;
}
}
AudioPlayer
public class AudioPlayer extends Thread {
Long currentFrameMusic;
Long currentFrameAdvertisement;
Clip clipMusic;
Clip clipAdvertisement;
private Thread t;
// current status of clip
String statusMusic;
String statusAdvertisement;
static AudioInputStream musicInputStream;
static AudioInputStream advertisementInputStream;
static String filePath;
Schedule schedule;
// constructor to initialize streams and clip
public AudioPlayer(Schedule schedule)
throws UnsupportedAudioFileException,
IOException, LineUnavailableException
{
//setup audio stream for music first
// create AudioInputStream object
this.schedule = schedule;
appendMusicFiles(schedule);
// create clip reference
clipMusic = AudioSystem.getClip();
// open audioInputStream to the clip
clipMusic.open(musicInputStream);
clipMusic.loop(Clip.LOOP_CONTINUOUSLY);
}
public void run(){
playMusic();
try {
checkShouldWePlayAnAdvertisement();
} catch (IOException e) {
e.printStackTrace();
} catch (UnsupportedAudioFileException e) {
e.printStackTrace();
} catch (LineUnavailableException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public void start(){
t = new Thread (this, "AudioPlayerThread");
t.start ();
}
public void start2() throws IOException, UnsupportedAudioFileException, LineUnavailableException, InterruptedException {
playMusic();
checkShouldWePlayAnAdvertisement();
}
public void playMusic()
{
//start the clip
clipMusic.start();
statusMusic = "play";
}
// Method to pause the audio
public void pauseMusic()
{
if (statusMusic.equals("paused"))
{
System.out.println("audio is already paused");
return;
}
this.currentFrameMusic =
this.clipMusic.getMicrosecondPosition();
clipMusic.stop();
statusMusic = "paused";
System.out.println("pausing music");
}
// Method to resume the audio
public void resumeAudioMusic() throws UnsupportedAudioFileException,
IOException, LineUnavailableException
{
if (statusMusic.equals("play"))
{
System.out.println("Audio is already "+
"being played");
return;
}
clipMusic.close();
resetAudioStreamMusic();
clipMusic.setMicrosecondPosition(currentFrameMusic);
System.out.println("resuming music");
this.playMusic();
}
// Method to restart the audio
public void restartMusic() throws IOException, LineUnavailableException,
UnsupportedAudioFileException
{
clipMusic.stop();
clipMusic.close();
resetAudioStreamMusic();
currentFrameMusic = 0L;
clipMusic.setMicrosecondPosition(0);
this.playMusic();
}
// Method to stop the audio
public void stopMusic() throws UnsupportedAudioFileException,
IOException, LineUnavailableException
{
currentFrameMusic = 0L;
clipMusic.stop();
clipMusic.close();
}
public void resetAudioStreamMusic() throws UnsupportedAudioFileException, IOException,
LineUnavailableException
{
clipMusic = AudioSystem.getClip();
appendMusicFiles(schedule);
// open audioInputStream to the clip
clipMusic.open(musicInputStream);
clipMusic.loop(Clip.LOOP_CONTINUOUSLY);
}
public static void appendMusicFiles(Schedule schedule) throws IOException, UnsupportedAudioFileException {
//add the first audio file to stream
AudioInputStream appendedFiles = AudioSystem.getAudioInputStream(
new File(schedule.getMusicScheduleItems().get(0).getMusic()
.getId() + "-music.wav"));
//loop through an combine
for(int i =1;i<schedule.getMusicScheduleItems().size();i++){
File file= new File(schedule.getMusicScheduleItems().get(i).getMusic()
.getId() + "-music.wav");
AudioInputStream toBeAppended = AudioSystem.getAudioInputStream(file);
//append them
appendedFiles =
new AudioInputStream(
new SequenceInputStream(appendedFiles, toBeAppended),
appendedFiles.getFormat(),
appendedFiles.getFrameLength() + toBeAppended.getFrameLength());
}
musicInputStream = appendedFiles;
}
//advertisement methods
public void playAdvertisements() throws LineUnavailableException, IOException, InterruptedException {
clipAdvertisement = AudioSystem.getClip();
// open audioInputStream to the clip
clipAdvertisement.open(advertisementInputStream);
System.out.println(clipAdvertisement.getMicrosecondLength());
//start the clip
clipAdvertisement.start();
Thread.sleep(clipAdvertisement.getMicrosecondLength() / 1000);
statusAdvertisement = "play";
System.out.println("playing advertisements");
}
// Method to pause the audio
public void pauseAdvertisements()
{
if (statusAdvertisement.equals("paused"))
{
System.out.println("audio is already paused");
return;
}
this.currentFrameAdvertisement =
this.clipAdvertisement.getMicrosecondPosition();
clipAdvertisement.stop();
statusAdvertisement = "paused";
}
// Method to resume the audio
public void resumeAudioAdvertisement() throws UnsupportedAudioFileException,
IOException, LineUnavailableException, InterruptedException {
if (statusAdvertisement.equals("play"))
{
System.out.println("Audio is already "+
"being played");
return;
}
clipAdvertisement.close();
resetAudioStreamAdvertisement();
clipAdvertisement.setMicrosecondPosition(currentFrameMusic);
this.playAdvertisements();
}
// Method to restart the audio
public void restartAdvertisement() throws IOException, LineUnavailableException,
UnsupportedAudioFileException, InterruptedException {
clipAdvertisement.stop();
clipAdvertisement.close();
resetAudioStreamAdvertisement();
currentFrameAdvertisement = 0L;
clipAdvertisement.setMicrosecondPosition(0);
this.playAdvertisements();
}
// Method to stop the audio
public void stopAdvertisement() throws UnsupportedAudioFileException,
IOException, LineUnavailableException, InterruptedException {
currentFrameAdvertisement = 0L;
clipAdvertisement.stop();
clipAdvertisement.close();
System.out.println("stopping advertisement");
}
public void resetAudioStreamAdvertisement() throws UnsupportedAudioFileException, IOException,
LineUnavailableException
{
advertisementInputStream = AudioSystem.getAudioInputStream(
new File(filePath).getAbsoluteFile());
clipAdvertisement.open(musicInputStream);
clipAdvertisement.loop(Clip.LOOP_CONTINUOUSLY);
}
public static void appendAdvertisementFiles(List<Advertisement> advertisementItems) throws IOException, UnsupportedAudioFileException {
//add the first audio file to stream
AudioInputStream appendedFiles = AudioSystem.getAudioInputStream(
new File(advertisementItems.get(0)
.getId() + "-advertisement.wav"));
//loop through an combine
for(int i =1;i<advertisementItems.size();i++){
File file= new File(advertisementItems.get(i)
.getId() + "-advertisement.wav");
AudioInputStream toBeAppended = AudioSystem.getAudioInputStream(file);
//append them
appendedFiles =
new AudioInputStream(
new SequenceInputStream(appendedFiles, toBeAppended),
appendedFiles.getFormat(),
appendedFiles.getFrameLength() + toBeAppended.getFrameLength());
}
advertisementInputStream = appendedFiles;
}
public void checkShouldWePlayAnAdvertisement() throws IOException, UnsupportedAudioFileException, LineUnavailableException, InterruptedException {
ArrayList<String> playedAtTimes = new ArrayList<>();
ArrayList<Advertisement> advertisementsToBePlayed = new ArrayList<>();
boolean found;
//played at times is used to keep track of what time we played advertisements
//so when the loop reruns and the time hasnt changed it doesnt play it again
while(true){
found = false;
ZonedDateTime zdt = ZonedDateTime.now();
String timeHHMM =zdt.toString().substring(11,16);
for(int i =0;i<schedule.getAdvertisementScheduleItems().size();i++)
{
if(schedule.getAdvertisementScheduleItems().get(i).getTimes()
.contains(timeHHMM))
{
//this item should be played now
if(playedAtTimes.contains(timeHHMM)){
//we already played this,but the time hasnt changed when the loop ran again
}else{
advertisementsToBePlayed.add(schedule.getAdvertisementScheduleItems().get(i).getAdvertisement());
found = true;
}
}
}
if(found== true){
playedAtTimes.add(timeHHMM);
appendAdvertisementFiles(advertisementsToBePlayed);
pauseMusic();
playAdvertisements();
stopAdvertisement();
resumeAudioMusic();
}
}
}
}
IotClient(part of listener)
public class IotClient extends Thread {
Thread t;
String username;
public IotClient(String username) {
this.username = username;
}
public void run(){
String clientEndpoint = "removve"; // replace <prefix> and <region> with your own
String clientId = "1"; // replace with your own client ID. Use unique client IDs for concurrent connections.
// AWS IAM credentials could be retrieved from AWS Cognito, STS, or other secure sources
AWSIotMqttClient client = new AWSIotMqttClient(clientEndpoint, clientId, "remove", "remove");
// optional parameters can be set before connect()
try {
client.connect();
} catch (AWSIotException e) {
e.printStackTrace();
}
AWSIotQos qos = AWSIotQos.QOS0;
AWSIotTopic topic = new MyTopic("schedule/"+ username, qos);
try {
client.subscribe(topic, true);
} catch (AWSIotException e) {
e.printStackTrace();
}
while(true){
}
}
public void start(){
if (t == null) {
t = new Thread (this, "IotClientThread");
t.start ();
}
}
MyTopic(part of listener)
public class MyTopic extends AWSIotTopic {
public MyTopic(String topic, AWSIotQos qos) {
super(topic, qos);
}
#Override
public void onMessage(AWSIotMessage message) {
System.out.println("Message recieved from topic: "+ message.getStringPayload());
}
}
Threads communicate via shared references to messages 'containers' objects in memory. That could be as simple as mere mutable field of a shared instance of some class, or more typical collection like list, map, but especially queues.
ArrayBlockingQueue is a good shared reference. There would be a queue for each message direction from one thread to another. If you had 3 threads that could truly talks to each other, you would have 3 pairs, thus 6 queues (2 for each pair). However, it is often the case that messages flow only in one direction, so you might save a few.
Now, the core of these communications is a mechanism to wait for some message (reader/consumer), and notify when a message is pushed (writer/producer).
Of course, you can learn (penty of tutorials out there) from the bottom going up, from the primitive wait/notify, or you can jump into classes like ArrayBlockingQueue which abstract the wait/notify of messages into take()/put(). I recommend starting from the bottom as things will make sense more rapidly when you meet other classes in java.util.concurrent.*
I cannot give you code, it would be largely not understandable at your level without having learned the basic of ITC (inter-thread communications).
Good learning!
PS: there are many pitfalls on the way, like thread safety, atomicity of writes, lock free algorithms, deadlocks, livelocks, starvation. Just the multi queue example above can lead to circular dependencies on message arrival, particularly when queue get full and block. This is a science!
Related
Context
I made a Java application, and need to run two instances of that application, synchronizing some of their attributes via socket each time there's some change. To communicate those changes, Serializable objects are sent through a socket using ObjectStreams (input and output) using read/writeUTF() for an identifier, and read/writeObject() and flush(). The app is the exact same .jar, run twice with some changes like having different ports and ip (if necessary).
Problem
I noticed that objects of some of my classes (e.g. Notification) were sent and received without any troubles, but objects from another class (RegisteredUsers) weren't sent (or received) properly. So I ran some tests to send objects between the two apps and found that the object is being sent and isn't null, it's attribute (a HashMap<String,User>) is also being sent and isn't null, but is always empty.
So I decided to scale it down to what the problem was exactly: I'm trying to write an object through a Stream, and read it in a different process of the same .jar, and with most classes it seems to work, but it doesn't with one.
There seems to be something I'm missing or don't understand about this serialization process, if the object is written and read during the execution of the same process it works, but not if this object is read on another instance of the same app. I even added a HashMap to Notification with the same creation process, but it still works, I really don't get it, what am I missing?
Code
I have taken some code from the bigger app and trimmed it down to the basic problem if anyone wants to test it. To reproduce the errors, run Main1, which will create the two files with an object persisted in each one (one with a Notification object and the other with a RegisteredUsers object) and shows their information, then Main2, which reads them from the files and shows their information, and the problem should be printed. That being that reg3's HashMap is empty and thus neither of the Users are registered.
Main1
public class Main1 {
public static void main(String[] args) {
String regFile = "registry.txt";
String notificationFile = "notification.txt";
Persistence pers = new Persistence();
RegisteredUsers reg1 = new RegisteredUsers();
RegisteredUsers reg2 = new RegisteredUsers();
reg1.register("Name1", "127.0.0.1");
reg1.register("Name2", "127.0.0.1");
try {
pers.writeReg(reg1, regFile);
} catch (IOException e) {
System.out.println("Error writing registry.");
}
try {
reg2 = pers.readReg(regFile);
} catch (IOException e) {
System.out.println("Error reading registry.");
}
System.out.println("Original registry: ");
System.out.println(reg1.isRegistered("Name1") + " " + reg1.isRegistered("Name2"));
System.out.println("Registry read from file: ");
System.out.println(reg2.isRegistered("Name1") + " " + reg2.isRegistered("Name2"));
Notification noti1 = new Notification("Name", "127.0.0.1");
Notification noti2 = new Notification(); //not necesary but it's the way it's done in the bigger app.
try {
pers.writeNotif(noti1, notificationFile);
} catch (IOException e) {
System.out.println("Error writing notification.");
}
try {
noti2 = pers.readNotif(notificationFile);
} catch (IOException e) {
System.out.println("Error reading notification.");
}
System.out.println("Original notification: ");
System.out.println(noti1.getAttributes().get(0) + " " + noti1.getAttributes().get(1));
System.out.println(noti1.getMap());
System.out.println("Notification read from file: ");
System.out.println(noti2.getAttributes().get(0) + " " + noti2.getAttributes().get(1));
System.out.println(noti2.getMap());
}
}
Main2
public class Main2 {
public static void main(String[] args) {
String regFile = "registry.txt";
String notificationFile = "notification.txt";
Persistence pers = new Persistence();
RegisteredUsers reg3 = new RegisteredUsers();
try {
reg3 = pers.readReg(regFile);
} catch (IOException e) {
System.out.println("Error reading registry.");
}
if (reg3 == null) {
System.out.println("reg3 is null");
}
if (reg3.getMap() == null)
System.out.println("reg3 has a null map");
if (reg3.getMap().isEmpty())
System.out.println("reg3 has an empty map");
System.out.println("Registry read from file on another process: ");
System.out.println(reg3.isRegistered("Name1") + " " + reg3.isRegistered("Name2"));
Notification noti3 = new Notification(); //not necesary but it's the way it's done in the bigger app.
try {
noti3 = pers.readNotif(notificationFile);
} catch (IOException e) {
System.out.println("Error reading notification.");
}
System.out.println("Notification read from file on another process: ");
System.out.println(noti3.getAttributes().get(0) + " " + noti3.getAttributes().get(1));
System.out.println(noti3.getMap());
}
}
A Class to persist the objects in the files:
public class Persistence {
public void writeReg(RegisteredUsers regus, String file) throws IOException {
try(FileOutputStream fos = new FileOutputStream(file);
ObjectOutputStream oos = new ObjectOutputStream(fos);) {
oos.writeObject(regus);
oos.flush();
}
}
public RegisteredUsers readReg(String file) throws IOException {
RegisteredUsers regus = null;
try(FileInputStream fis = new FileInputStream(file);
ObjectInputStream ois = new ObjectInputStream(fis);) {
regus = (RegisteredUsers) ois.readObject();
} catch (ClassNotFoundException e) {
System.out.println("Wrong class.");
}
return regus;
}
public void writeNotif(Notification regus, String file) throws IOException {
try(FileOutputStream fos = new FileOutputStream(file);
ObjectOutputStream oos = new ObjectOutputStream(fos);) {
oos.writeObject(regus);
oos.flush();
}
}
public Notification readNotif(String file) throws IOException {
Notification notif = null;
try(FileInputStream fis = new FileInputStream(file);
ObjectInputStream ois = new ObjectInputStream(fis);) {
notif = (Notification) ois.readObject();
} catch (ClassNotFoundException e) {
System.out.println("Wrong class.");
}
return notif;
}
}
RegisteredUsers
public class RegisteredUsers implements Serializable {
private static HashMap<String, User> users;
public RegisteredUsers() {
users = new HashMap<String, User>();
}
public HashMap<String, User> getMap() {
return users;
}
public boolean isRegistered(String name) {
User us = users.get(name);
return us != null;
}
public void register(String name, String ip) {
users.put(name, new User(name, ip, false));
}
}
Notification
public class Notification implements Serializable {
private ArrayList<String> attributes;
private HashMap<String, User> map = new HashMap<>();
public Notification() {
}
public Notification(String name, String ip) {
attributes = new ArrayList<String>();
attributes.add(0, name);
attributes.add(1, ip);
map.put(ip, new User(name, ip, false));
}
public ArrayList<String> getAttributes() {
return attributes;
}
public HashMap<String, User> getMap() {
return map;
}
}
User
public class User implements Serializable {
private String name;
private String ip;
private boolean connection_state;
public User(String name, String ip, boolean connection_state) {
this.name = name;
this.ip = ip;
this.connection_state = connection_state;
}
}
In java static fields are implicitly transient, and transient fields are not serialized.
If you modify the RegisterdUsers to
public class RegisteredUsers implements Serializable {
private HashMap<String, User> users; // static modifier is removed
...
}
The serialization will work.
I'm implementing some simple netty server and client to send and revieve files. Something which is similar to cloud storage.
I have a server which handles incoming requests and sends the files back to the client. I also want my apps to be able to handle with big files, that's why I divide such files into chunks and send them chunk by chunk. But there's an issue I can't resolve.
Let's say:
We have a 4 gb file on a server.
It is divided into 40 000 chunks.
Then they are sent to a client application, and I can see that all the chunks at the server are written into socket, as I use int field as a message number (chunk number) and put into log a message number which is being written.
But then when a client receives messages (chunks), in the case of large files the process doesn't finish successfully and only some (it depends on the size of a file) of the chunks are received by a client.
A client starts receiving consecutive messages - 1, 2, 3, 4 ... 27878, 27879 and then stops with no exception, although the last message from server was, for example 40000.
Almost forgot to say that I use JavaFX for the client app.
So, I tried to play with xms xmx java vm options but it didn't help.
Server
public class Server {
public void run() throws Exception {
EventLoopGroup mainGroup = new NioEventLoopGroup();
EventLoopGroup workerGroup = new NioEventLoopGroup();
try {
ServerBootstrap b = new ServerBootstrap();
b.group(mainGroup, workerGroup)
.channel(NioServerSocketChannel.class)
.childHandler(new ChannelInitializer<SocketChannel>() {
protected void initChannel(SocketChannel socketChannel) throws Exception {
socketChannel.pipeline().addLast(
new ObjectDecoder(Constants.FRAME_SIZE, ClassResolvers.cacheDisabled(null)),
new ObjectEncoder(),
new MainHandler()
);
}
})
.childOption(ChannelOption.SO_KEEPALIVE, true);
ChannelFuture future = b.bind(8189).sync();
future.channel().closeFuture().sync();
} finally {
mainGroup.shutdownGracefully();
workerGroup.shutdownGracefully();
}
}
public static void main(String[] args) throws Exception {
new Server().run();
}
}
Server handler
#Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
try {
if (msg == null) {
return;
}
if (msg instanceof FileRequest) {
FileRequest fr = (FileRequest) msg;
switch (fr.getFileCommand()) {
case DOWNLOAD:
sendFileToClient(ctx, fr.getFilename());
break;
case LIST_FILES:
listFiles(ctx);
break;
case DELETE:
deleteFileOnServer(fr);
listFiles(ctx);
break;
case SEND:
saveFileOnServer(fr);
listFiles(ctx);
break;
case SEND_PARTIAL_DATA:
savePartialDataOnServer(fr);
break;
}
}
} finally {
ReferenceCountUtil.release(msg);
}
}
Methods for sending files in chunks
private void sendFileToClient(ChannelHandlerContext ctx, String fileName) throws IOException {
Path path = Paths.get("server_storage/" + fileName);
if (Files.exists(path)) {
if (Files.size(path) > Constants.FRAME_SIZE) {
sendServerDataFrames(ctx, path);
ctx.writeAndFlush(new FileRequest(FileCommand.LIST_FILES));
} else {
FileMessage fm = new FileMessage(path);
ctx.writeAndFlush(fm);
}
}
}
private void sendServerDataFrames(ChannelHandlerContext ctx, Path path) throws IOException {
byte[] byteBuf = new byte[Constants.FRAME_CHUNK_SIZE];
FileMessage fileMessage = new FileMessage(path, byteBuf, 1);
FileRequest fileRequest = new FileRequest(FileCommand.SEND_PARTIAL_DATA, fileMessage);
FileInputStream fis = new FileInputStream(path.toFile());
int read;
while ((read = fis.read(byteBuf)) > 0) {
if (read < Constants.FRAME_CHUNK_SIZE) {
byteBuf = Arrays.copyOf(byteBuf, read);
fileMessage.setData(byteBuf);
}
ctx.writeAndFlush(fileRequest);
fileMessage.setMessageNumber(fileMessage.getMessageNumber() + 1);
}
System.out.println("server_storage/" + path.getFileName() + ", server last frame number: " + fileMessage.getMessageNumber());
System.out.println("server_storage/" + path.getFileName() + ": closing file stream.");
fis.close();
}
client handlers
#Override
public void initialize(URL location, ResourceBundle resources) {
Network.start();
Thread t = new Thread(() -> {
try {
while (true) {
AbstractMessage am = Network.readObject();
if (am instanceof FileMessage) {
FileMessage fm = (FileMessage) am;
Files.write(Paths.get("client_storage/" + fm.getFilename()), fm.getData(), StandardOpenOption.CREATE);
refreshLocalFilesList();
}
if (am instanceof FilesListMessage) {
FilesListMessage flm = (FilesListMessage) am;
refreshServerFilesList(flm.getFilesList());
}
if (am instanceof FileRequest) {
FileRequest fr = (FileRequest) am;
switch (fr.getFileCommand()) {
case DELETE:
deleteFile(fr.getFilename());
break;
case SEND_PARTIAL_DATA:
receiveFrames(fr);
break;
case LIST_FILES:
refreshLocalFilesList();
break;
}
}
}
} catch (ClassNotFoundException | IOException e) {
e.printStackTrace();
} finally {
Network.stop();
}
});
t.setDaemon(true);
t.start();
refreshLocalFilesList();
Network.sendMsg(new FileRequest(FileCommand.LIST_FILES));
}
private void receiveFrames(FileRequest fm) throws IOException {
Utils.processBytes(fm.getFileMessage(), "client_storage/");
}
public final class Utils {
public static void processBytes(FileMessage fm, String pathPart) {
Path path = Paths.get(pathPart + fm.getFilename());
byte[] data = fm.getData();
System.out.println(pathPart + path.getFileName() + ": " + fm.getMessageNumber());
try {
if (fm.getMessageNumber() == 1) {
Files.write(path, data, StandardOpenOption.CREATE_NEW);
} else {
Files.write(path, data, StandardOpenOption.WRITE, StandardOpenOption.APPEND);
}
}
catch (IOException e) {
e.printStackTrace();
}
}
}
That what I see on server.
server_storage/DVD5_OFFICE_2010_SE_SP2_VOLUME_X86_RU-KROKOZ.iso: 42151
server_storage/DVD5_OFFICE_2010_SE_SP2_VOLUME_X86_RU-KROKOZ.iso: 42152
server_storage/DVD5_OFFICE_2010_SE_SP2_VOLUME_X86_RU-KROKOZ.iso, server last frame number: 42153
server_storage/DVD5_OFFICE_2010_SE_SP2_VOLUME_X86_RU-KROKOZ.iso: closing file stream.
And this one is on a client.
client_storage/DVD5_OFFICE_2010_SE_SP2_VOLUME_X86_RU-KROKOZ.iso: 29055
client_storage/DVD5_OFFICE_2010_SE_SP2_VOLUME_X86_RU-KROKOZ.iso: 29056
client_storage/DVD5_OFFICE_2010_SE_SP2_VOLUME_X86_RU-KROKOZ.iso: 29057
And there is no issue when sending files from the client to the the server. I can see in debugger and in the windows task manager that both processes are working simultaniously but it's not like this when a file is sent from the server to the client. First all the chunks are read and then they are sent to a client and it starts to receive them but failed to get all of them.
Please help. I have no idea what it could be. Thanks in advance.
I'm creating a token ring with sensors where every sensor is a process apart. When i start a sensor it communicates with the gateway and gets the list of the actual sensors already on the system .
The problem is that every time i start a new process i want every already existing sensor to get the updated list, so to understand that other sensors have been added and the list is no longer the one they had but a new updated one.(So lets say the processes must always have the same list). I use a server which i call serverSocket which listens for messages. I can make it possible for the server to understand that the list has been changed but what i cant do is how to change the value of the sensorList found on my SensorClient class, to be updated? In the code bellow i show what i'm doing but the sensorList keeps being the old one,not being updated :/ Can anyone please help me? Thank you :)
SensorClient where i start a new process sensor
public class SensorClient {
public static void main(String[] args) throws Exception {
Sensor sensor = new Sensor(type,identificator,portnumber,ipnumber,gatewayAddr,timestamp);
Gson gson = new Gson();
String message = gson.toJson(sensor);
Client c = Client.create();
WebResource r = c.resource("http://localhost:9999/gateway/");
ClientResponse response = r.path("sensors/add").type(MediaType.APPLICATION_JSON).accept(MediaType.APPLICATION_JSON).post(ClientResponse.class, message);
if (response.getStatus() == 200) {
repeat = false;
Type collectionType = new TypeToken<ArrayList<Sensor>>(){}.getType();
ArrayList<Sensor> sensorList = gson.fromJson(response.getEntity(String.class), collectionType);
System.out.println("Starting the sensor ...");
System.out.println("Push exit when you want to delete the sensor!");
int position = 0;
for(int i = 0; i< sensorList.size();i++){ if(sensorList.get(i).getIdentificator().equalsIgnoreCase(sensor.getIdentificator()) ) position = i;
}
sensors.Sensor.simulation(type, identificator);// special thread for sensors simulations
createSensor.getInstance().setPrevNextWhenAdd(position,sensorList);
serverSocket serverSocket = new serverSocket(portnumber,sensorList,position,sensorList.get(position).getNext());
serverSocket.start();
StopSensor stopSensor = new StopSensor(identificator,portnumber,position,sensorList);
stopSensor.start();
oneSensor s = new oneSensor(portnumber,sensorList);
s.start();
} else {
repeat = true;
count +=1;
System.out.println("Error. Wrong data! ");
}
}
while (repeat );
}
}
}
The serverSocket thread
public class serverSocket extends Thread {
public int port,nextPort;
ArrayList<gateway.Sensor> sensorList;
public static int position;
public serverSocket(int port, ArrayList<gateway.Sensor> sensorList,int position,int nextPort) {
this.port = port;
this.nextPort=nextPort;
this.sensorList= sensorList;
this.position=position;}
public void run() {
ServerSocket welcomeSocket;
Socket connectionSocket;
try {
welcomeSocket = new ServerSocket(port);
while (true) {
connectionSocket = welcomeSocket.accept();
receivedMessages thread = new receivedMessages(connectionSocket,sensorList,position,nextPort);
thread.start();
}
} catch (IOException e) {
e.printStackTrace();
System.err.println("Error!!!!!!!!!");
}
}
}
The receivedMessages thread
public class receivedMessages extends Thread {
private BufferedReader inFromClient;
private Socket connectionSocket;
ArrayList<gateway.Sensor> sensorList;
int position,nextPort;
public receivedMessages(Socket socket, ArrayList<gateway.Sensor> sensorList,int position,int nextPort){
connectionSocket = socket;
this.sensorList=sensorList;
this.position=position;
this.nextPort=nextPort;
try {
inFromClient = new BufferedReader( new InputStreamReader(connectionSocket.getInputStream()));
} catch (IOException e) { e.printStackTrace(); }
}
#Override
public void run() {
try {
String message = (inFromClient.readLine().toString());
if (message.startsWith("Next") || message.startsWith("Previous")) {
System.out.println(message);
} else if (message.startsWith("The")) {
System.out.println(message); createSensor.getInstance().setPrevNextWhenDelete(position, sensorList);
} else {// i receive the message that the list has changed
System.out.println(message);
sensorList = createSensor.getInstance().getSensorList();
System.out.println("Updated " + sensorList);}
This class has methods used by gateway to register a sensor when it makes the request
public class createSensor {
private static createSensor instance = null;
private ArrayList<Sensor> sensor = new ArrayList<>();
public int position, prevPosition, nextPosition, prevPort, nextPort;
private createSensor() { }
public static synchronized createSensor getInstance() {
if (instance == null) {
instance = new createSensor();
}
return instance;
}
public synchronized ArrayList insertSensor(String type, String identificator, int port, String id, String gatwayAddr, long timestamp) throws IOException {
sensor.add(new Sensor(type, identificator, port, id, gatwayAddr, timestamp));
return new ArrayList<>(sensor); //
}
}
public synchronized boolean hasMeasurements() {
while (InnerBuffer.getInstance().emptyInnerBuffer())
return false;
return true;
}
public synchronized void setPrevNextWhenDelete(int position,ArrayList<Sensor> sensorList) throws IOException {
//code
}
public synchronized ArrayList<Sensor> getSensorList() {
return new ArrayList<>(sensor);
}
public synchronized int size() {
return sensor.size();
}
public synchronized String returnRecentMeasurement (String id){
String recentMeasurement=null;
for (Sensor sensori : sensor) {
if (sensori.getIdentificator().equalsIgnoreCase(id))
recentMeasurement= InnerBuffer.getInstance().returnRecentMeasurements(id);
else
recentMeasurement = null;}
return recentMeasurement;
}
public synchronized void setPrevNextWhenAdd() throws IOException { //some other code where int position, prevPosition, nextPosition, prevPort, nextPort get their values. }}
I wrote an QuickFix program and created a config file.. Now, when I'm running my application, nothing happens.. So I ask me, how I can recognize if the connection (my program is of type initiator) is established or not.
I've added to the methods that I implement from the Application interface
void fromAdmin(Message message, SessionID sessionId)
void fromApp(Message message, SessionID sessionId)
void onCreate(SessionID sessionId)
void onLogon(SessionID sessionId)
void onLogout(SessionID sessionId)
void toAdmin(Message message, SessionID sessionId)
void toApp(Message message, SessionID sessionId)
a System.out and a logger.info, but nothing of them fires something.
If a connection established, the methods onCreate and onLogon are invoked, or?! So the System.outs in these methods should write something..
But are there any other opportunitys to check wheter the connection established, respectively the config file is valid.
P.S.: I use the SessionSettings to read the config File in.. But I can't find a method in the SessionSettings like validateConfigFile() or something like that.
Has your config file got FileLogPath=log?
Then to debug you look at the FIX messages log, usually in the bin/debug/log directory, which is where it would be for the above config file. If it's not a security risk then paste your config file here please. Also, yes add the System.Out to your Application interface. Here's mine:
public void FromApp(Message msg, SessionID s)
{
Console.WriteLine("IN: " + msg.ToString());
You can download Banzai application which is an accpetor/iniciator with grafic interface. You can iniciate a session, and send/receive messages from your app.
Main class you need in your Iniciator
public class Banzai {
private static final CountDownLatch shutdownLatch = new CountDownLatch(1);
/** enable logging for this class */
private static Logger log = LoggerFactory.getLogger(Banzai.class);
private static Banzai banzai;
private boolean initiatorStarted = false;
private Initiator initiator = null;
private JFrame frame = null;
public Banzai(String[] args) throws Exception {
InputStream inputStream = null;
if (args.length == 0) {
inputStream = new BufferedInputStream(new FileInputStream(new File("config/banzai.cfg")));
} else if (args.length == 1) {
inputStream = new FileInputStream(args[0]);
}
if (inputStream == null) {
System.out.println("usage: " + Banzai.class.getName() + " [configFile].");
return;
}
SessionSettings settings = new SessionSettings(inputStream);
inputStream.close();
boolean logHeartbeats = Boolean.valueOf(System.getProperty("logHeartbeats", "true")).booleanValue();
OrderTableModel orderTableModel = new OrderTableModel();
ExecutionTableModel executionTableModel = new ExecutionTableModel();
BanzaiApplication application = new BanzaiApplication(orderTableModel, executionTableModel);
MessageStoreFactory messageStoreFactory = new FileStoreFactory(settings);
LogFactory logFactory = new ScreenLogFactory(true, true, true, logHeartbeats);
MessageFactory messageFactory = new DefaultMessageFactory();
initiator = new SocketInitiator(application, messageStoreFactory, settings, logFactory, messageFactory);
frame = new BanzaiFrame(orderTableModel, executionTableModel, application);
frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
}
public synchronized void logon() {
if (!initiatorStarted) {
try {
initiator.start();
initiatorStarted = true;
} catch (Exception e) {
log.error("Logon failed", e);
}
} else {
Iterator<SessionID> sessionIds = initiator.getSessions().iterator();
while (sessionIds.hasNext()) {
SessionID sessionId = (SessionID) sessionIds.next();
Session.lookupSession(sessionId).logon();
}
}
}
public void logout() {
Iterator<SessionID> sessionIds = initiator.getSessions().iterator();
while (sessionIds.hasNext()) {
SessionID sessionId = (SessionID) sessionIds.next();
Session.lookupSession(sessionId).logout("user requested");
}
}
public void stop() {
shutdownLatch.countDown();
}
public JFrame getFrame() {
return frame;
}
public static Banzai get() {
return banzai;
}
public static void main(String args[]) throws Exception {
try {
UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());
} catch (Exception e) {
log.info(e.getMessage(), e);
}
banzai = new Banzai(args);
if (!System.getProperties().containsKey("openfix")) {
banzai.logon();
}
shutdownLatch.await();
}
}
NOTE: The information in your .settings has to match with the information you have in your code.
After you start both codes you will start to receive logon, and heartbeat messages.
I'm using ScheduledThreadPoolExecutor to create a file every fileIntervalInSeconds seconds:
executorService = new ScheduledThreadPoolExecutor(1);
executorService.scheduleAtFixedRate(new Runnable()
{
#Override
public void run()
{
File file = new File(fileName);
if (file.exists())
{
Log.debug("creating new file");
openFileWriter(file);
}
}
}, fileIntervalInSeconds, fileIntervalInSeconds, TimeUnit.SECONDS);
}
private void openFileWriter() throws FileSystemNotificationException
{
// 1 - close exist writer
writer.close();
// 2 - rename to backup file name
...
// 3 - create new file
FileWriter writerFile = new FileWriter(fileName, true);
writer = new PrintWriter(writerFile);
}
And i'm writing alert Messages to the file all the time:
private synchronized void writeLine(String line) throws InterruptedException
{
writer.println(line);
}
My problem is:
how can i ensure that i'm using writer when it is not closed? (writer.close())
How can i wait to the ScheduledThreadPoolExecutor to finish creating the file before start writing
How about checking the file exists before you write to it. No need for a backrgound thread or synchronization
private synchronized void writeLine(String line) {
if (!file.exists())
reopenWritingFile();
writer.println(line);
}
You could simply create the new file every hour within your write method. You would have some slight overhead for the time check but that should be negligible. The example below will create a new log file every hour with the time in milliseconds added to the front of the file name. You can format the time however suits you.
public class LogWriter {
private long lastCreationTime;
PrintWriter writer;
String logFileName;
public LogWriter(String logFileName) {
this.logFileName = logFileName;
createLogFile(logFileName);
}
private void createLogFile(String fileName) {
if(writer != null) {
writer.close();
}
lastCreationTime = System.currentTimeMillis();
FileWriter writerFile;
try {
writerFile = new FileWriter(lastCreationTime + "_" + fileName, true);
writer = new PrintWriter(writerFile);
} catch (IOException e) {
e.printStackTrace();
}
}
private synchronized void writeLine(String line) {
if(lastCreationTime < System.currentTimeMillis() - 3600000) {
createLogFile(logFileName);
}
writer.write(line);
}
}
You have two alternatives:
Create the new writer before closing the old one.
Establish a lock before closing the writer which you check when writing.
Examples:
volatile PrintWriter writer;
ReadWriteLock lock = new ReentrantReadWriteLock();
Lock writeLock = lock.writeLock();
Lock readLock = lock.readLock();
private void openFileWriterWithLock() throws IOException {
if (writeLock.tryLock()) {
try {
// 1 - close exist writer
writer.close();
// 2 - rename to backup file name
//...
// 3 - create new file
FileWriter writerFile = new FileWriter(fileName, true);
writer = new PrintWriter(writerFile);
} finally {
writeLock.unlock();
}
}
}
private synchronized void writeLineWithLock(String line) throws InterruptedException {
readLock.lock();
try {
writer.println(line);
} finally {
readLock.unlock();
}
}
private void openFileWriterWithoutLock() throws IOException {
// 0. Note old file.
PrintWriter oldWriter = writer;
// 1. Create new file.
FileWriter writerFile = new FileWriter(fileName, true);
// 2. Swap the new one in.
writer = new PrintWriter(writerFile);
// 3 - close old writer
oldWriter.close();
}
private synchronized void writeLineWithoutLock(String line) throws InterruptedException {
writer.println(line);
}
How about having a separate thread that handles the logging instead of that rather complicated construct?
public class Logger extends Thread {
private final LinkedBlockingQueue<String> linesToWrite = new LinkedBlockingQueue<>();
private final String filename;
private Logger(String filename) {
super("Logging thread");
this.filename = filename;
this.setDaemon(true);
this.setPriority(Thread.MIN_PRIORITY);
}
#Override
public void run() {
try (BufferedWriter out = new BufferedWriter(new FileWriter(filename, true))) {
String line;
while (this.isInterrupted() == false) {
line = linesToWrite.take();
out.write(line);
out.newLine();
out.flush();
}
} catch (InterruptedException e) {
} catch (IOException ex) {
System.out.println("Failed to access log file: " + ex);
}
}
public void log(final String line) {
this.linesToWrite.add(line);
}
Then initialize the logger once:
final Logger logger = new Logger("test.log");
logger.start();
And then you can use it from anywhere in a thread-safe way like this:
logger.log("Test message");
You don't need to stop the logger, because Java will ensure with the try construct that the file is properly closed. However if you want, you can stop it like this:
logger.interrupt();
Now you can do all file manipulation in a single-threaded way, because there is only one thread accessing the log files ever at any time.