We Keep Coding

DL4J-Image become too bright

Currently, I've been asked to write CNN code using DL4J using YOLOv2 architecture. But the problem is after the model has complete, I do a simple GUI for validation testing then the image shown is too bright and sometimes the image can be displayed. Im not sure where does this problem comes from whether at earliest stage of training or else. Here, I attach the code that I have for now. For Iterator:
public class faceMaskIterator {
private static final Logger log = org.slf4j.LoggerFactory.getLogger(faceMaskIterator.class);
private static final int seed = 123;
private static Random rng = new Random(seed);
private static String dataDir;
private static Path pathDirectory;
private static InputSplit trainData, testData;
private static final String[] allowedFormats = NativeImageLoader.ALLOWED_FORMATS;
private static final double splitRatio = 0.8;
private static final int nChannels = 3;
public static final int gridWidth = 13;
public static final int gridHeight = 13;
public static final int yolowidth = 416;
public static final int yoloheight = 416;
private static RecordReaderDataSetIterator makeIterator(InputSplit split, Path dir, int batchSize) throws Exception {
ObjectDetectionRecordReader recordReader = new ObjectDetectionRecordReader(yoloheight, yolowidth, nChannels,
gridHeight, gridWidth, new VocLabelProvider(dir.toString()));
recordReader.initialize(split);
RecordReaderDataSetIterator iter = new RecordReaderDataSetIterator(recordReader, batchSize, 1, 1,true);
iter.setPreProcessor(new ImagePreProcessingScaler(0, 1));
return iter;
}
public static RecordReaderDataSetIterator trainIterator(int batchSize) throws Exception {
return makeIterator(trainData, pathDirectory, batchSize);
}
public static RecordReaderDataSetIterator testIterator(int batchSize) throws Exception {
return makeIterator(testData, pathDirectory, batchSize);
}
public static void setup() throws IOException {
log.info("Load data...");
dataDir = Paths.get(
System.getProperty("user.home"),
Helper.getPropValues("dl4j_home.data")
).toString();
pathDirectory = Paths.get(dataDir,"face_mask_dataset");
FileSplit fileSplit = new FileSplit(new File(pathDirectory.toString()),allowedFormats,rng);
PathFilter pathFilter = new RandomPathFilter(rng,allowedFormats);
InputSplit[] sample = fileSplit.sample(pathFilter, splitRatio,1-splitRatio);
trainData = sample[0];
testData = sample[1];
}}
For training:
public class faceMaskPreTrained {
private static final Logger log = LoggerFactory.getLogger(ai.certifai.groupProjek.faceMaskPreTrained.class);
private static int seed = 420;
private static double detectionThreshold = 0.9;
private static int nBoxes = 3;
private static double lambdaNoObj = 0.7;
private static double lambdaCoord = 1.0;
private static double[][] priorBoxes = {{1, 1}, {2, 1}, {1, 2}};
private static int batchSize = 3;
private static int nEpochs = 1;
private static double learningRate = 1e-4;
private static int nClasses = 3;
private static List<String> labels;
private static File modelFilename = new File(System.getProperty("user.dir"), "generated-models/facemask_detector.zip");
private static ComputationGraph model;
private static Frame frame = null;
private static final Scalar GREEN = RGB(0, 255.0, 0);
private static final Scalar YELLOW = RGB(255, 255, 0);
private static final Scalar RED = RGB(255, 0, 0);
private static Scalar[] colormap = {GREEN, YELLOW, RED};
private static String labeltext = null;
public static void main(String[] args) throws Exception {
faceMaskIterator.setup();
RecordReaderDataSetIterator trainIter = faceMaskIterator.trainIterator(batchSize);
RecordReaderDataSetIterator testIter = faceMaskIterator.testIterator(1);
labels = trainIter.getLabels();
if (modelFilename.exists()) {
Nd4j.getRandom().setSeed(seed);
log.info("Load model...");
model = ModelSerializer.restoreComputationGraph(modelFilename);
} else {
Nd4j.getRandom().setSeed(seed);
INDArray priors = Nd4j.create(priorBoxes);
log.info("Build model...");
ComputationGraph pretrained = (ComputationGraph) YOLO2.builder().build().initPretrained();
FineTuneConfiguration fineTuneConf = getFineTuneConfiguration();
model = getComputationGraph(pretrained, priors, fineTuneConf);
System.out.println(model.summary(InputType.convolutional(
faceMaskIterator.yoloheight,
faceMaskIterator.yolowidth,
nClasses)));
log.info("Train model...");
UIServer server = UIServer.getInstance();
StatsStorage storage = new InMemoryStatsStorage();
server.attach(storage);
model.setListeners(new ScoreIterationListener(5), new StatsListener(storage,5));
for (int i = 1; i < nEpochs + 1; i++) {
trainIter.reset();
while (trainIter.hasNext()) {
model.fit(trainIter.next());
}
log.info("*** Completed epoch {} ***", i);
}
ModelSerializer.writeModel(model, modelFilename, true);
System.out.println("Model saved.");
}
// Evaluate the model's accuracy by using the test iterator.
OfflineValidationWithTestDataset(testIter);
// Inference the model and process the webcam stream and make predictions.
doInference();
}
private static ComputationGraph getComputationGraph(ComputationGraph pretrained, INDArray priors, FineTuneConfiguration fineTuneConf) {
return new TransferLearning.GraphBuilder(pretrained)
.fineTuneConfiguration(fineTuneConf)
.removeVertexKeepConnections("conv2d_23")
.removeVertexKeepConnections("outputs")
.addLayer("conv2d_23",
new ConvolutionLayer.Builder(1, 1)
.nIn(1024)
.nOut(nBoxes * (5 + nClasses))
.stride(1, 1)
.convolutionMode(ConvolutionMode.Same)
.weightInit(WeightInit.XAVIER)
.activation(Activation.IDENTITY)
.build(),
"leaky_re_lu_22")
.addLayer("outputs",
new Yolo2OutputLayer.Builder()
.lambdaNoObj(lambdaNoObj)
.lambdaCoord(lambdaCoord)
.boundingBoxPriors(priors.castTo(DataType.FLOAT))
.build(),
"conv2d_23")
.setOutputs("outputs")
.build();
}
private static FineTuneConfiguration getFineTuneConfiguration() {
return new FineTuneConfiguration.Builder()
.seed(seed)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.gradientNormalization(GradientNormalization.RenormalizeL2PerLayer)
.gradientNormalizationThreshold(1.0)
.updater(new Adam.Builder().learningRate(learningRate).build())
.l2(0.00001)
.activation(Activation.IDENTITY)
.trainingWorkspaceMode(WorkspaceMode.ENABLED)
.inferenceWorkspaceMode(WorkspaceMode.ENABLED)
.build();
}
// Evaluate visually the performance of the trained object detection model
private static void OfflineValidationWithTestDataset(RecordReaderDataSetIterator test) throws InterruptedException {
NativeImageLoader imageLoader = new NativeImageLoader();
CanvasFrame canvas = new CanvasFrame("Validate Test Dataset");
OpenCVFrameConverter.ToMat converter = new OpenCVFrameConverter.ToMat();
org.deeplearning4j.nn.layers.objdetect.Yolo2OutputLayer yout = (org.deeplearning4j.nn.layers.objdetect.Yolo2OutputLayer) model.getOutputLayer(0);
Mat convertedMat = new Mat();
Mat convertedMat_big = new Mat();
while (test.hasNext() && canvas.isVisible()) {
org.nd4j.linalg.dataset.DataSet ds = test.next();
INDArray features = ds.getFeatures();
INDArray results = model.outputSingle(features);
List<DetectedObject> objs = yout.getPredictedObjects(results, detectionThreshold);
YoloUtils.nms(objs, 0.4);
Mat mat = imageLoader.asMat(features);
mat.convertTo(convertedMat, CV_8U, 255, 0);
int w = mat.cols() * 2;
int h = mat.rows() * 2;
resize(convertedMat, convertedMat_big, new Size(w, h));
convertedMat_big = drawResults(objs, convertedMat_big, w, h);
canvas.showImage(converter.convert(convertedMat_big));
canvas.waitKey();
}
canvas.dispose();
}
// Stream video frames from Webcam and run them through YOLOv2 model and get predictions
private static void doInference() {
String cameraPos = "front";
int cameraNum = 0;
Thread thread = null;
NativeImageLoader loader = new NativeImageLoader(
faceMaskIterator.yolowidth,
faceMaskIterator.yoloheight,
3,
new ColorConversionTransform(COLOR_BGR2RGB));
ImagePreProcessingScaler scaler = new ImagePreProcessingScaler(0, 1);
if (!cameraPos.equals("front") && !cameraPos.equals("back")) {
try {
throw new Exception("Unknown argument for camera position. Choose between front and back");
} catch (Exception e) {
e.printStackTrace();
}
}
FrameGrabber grabber = null;
try {
grabber = FrameGrabber.createDefault(cameraNum);
} catch (FrameGrabber.Exception e) {
e.printStackTrace();
}
OpenCVFrameConverter.ToMat converter = new OpenCVFrameConverter.ToMat();
try {
grabber.start();
} catch (FrameGrabber.Exception e) {
e.printStackTrace();
}
CanvasFrame canvas = new CanvasFrame("Object Detection");
int w = grabber.getImageWidth();
int h = grabber.getImageHeight();
canvas.setCanvasSize(w, h);
while (true) {
try {
frame = grabber.grab();
} catch (FrameGrabber.Exception e) {
e.printStackTrace();
}
//if a thread is null, create new thread
if (thread == null) {
thread = new Thread(() ->
{
while (frame != null) {
try {
Mat rawImage = new Mat();
//Flip the camera if opening front camera
if (cameraPos.equals("front")) {
Mat inputImage = converter.convert(frame);
flip(inputImage, rawImage, 1);
} else {
rawImage = converter.convert(frame);
}
Mat resizeImage = new Mat();
resize(rawImage, resizeImage, new Size(faceMaskIterator.yolowidth, faceMaskIterator.yoloheight));
INDArray inputImage = loader.asMatrix(resizeImage);
scaler.transform(inputImage);
INDArray outputs = model.outputSingle(inputImage);
org.deeplearning4j.nn.layers.objdetect.Yolo2OutputLayer yout = (org.deeplearning4j.nn.layers.objdetect.Yolo2OutputLayer) model.getOutputLayer(0);
List<DetectedObject> objs = yout.getPredictedObjects(outputs, detectionThreshold);
YoloUtils.nms(objs, 0.4);
rawImage = drawResults(objs, rawImage, w, h);
canvas.showImage(converter.convert(rawImage));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
});
thread.start();
}
KeyEvent t = null;
try {
t = canvas.waitKey(33);
} catch (InterruptedException e) {
e.printStackTrace();
}
if ((t != null) && (t.getKeyCode() == KeyEvent.VK_Q)) {
break;
}
}
}
private static Mat drawResults(List<DetectedObject> objects, Mat mat, int w, int h) {
for (DetectedObject obj : objects) {
double[] xy1 = obj.getTopLeftXY();
double[] xy2 = obj.getBottomRightXY();
String label = labels.get(obj.getPredictedClass());
int x1 = (int) Math.round(w * xy1[0] / faceMaskIterator.gridWidth);
int y1 = (int) Math.round(h * xy1[1] / faceMaskIterator.gridHeight);
int x2 = (int) Math.round(w * xy2[0] / faceMaskIterator.gridWidth);
int y2 = (int) Math.round(h * xy2[1] / faceMaskIterator.gridHeight);
//Draw bounding box
rectangle(mat, new Point(x1, y1), new Point(x2, y2), colormap[obj.getPredictedClass()], 2, 0, 0);
//Display label text
labeltext = label + " " + String.format("%.2f", obj.getConfidence() * 100) + "%";
int[] baseline = {0};
Size textSize = getTextSize(labeltext, FONT_HERSHEY_DUPLEX, 1, 1, baseline);
rectangle(mat, new Point(x1 + 2, y2 - 2), new Point(x1 + 2 + textSize.get(0), y2 - 2 - textSize.get(1)), colormap[obj.getPredictedClass()], FILLED, 0, 0);
putText(mat, labeltext, new Point(x1 + 2, y2 - 2), FONT_HERSHEY_DUPLEX, 1, RGB(0, 0, 0));
}
return mat;
}

CanvasFrame tries to do gamma correction by default because it's typically needed by cameras used for CV, but cheap webcams usually output gamma corrected images, so make sure to let CanvasFrame know about it this way:
// We should also specify the relative monitor/camera response for proper gamma correction.
CanvasFrame frame = new CanvasFrame("Some Title", CanvasFrame.getDefaultGamma()/grabber.getGamma());
https://github.com/bytedeco/javacv/

How to keep track of audio playback position?

I created a thread to play an mp3 file in Java by converting it to an array of bytes.
I'm wondering if I can keep track of the current play position as the mp3 is being played.
First, I set up my music stream like so:
try {
AudioInputStream in = AudioSystem.getAudioInputStream(file);
musicInputStream = AudioSystem.getAudioInputStream(MUSIC_FORMAT, in);
DataLine.Info dataLineInfo = new DataLine.Info(SourceDataLine.class, MUSIC_FORMAT);
musicDataLine = (SourceDataLine) AudioSystem.getLine(dataLineInfo);
musicDataLine.open(MUSIC_FORMAT);
musicDataLine.start();
startMusicThread();
} catch(Exception e) {
e.printStackTrace();
}
Next, my music thread looks like this:
private class MusicThread extends Thread {
byte musicBuffer[] = new byte[BUFFER_SIZE];
public void run() {
try {
int musicCount = 0;
while(writeOutput){
if(writeMusic && (musicCount = musicInputStream.read(musicBuffer, 0, musicBuffer.length)) > 0){
musicDataLine.write(musicBuffer, 0, musicCount);
}
}
} catch (Exception e) {
System.out.println("AudioStream Exception - Music Thread"+e);
e.printStackTrace();
}
}
}
I thought of one possibility, to create another thread with a timer that slowly ticks down, second by second, to show the remaining amount of time for the mp3 song. But that doesn't seem like a good solution at all.

Your int musicCount (the return value from AudioInputStream.read(...)) tells you the number of bytes read, so with that you can do a small computation to figure out your place in the stream always. (DataLine has some methods to do some of the math for you but they can't always be used...see below.)
int musicCount = 0;
int totalBytes = 0;
while ( loop stuff ) {
// accumulate it
// and do whatever you need with it
totalBytes += musicCount;
musicDataLine.write(...);
}
To get the number of seconds elapsed, you can do the following things:
AudioFormat fmt = musicInputStream.getFormat();
long framesRead = totalBytes / fmt.getFrameSize();
long totalFrames = musicInputStream.getFrameLength();
double totalSeconds = (double) totalFrames / fmt.getSampleRate();
double elapsedSeconds =
((double) framesRead / (double) totalFrames) * totalSeconds;
So you'd just get the elapsed time each loop and put it wherever you need it to go. Note that the accuracy of this kind of depends on the size of your buffer. The smaller the buffer, the more accurate.
Also, Clip has some methods to query this for you (but you'd probably have to change what you're doing a lot).
These methods (get(Long)FramePosition/getMicrosecondPosition) are inherited from DataLine, so you can also call them on the SourceDataLine as well if you don't want to do the math yourself. However, you basically need to make a new line for every file you play, so it depends on how you're using the line. (Personally I'd rather just do the division myself since asking the line is kind of opaque.)
BTW:
musicDataLine.open(MUSIC_FORMAT);
You should open the line with your own buffer size specified, using the (AudioFormat, int) overload. SourceDataLine.write(...) only blocks when its internal buffer is full, so if it's a different size from your byte array, sometimes your loop is blocking, other times it's just spinning.
MCVE for good measure:
import javax.swing.*;
import java.awt.*;
import java.awt.event.*;
import java.io.*;
import java.util.*;
import javax.sound.sampled.*;
public class SimplePlaybackProgress
extends WindowAdapter implements Runnable, ActionListener {
class AudioPlayer extends Thread {
volatile boolean shouldPlay = true;
final int bufferSize;
final AudioFormat fmt;
final AudioInputStream audioIn;
final SourceDataLine audioOut;
final long frameSize;
final long totalFrames;
final double sampleRate;
AudioPlayer(File file)
throws UnsupportedAudioFileException,
IOException,
LineUnavailableException {
audioIn = AudioSystem.getAudioInputStream(file);
fmt = audioIn.getFormat();
bufferSize = fmt.getFrameSize() * 8192;
frameSize = fmt.getFrameSize();
totalFrames = audioIn.getFrameLength();
sampleRate = fmt.getSampleRate();
try {
audioOut = AudioSystem.getSourceDataLine(audioIn.getFormat());
audioOut.open(fmt, bufferSize);
} catch (LineUnavailableException x) {
try {
audioIn.close();
} catch(IOException suppressed) {
// Java 7+
// x.addSuppressed(suppressed);
}
throw x;
}
}
#Override
public void run() {
final byte[] buffer = new byte[bufferSize];
long framePosition = 0;
try {
audioOut.start();
while (shouldPlay) {
int bytesRead = audioIn.read(buffer);
if (bytesRead < 0) {
break;
}
int bytesWritten = audioOut.write(buffer, 0, bytesRead);
if (bytesWritten != bytesRead) {
// shouldn't happen
throw new RuntimeException(String.format(
"read: %d, wrote: %d", bytesWritten, bytesRead));
}
framePosition += bytesRead / frameSize;
// or
// framePosition = audioOut.getLongFramePosition();
updateProgressBar(framePosition);
}
audioOut.drain();
audioOut.stop();
} catch (Throwable x) {
showErrorMessage(x);
} finally {
updateProgressBar(0);
try {
audioIn.close();
} catch (IOException x) {
showErrorMessage(x);
}
audioOut.close();
}
}
void updateProgressBar(
final long framePosition) {
SwingUtilities.invokeLater(new Runnable() {
#Override
public void run() {
double fractionalProgress =
(double) framePosition / (double) totalFrames;
int progressValue = (int) Math.round(
fractionalProgress * theProgressBar.getMaximum());
theProgressBar.setValue(progressValue);
int secondsElapsed = (int) Math.round(
(double) framePosition / sampleRate);
int minutes = secondsElapsed / 60;
int seconds = secondsElapsed % 60;
theProgressBar.setString(String.format(
"%d:%02d", minutes, seconds));
}
});
}
void stopPlaybackAndDrain() throws InterruptedException {
shouldPlay = false;
this.join();
}
}
/* * */
public static void main(String[] args) {
SwingUtilities.invokeLater(new SimplePlaybackProgress());
}
JFrame theFrame;
JButton theButton;
JProgressBar theProgressBar;
// this should only ever have 1 thing in it...
// multithreaded code with poor behavior just bugs me,
// even for improbable cases, so the queue makes it more robust
final Queue<AudioPlayer> thePlayerQueue = new ArrayDeque<AudioPlayer>();
#Override
public void run() {
theFrame = new JFrame("Playback Progress");
theFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
theButton = new JButton("Open");
theProgressBar = new JProgressBar(
SwingConstants.HORIZONTAL, 0, 1000);
theProgressBar.setStringPainted(true);
theProgressBar.setString("0:00");
Container contentPane = theFrame.getContentPane();
((JPanel) contentPane).setBorder(
BorderFactory.createEmptyBorder(8, 8, 8, 8));
contentPane.add(theButton, BorderLayout.WEST);
contentPane.add(theProgressBar, BorderLayout.CENTER);
theFrame.pack();
theFrame.setResizable(false);
theFrame.setLocationRelativeTo(null);
theFrame.setVisible(true);
theButton.addActionListener(this);
theFrame.addWindowListener(this);
}
#Override
public void actionPerformed(ActionEvent ae) {
JFileChooser dialog = new JFileChooser();
int option = dialog.showOpenDialog(theFrame);
if (option == JFileChooser.APPROVE_OPTION) {
File file = dialog.getSelectedFile();
try {
enqueueNewPlayer(new AudioPlayer(file));
} catch (UnsupportedAudioFileException x) { // ew, Java 6
showErrorMessage(x); //
} catch (IOException x) { //
showErrorMessage(x); //
} catch (LineUnavailableException x) { //
showErrorMessage(x); //
} //
}
}
#Override
public void windowClosing(WindowEvent we) {
stopEverything();
}
void enqueueNewPlayer(final AudioPlayer newPlayer) {
// stopPlaybackAndDrain calls join
// so we want to do it off the EDT
new Thread() {
#Override
public void run() {
synchronized (thePlayerQueue) {
stopEverything();
newPlayer.start();
thePlayerQueue.add(newPlayer);
}
}
}.start();
}
void stopEverything() {
synchronized (thePlayerQueue) {
while (!thePlayerQueue.isEmpty()) {
try {
thePlayerQueue.remove().stopPlaybackAndDrain();
} catch (InterruptedException x) {
// shouldn't happen
showErrorMessage(x);
}
}
}
}
void showErrorMessage(Throwable x) {
x.printStackTrace(System.out);
String errorMsg = String.format(
"%s:%n\"%s\"", x.getClass().getSimpleName(), x.getMessage());
JOptionPane.showMessageDialog(theFrame, errorMsg);
}
}
For Clip, you'd just have something like a Swing timer (or other side-thread) and query it however often:
new javax.swing.Timer(100, new ActionListener() {
#Override
public void actionPerformed(ActionEvent ae) {
long usPosition = theClip.getMicrosecondPosition();
// put it somewhere
}
}).start();
Related:
How to calculate the level/amplitude/db of audio signal in java?
How to make waveform rendering more interesting?

Read delay between frames in animated GIF

How do you read the animated GIF's control block between each frame? I'm interested in the delay between each frame. I've looked at the Javadoc for ImageReader and I'm not seeing anything.
Here's my code for reading all the frames from the animated GIF, how would I enhance it to read the metadata about each frame embedded in the animated GIF?
List<BufferedImage> list = new ArrayList<BufferedImage>();
try {
ImageReader reader = ImageIO.getImageReadersBySuffix("gif").next();
reader.setInput(ImageIO.createImageInputStream(urlImage.openStream()));
int i = reader.getMinIndex();
int numImages = reader.getNumImages(true);
while (i < numImages)
{
list.add(reader.read(i++));
}
// do stuff with frames of image...
} catch (Exception e) {
e.printStackTrace();
}

You will have to seek out "delayTime" attribute from metadata node. Use the following working example to understand:
public class GiffTest {
/**
* #param args the command line arguments
*/
public static void main(String[] args) throws IOException {
ImageReader reader = ImageIO.getImageReadersBySuffix("gif").next();
reader.setInput(ImageIO.createImageInputStream(new FileInputStream("H:\\toonGif.gif")));
int i = reader.getMinIndex();
int numImages = reader.getNumImages(true);
IIOMetadata imageMetaData = reader.getImageMetadata(0);
String metaFormatName = imageMetaData.getNativeMetadataFormatName();
IIOMetadataNode root = (IIOMetadataNode)imageMetaData.getAsTree(metaFormatName);
IIOMetadataNode graphicsControlExtensionNode = getNode(root, "GraphicControlExtension");
System.out.println(graphicsControlExtensionNode.getAttribute("delayTime"));
}
private static IIOMetadataNode getNode(IIOMetadataNode rootNode, String nodeName) {
int nNodes = rootNode.getLength();
for (int i = 0; i < nNodes; i++) {
if (rootNode.item(i).getNodeName().compareToIgnoreCase(nodeName)== 0) {
return((IIOMetadataNode) rootNode.item(i));
}
}
IIOMetadataNode node = new IIOMetadataNode(nodeName);
rootNode.appendChild(node);
return(node);
}
}

GIF animation plays extremely fast in JFrame

import java.awt.image.BufferedImage;
import java.io.IOException;
import java.io.InputStream;
import javax.imageio.ImageIO;
import javax.swing.Icon;
import javax.swing.ImageIcon;
import javax.swing.JFrame;
import javax.swing.JLabel;
import common.ResourcesToAccess;
public class RecordingStartingThread extends Thread{
#Override
public void run(){
JFrame f = new JFrame();
ImageIcon reel = new ImageIcon("src/images/reel.GIF");
JLabel label = new JLabel(reel);
reel.setImageObserver(label);
f.getContentPane().add(label);
f.setUndecorated(true);
f.setSize(300, 300);
f.setVisible(true);
}
public static void main(String[] args) {
new RecordingStartingThread().start();
}
}
Issue: GIF plays extremely fast.
Question: How do I make sure that GIF plays at a normal speed?

As for GIF speed playback - I've encountered this problem too. If I remember correctly this was caused by the "default" (or not provided?) value for frame rate in GIF file. Some web browsers "overrided" that frame rate so that GIF played correctly.
As a result I created a class that converts GIF (read GIF -> write GIF) and give frame rate provided by the user. com.madgag.gif.fmsware.AnimatedGifEncoder class is an external library that I link to the project via Maven: animated-gif-lib-1.0.jar
public final class GIFUtils {
private GIFUtils() {
}
public static List<BufferedImage> extractFrames(String filePath) throws IOException {
return extractFrames(new File(filePath));
}
public static List<BufferedImage> extractFrames(File file) throws IOException {
List<BufferedImage> imgs = new LinkedList<BufferedImage>();
ImageReader reader = ImageIO.getImageReadersBySuffix("GIF").next();
ImageInputStream in = null;
try {
in = ImageIO.createImageInputStream(new FileInputStream(file));
reader.setInput(in);
BufferedImage img = null;
int count = reader.getNumImages(true);
for(int i = 0; i < count; i++) {
Node tree = reader.getImageMetadata(i).getAsTree("javax_imageio_gif_image_1.0");
int x = Integer.valueOf(tree.getChildNodes().item(0).getAttributes()
.getNamedItem("imageLeftPosition").getNodeValue());
int y = Integer.valueOf(tree.getChildNodes().item(0).getAttributes()
.getNamedItem("imageTopPosition").getNodeValue());
BufferedImage image = reader.read(i);
if(img == null) {
img = new BufferedImage(image.getWidth() + x, image.getHeight() + y,
BufferedImage.TYPE_4BYTE_ABGR);
}
Graphics2D g = img.createGraphics();
ImageUtils.setBestRenderHints(g);
g.drawImage(image, x, y, null);
imgs.add(ImageUtils.copy(img));
}
}
finally {
if(in != null) {
in.close();
}
}
return imgs;
}
public static void writeGif(List<BufferedImage> images, File gifFile, int millisForFrame)
throws FileNotFoundException, IOException {
BufferedImage firstImage = images.get(0);
int type = firstImage.getType();
ImageOutputStream output = new FileImageOutputStream(gifFile);
// create a gif sequence with the type of the first image, 1 second
// between frames, which loops continuously
GifSequenceWriter writer = new GifSequenceWriter(output, type, 100, false);
// write out the first image to our sequence...
writer.writeToSequence(firstImage);
for(int i = 1; i < images.size(); i++) {
BufferedImage nextImage = images.get(i);
writer.writeToSequence(nextImage);
}
writer.close();
output.close();
}
public static Image createGif(List<BufferedImage> images, int millisForFrame) {
AnimatedGifEncoder g = new AnimatedGifEncoder();
ByteArrayOutputStream out = new ByteArrayOutputStream(5 * 1024 * 1024);
g.start(out);
g.setDelay(millisForFrame);
g.setRepeat(1);
for(BufferedImage i : images) {
g.addFrame(i);
}
g.finish();
byte[] bytes = out.toByteArray();
return Toolkit.getDefaultToolkit().createImage(bytes);
}
And GifSequenceWriter looks like this:
public class GifSequenceWriter {
protected ImageWriter gifWriter;
protected ImageWriteParam imageWriteParam;
protected IIOMetadata imageMetaData;
public GifSequenceWriter(ImageOutputStream outputStream, int imageType, int timeBetweenFramesMS,
boolean loopContinuously) throws IIOException, IOException {
gifWriter = getWriter();
imageWriteParam = gifWriter.getDefaultWriteParam();
ImageTypeSpecifier imageTypeSpecifier = ImageTypeSpecifier.createFromBufferedImageType(imageType);
imageMetaData = gifWriter.getDefaultImageMetadata(imageTypeSpecifier, imageWriteParam);
String metaFormatName = imageMetaData.getNativeMetadataFormatName();
IIOMetadataNode root = (IIOMetadataNode) imageMetaData.getAsTree(metaFormatName);
IIOMetadataNode graphicsControlExtensionNode = getNode(root, "GraphicControlExtension");
graphicsControlExtensionNode.setAttribute("disposalMethod", "none");
graphicsControlExtensionNode.setAttribute("userInputFlag", "FALSE");
graphicsControlExtensionNode.setAttribute("transparentColorFlag", "FALSE");
graphicsControlExtensionNode.setAttribute("delayTime", Integer.toString(timeBetweenFramesMS / 10));
graphicsControlExtensionNode.setAttribute("transparentColorIndex", "0");
IIOMetadataNode commentsNode = getNode(root, "CommentExtensions");
commentsNode.setAttribute("CommentExtension", "Created by MAH");
IIOMetadataNode appEntensionsNode = getNode(root, "ApplicationExtensions");
IIOMetadataNode child = new IIOMetadataNode("ApplicationExtension");
child.setAttribute("applicationID", "NETSCAPE");
child.setAttribute("authenticationCode", "2.0");
int loop = loopContinuously ? 0 : 1;
child.setUserObject(new byte[] { 0x1, (byte) (loop & 0xFF), (byte) (loop >> 8 & 0xFF) });
appEntensionsNode.appendChild(child);
imageMetaData.setFromTree(metaFormatName, root);
gifWriter.setOutput(outputStream);
gifWriter.prepareWriteSequence(null);
}
public void writeToSequence(RenderedImage img) throws IOException {
gifWriter.writeToSequence(new IIOImage(img, null, imageMetaData), imageWriteParam);
}
public void close() throws IOException {
gifWriter.endWriteSequence();
}
private static ImageWriter getWriter() throws IIOException {
Iterator<ImageWriter> iter = ImageIO.getImageWritersBySuffix("gif");
if(!iter.hasNext()) {
throw new IIOException("No GIF Image Writers Exist");
}
return iter.next();
}
private static IIOMetadataNode getNode(IIOMetadataNode rootNode, String nodeName) {
int nNodes = rootNode.getLength();
for(int i = 0; i < nNodes; i++) {
if(rootNode.item(i).getNodeName().compareToIgnoreCase(nodeName) == 0) {
return (IIOMetadataNode) rootNode.item(i);
}
}
IIOMetadataNode node = new IIOMetadataNode(nodeName);
rootNode.appendChild(node);
return node;
}
}

The easiest fix for this problem is to just create a .gif file which is "well-formed", i.e. contains the appropriate frame rate. This can be achieved with the help of various online converters, just google "gif speed changer".

Xuggler screenRecording code affecting sound recording

I am working on an application for screencast with audio. screen recording with sound is working fine but the issue is that suppose I do recording of 5 mins then generated video file is of 5 min but generated audio file is of 4 min 45 sec. So basically the issue is that both audio and video are not in sync, audio file duration is less as compared to video file.
Both audio and video file recording are running in separate thread but still something is wrong.
VideoCapturing code:
public void run() {
setVideoParameters();
FRAME_RATE = frameRate;
// let's make a IMediaWriter to write the file.
writer = ToolFactory.makeWriter(movieFile.getName());
screenBounds = new Rectangle(RecorderSettings.m_CapRectX,
RecorderSettings.m_CapRecY,
(int) RecorderSettings.m_CapRectWidth,
(int) RecorderSettings.m_CapRecHeight);
// We tell it we're going to add one video stream, with id 0,
// at position 0, and that it will have a fixed frame rate of
// FRAME_RATE.
// ScreenWidth && ScreenHeight multiplied by 3/4 to reduce pixel to 3/4
// of actual.
// writer.addVideoStream(0, 0, ICodec.ID.CODEC_ID_MPEG4,
// screenBounds.width , screenBounds.height );
writer.addVideoStream(0, 0, vcodec.getID(),
(screenBounds.width * upperLimit) / lowerLimit,
(screenBounds.height * upperLimit) / lowerLimit);
// To have start time of recording
startTime = System.nanoTime();
while (isStopProceesBtnClk) {
try {
if (!isStopProceesBtnClk) {
break;
} else {
synchronized (this) {
while (isPauseProceesBtnClk) {
try {
// catches starting time of pause.
pauseStartTime = System.nanoTime();
wait();
} catch (Exception e) {
e.printStackTrace();
}
}
}
BufferedImage screen = getDesktopScreenshot();
// convert to the right image type
BufferedImage bgrScreen = convertToType(screen, BufferedImage.TYPE_3BYTE_BGR);
// encode the image to stream #0
if (totalPauseTime > 0) {
writer.encodeVideo(0, bgrScreen, (System.nanoTime() - startTime)- totalPauseTime, TimeUnit.NANOSECONDS);
} else {
writer.encodeVideo(0, bgrScreen, System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
}
// sleep for frame rate milliseconds
try {
Thread.sleep((long) (1000 / FRAME_RATE));
} catch (InterruptedException e) {
// ignore
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
try {
writer.close();
writer = null;
Runtime.getRuntime().gc();
} catch (Exception e) {
// ignore errors
}
// tell the writer to close and write the trailer if needed
}
public static BufferedImage convertToType(BufferedImage sourceImage, int targetType) {
BufferedImage image;
// if the source image is already the target type, return the source
// image
if (sourceImage.getType() == targetType) {
image = sourceImage;
}
// otherwise create a new image of the target type and draw the new
// image
else {
image = new BufferedImage(sourceImage.getWidth(), sourceImage.getHeight(), targetType);
if (true) {
int x = MouseInfo.getPointerInfo().getLocation().x - 25;
int y = MouseInfo.getPointerInfo().getLocation().y - 37;
Graphics2D graphics2D = sourceImage.createGraphics();// getGraphics().drawImage(m_MouseIcon,
// x, y, 48, 48, null);
graphics2D.drawImage(SimpleWebBrowserExample.m_MouseIcon, x, y,
48, 48, null);
}
image.getGraphics().drawImage(sourceImage, 0, 0, null);
}
return image;
}
private BufferedImage getDesktopScreenshot() {
try {
// Robot captures screen shot
Robot robot = new Robot();
Rectangle captureSize = new Rectangle(screenBounds);
return robot.createScreenCapture(captureSize);
} catch (AWTException e) {
e.printStackTrace();
return null;
}
}
AudioCapturing Code:
public void run() {
init();
DataLine.Info info = new DataLine.Info(TargetDataLine.class,audioFormat,(int) (m_AudioFreq * sampleSizeInBytes));
try
{
m_TargetLine = (TargetDataLine) AudioSystem.getLine(info);
m_TargetLine.open(audioFormat, info.getMaxBufferSize());
}
catch(Exception exp){
exp.printStackTrace();
}
AudioFileFormat.Type targetType = AudioFileFormat.Type.WAVE;
try
{
m_outputFile = new File(bufferFileName);
while (m_outputFile.exists() && !m_outputFile.delete())
{
m_outputFile = BerylsUtility.getNextFile(m_outputFile);
}
FileOutputStream outFileStream = new FileOutputStream(m_outputFile);
audioOutStream = new BufferedOutputStream(outFileStream,memoryBufferSize);
}
catch (FileNotFoundException fe){
System.out.println("FileNotFoundException in VoiceCapturing.java :: " + fe);
}
catch (OutOfMemoryError oe){
System.out.println("OutOfMemoryError in VoiceCapturing.java " + oe);
}
while (isStopProceesBtnClk) {
try {
if (!isStopProceesBtnClk) {
break;
} else {
synchronized (this) {
while (isPauseProceesBtnClk) {
try {
wait();
} catch (Exception e) {
e.printStackTrace();
}
}
}
try
{
m_TargetLine.start();
int cnt = m_TargetLine.read(tempBuffer,0,tempBuffer.length);
if(cnt > 0){
audioOutStream.write(tempBuffer,0,cnt);
}
}
catch (Exception e){
System.out.println("Exception in VoiceCapturing.java :: " + e);
}
/*finally{
finish();
}*/
}
} catch (Exception e) {
e.printStackTrace();
}
}
finish();
}
public synchronized void finish()
{
try
{
System.out.println("AudioFinish");
audioOutStream.close();
FileInputStream audioInAgain = new FileInputStream(m_outputFile);
long sampleBytes = m_outputFile.length();
long sizeOfFrame = (long) m_SampleRate * m_Channels / 8;
BufferedInputStream buffAudioIn = new BufferedInputStream(audioInAgain, memoryBufferSize);
AudioInputStream a_input = new AudioInputStream(buffAudioIn, audioFormat, sampleBytes / sizeOfFrame);
while (m_AudioFile.exists() && !m_AudioFile.canWrite())
{
m_AudioFile = BerylsUtility.getNextFile(m_AudioFile);
}
AudioSystem.write(a_input, m_targetType, m_AudioFile);
buffAudioIn.close();
m_outputFile.delete();
}
catch (Exception e)
{
e.printStackTrace();
}
}
could someone guide me on this...
Thanks.