I have inherited a code snippet which draws audio waveform of a given file. But this waveform is a simple image built using JAVA vector graphics without any labeling, Axes information etc. I would like to port it to the jfreechart to increase it's informative value. My problem is that the code is cryptic to say the least.
public class Plotter {
AudioInputStream audioInputStream;
Vector<Line2D.Double> lines = new Vector<Line2D.Double>();
String errStr;
Capture capture = new Capture();
double duration, seconds;
//File file;
String fileName = "out.png";
SamplingGraph samplingGraph;
String waveformFilename;
Color imageBackgroundColor = new Color(20,20,20);
public Plotter(URL url, String waveformFilename) throws Exception {
if (url != null) {
try {
errStr = null;
this.fileName = waveformFilename;
audioInputStream = AudioSystem.getAudioInputStream(url);
long milliseconds = (long)((audioInputStream.getFrameLength() * 1000) / audioInputStream.getFormat().getFrameRate());
duration = milliseconds / 1000.0;
samplingGraph = new SamplingGraph();
samplingGraph.createWaveForm(null);
} catch (Exception ex) {
reportStatus(ex.toString());
throw ex;
}
} else {
reportStatus("Audio file required.");
}
}
/**
* Render a WaveForm.
*/
class SamplingGraph implements Runnable {
private Thread thread;
private Font font10 = new Font("serif", Font.PLAIN, 10);
private Font font12 = new Font("serif", Font.PLAIN, 12);
Color jfcBlue = new Color(000, 000, 255);
Color pink = new Color(255, 175, 175);
public SamplingGraph() {
}
public void createWaveForm(byte[] audioBytes) {
lines.removeAllElements(); // clear the old vector
AudioFormat format = audioInputStream.getFormat();
if (audioBytes == null) {
try {
audioBytes = new byte[
(int) (audioInputStream.getFrameLength()
* format.getFrameSize())];
audioInputStream.read(audioBytes);
} catch (Exception ex) {
reportStatus(ex.getMessage());
return;
}
}
int w = 500;
int h = 200;
int[] audioData = null;
if (format.getSampleSizeInBits() == 16) {
int nlengthInSamples = audioBytes.length / 2;
audioData = new int[nlengthInSamples];
if (format.isBigEndian()) {
for (int i = 0; i < nlengthInSamples; i++) {
/* First byte is MSB (high order) */
int MSB = (int) audioBytes[2*i];
/* Second byte is LSB (low order) */
int LSB = (int) audioBytes[2*i+1];
audioData[i] = MSB << 8 | (255 & LSB);
}
} else {
for (int i = 0; i < nlengthInSamples; i++) {
/* First byte is LSB (low order) */
int LSB = (int) audioBytes[2*i];
/* Second byte is MSB (high order) */
int MSB = (int) audioBytes[2*i+1];
audioData[i] = MSB << 8 | (255 & LSB);
}
}
} else if (format.getSampleSizeInBits() == 8) {
int nlengthInSamples = audioBytes.length;
audioData = new int[nlengthInSamples];
if (format.getEncoding().toString().startsWith("PCM_SIGN")) {
for (int i = 0; i < audioBytes.length; i++) {
audioData[i] = audioBytes[i];
}
} else {
for (int i = 0; i < audioBytes.length; i++) {
audioData[i] = audioBytes[i] - 128;
}
}
}
int frames_per_pixel = audioBytes.length / format.getFrameSize()/w;
byte my_byte = 0;
double y_last = 0;
int numChannels = format.getChannels();
for (double x = 0; x < w && audioData != null; x++) {
int idx = (int) (frames_per_pixel * numChannels * x);
if (format.getSampleSizeInBits() == 8) {
my_byte = (byte) audioData[idx];
} else {
my_byte = (byte) (128 * audioData[idx] / 32768 );
}
double y_new = (double) (h * (128 - my_byte) / 256);
lines.add(new Line2D.Double(x, y_last, x, y_new));
y_last = y_new;
}
saveToFile();
}
public void saveToFile() {
int w = 500;
int h = 200;
int INFOPAD = 15;
BufferedImage bufferedImage = new BufferedImage(w, h, BufferedImage.TYPE_INT_RGB);
Graphics2D g2 = bufferedImage.createGraphics();
createSampleOnGraphicsContext(w, h, INFOPAD, g2);
g2.dispose();
// Write generated image to a file
try {
// Save as PNG
File file = new File(fileName);
System.out.println(file.getAbsolutePath());
ImageIO.write(bufferedImage, "png", file);
JOptionPane.showMessageDialog(null,
new JLabel(new ImageIcon(fileName)));
} catch (IOException e) {
}
}
private void createSampleOnGraphicsContext(int w, int h, int INFOPAD, Graphics2D g2) {
g2.setBackground(imageBackgroundColor);
g2.clearRect(0, 0, w, h);
g2.setColor(Color.white);
g2.fillRect(0, h-INFOPAD, w, INFOPAD);
if (errStr != null) {
g2.setColor(jfcBlue);
g2.setFont(new Font("serif", Font.BOLD, 18));
g2.drawString("ERROR", 5, 20);
AttributedString as = new AttributedString(errStr);
as.addAttribute(TextAttribute.FONT, font12, 0, errStr.length());
AttributedCharacterIterator aci = as.getIterator();
FontRenderContext frc = g2.getFontRenderContext();
LineBreakMeasurer lbm = new LineBreakMeasurer(aci, frc);
float x = 5, y = 25;
lbm.setPosition(0);
while (lbm.getPosition() < errStr.length()) {
TextLayout tl = lbm.nextLayout(w-x-5);
if (!tl.isLeftToRight()) {
x = w - tl.getAdvance();
}
tl.draw(g2, x, y += tl.getAscent());
y += tl.getDescent() + tl.getLeading();
}
} else if (capture.thread != null) {
g2.setColor(Color.black);
g2.setFont(font12);
//g2.drawString("Length: " + String.valueOf(seconds), 3, h-4);
} else {
g2.setColor(Color.black);
g2.setFont(font12);
//g2.drawString("File: " + fileName + " Length: " + String.valueOf(duration) + " Position: " + String.valueOf(seconds), 3, h-4);
if (audioInputStream != null) {
// .. render sampling graph ..
g2.setColor(jfcBlue);
for (int i = 1; i < lines.size(); i++) {
g2.draw((Line2D) lines.get(i));
}
// .. draw current position ..
if (seconds != 0) {
double loc = seconds/duration*w;
g2.setColor(pink);
g2.setStroke(new BasicStroke(3));
g2.draw(new Line2D.Double(loc, 0, loc, h-INFOPAD-2));
}
}
}
}
public void start() {
thread = new Thread(this);
thread.setName("SamplingGraph");
thread.start();
seconds = 0;
}
public void stop() {
if (thread != null) {
thread.interrupt();
}
thread = null;
}
public void run() {
seconds = 0;
while (thread != null) {
if ( (capture.line != null) && (capture.line.isActive()) ) {
long milliseconds = (long)(capture.line.getMicrosecondPosition() / 1000);
seconds = milliseconds / 1000.0;
}
try { thread.sleep(100); } catch (Exception e) { break; }
while ((capture.line != null && !capture.line.isActive()))
{
try { thread.sleep(10); } catch (Exception e) { break; }
}
}
seconds = 0;
}
} // End class SamplingGraph
/**
* Reads data from the input channel and writes to the output stream
*/
class Capture implements Runnable {
TargetDataLine line;
Thread thread;
public void start() {
errStr = null;
thread = new Thread(this);
thread.setName("Capture");
thread.start();
}
public void stop() {
thread = null;
}
private void shutDown(String message) {
if ((errStr = message) != null && thread != null) {
thread = null;
samplingGraph.stop();
System.err.println(errStr);
}
}
public void run() {
duration = 0;
audioInputStream = null;
// define the required attributes for our line,
// and make sure a compatible line is supported.
AudioFormat format = audioInputStream.getFormat();
DataLine.Info info = new DataLine.Info(TargetDataLine.class,
format);
if (!AudioSystem.isLineSupported(info)) {
shutDown("Line matching " + info + " not supported.");
return;
}
// get and open the target data line for capture.
try {
line = (TargetDataLine) AudioSystem.getLine(info);
line.open(format, line.getBufferSize());
} catch (LineUnavailableException ex) {
shutDown("Unable to open the line: " + ex);
return;
} catch (SecurityException ex) {
shutDown(ex.toString());
//JavaSound.showInfoDialog();
return;
} catch (Exception ex) {
shutDown(ex.toString());
return;
}
// play back the captured audio data
ByteArrayOutputStream out = new ByteArrayOutputStream();
int frameSizeInBytes = format.getFrameSize();
int bufferLengthInFrames = line.getBufferSize() / 8;
int bufferLengthInBytes = bufferLengthInFrames * frameSizeInBytes;
byte[] data = new byte[bufferLengthInBytes];
int numBytesRead;
line.start();
while (thread != null) {
if((numBytesRead = line.read(data, 0, bufferLengthInBytes)) == -1) {
break;
}
out.write(data, 0, numBytesRead);
}
// we reached the end of the stream. stop and close the line.
line.stop();
line.close();
line = null;
// stop and close the output stream
try {
out.flush();
out.close();
} catch (IOException ex) {
ex.printStackTrace();
}
// load bytes into the audio input stream for playback
byte audioBytes[] = out.toByteArray();
ByteArrayInputStream bais = new ByteArrayInputStream(audioBytes);
audioInputStream = new AudioInputStream(bais, format, audioBytes.length / frameSizeInBytes);
long milliseconds = (long)((audioInputStream.getFrameLength() * 1000) / format.getFrameRate());
duration = milliseconds / 1000.0;
try {
audioInputStream.reset();
} catch (Exception ex) {
ex.printStackTrace();
return;
}
samplingGraph.createWaveForm(audioBytes);
}
} // End class Capture
}
I have gone through it several times and know that the below part is where the audio values are calculated but my problem is that I have no idea how can I retrieve the time information at that point, i.e that value belongs to what time interval.
int frames_per_pixel = audioBytes.length / format.getFrameSize()/w;
byte my_byte = 0;
double y_last = 0;
int numChannels = format.getChannels();
for (double x = 0; x < w && audioData != null; x++) {
int idx = (int) (frames_per_pixel * numChannels * x);
if (format.getSampleSizeInBits() == 8) {
my_byte = (byte) audioData[idx];
} else {
my_byte = (byte) (128 * audioData[idx] / 32768 );
}
double y_new = (double) (h * (128 - my_byte) / 256);
lines.add(new Line2D.Double(x, y_last, x, y_new));
y_last = y_new;
}
I would like to plot it using XYSeriesPLot of jfreechart but having trouble calculating required values of x(time ) and y (this is amplitude but is it y_new in this code)?
I understand it is a very easy thing but I am new to this whole audio stuff, I understand the theory behind audio files but this seems to be a simple problem with a tough solution
enter link description here
The key thing to realize is that, in the provided code, the plot is expected to be at a much lower resolution than the actual audio data. For example, consider the following waveform:
The plotting code then represents the data as the blue boxes in the graph:
When the boxes are 1-pixel wide, this correspond to the lines with endpoints (x,y_last) and (x,y_new). As you can see, when the waveform is sufficiently smooth the range of amplitudes from y_last to y_new is a fair approximation to the samples within the box.
Now this representation can be convenient when trying to render the waveform in a pixel-by-pixel fashion (raster display). However, for XYPlot graphs (as can be found in jfreechart) you only need to specify a sequence of (x,y) points and the XYPlot takes care of drawing segments between those point. This corresponds to the green line in the following graph:
In theory, you could just provide every single sample as-is to the XYPlot. However, unless you have few samples, this tends to be quite heavy to plot. So, typically one would downsample the data first. If the waveform is sufficiently smooth the downsampling process reduces to a decimation (i.e. taking 1 every N samples). The decimation factor N then controls the tradeoff between rendering performance and waveform approximation accuracy. Note that if the decimation factor frames_per_pixel used in the provided code to generate a good raster display (i.e. one where the waveform feature that you'll like to see are not hidden by the blocky pixel look, and that does not show aliasing artifacts), the same factor should still be sufficient for the XYPlot (in fact you may be able to downsample a bit more).
As far as mapping the samples to a time/amplitude axes, I would not use the x and y parameters as they are defined in the plotting code provided: they are just pixel indices applicable to a raster-type display (as is the blue box representation above).
Rather I'd map the sample index (idx in the provided code) directly to the time axis by dividing by the sampling rate (which you can get from format.getFrameRate()).
Similarly, I'd map the full-scale sample values to [-1,+1] range by dividing the audioData[idx] samples by either 128 for 8-bits-per-sample data, and by 32768 for 16-bits-per-sample data.
The w and h parameters' main purpose would remain to configure the plotting area size, but would no longer be directly required to compute the XYPlot input (the XYPlot itself takes care of mapping time/amplitude values to pixel coordinates). The w parameter on the other hand also served the additional purpose of determining the number of points to draw. Now you may want to control the number of points based on how much decimation the waveform can sustain without showing too much distortion, or you could keep it as-is to display the waveform at the maximum available plot resolution (with some performance cost).
Note however that you may have to convert frames_per_pixel to a floating point value if you are expecting to display waveforms with fewer than w samples.
Related
tl;dr: when I calculate and visualize an FFT for any audio sample, the visualization is full of background noise to the point of swallowing the signal. Why?
Full question/details: I'm (long-term) attempting to make an audio fingerprinter following the blog post here. The code given is incomplete and this is my first time doing this kind of audio processing, so I'm filling in blanks in both the code and my knowledge as I go.
The post first explains running the audio sample through a windowed FFT. I'm using the Apache Commons FastFourierTransform class for this, and I've sanity checked some very simple bit patterns against their computed FFTs with good results.
The post then detours into making a basic spectrum analyzer to confirm that the FFT is working as intended, and here's where I see my issue.
The post's spectrum analyzer is very simple code. results is a Complex[][] containing the raw results of the FFT.
for(int i = 0; i < results.length; i++) {
int freq = 1;
for(int line = 1; line < size; line++) {
// To get the magnitude of the sound at a given frequency slice
// get the abs() from the complex number.
// In this case I use Math.log to get a more managable number (used for color)
double magnitude = Math.log(results[i][freq].abs()+1);
// The more blue in the color the more intensity for a given frequency point:
g2d.setColor(new Color(0,(int)magnitude*10,(int)magnitude*20));
// Fill:
g2d.fillRect(i*blockSizeX, (size-line)*blockSizeY,blockSizeX,blockSizeY);
// I used a improviced logarithmic scale and normal scale:
if (logModeEnabled && (Math.log10(line) * Math.log10(line)) > 1) {
freq += (int) (Math.log10(line) * Math.log10(line));
} else {
freq++;
}
}
}
The post's visualization results as shown are good quality. This is a picture of a sample from Aphex Twin's song "Equation", which has an image of the artist's face encoded into it:
Indeed, when I take a short sample from the song (starting around 5:25) and run it through the online spectrum analyzer here for a sanity check, I get a pretty legible rendition of the face:
But my own results on the exact same audio file are a lot noisier, to the point that I have to mess with the spectrum analyzer's colors just to get something to show at all, and I never get to see the full face:
I get this kind of heavy background noise with any audio sample I try, across a variety of factors - MP3 or WAV, mono or stereo, short sample or long sample, a simple audio pattern or a complex song.
I've experimented with different FFT window sizes, conversion from raw FFT frequency output to power or dB, and different ways of visualizing the FFT output just in case the issue is with the visualization. None of that has helped.
I looked up the WebAudio implementation behind the Academo online spectrum analyzer, and it looks like there's a lot going on there: a Blackman window instead of my simple rectangular window to smooth the audio sampling; an interesting FFT with a built-in multiplication by 1/N, which seems to match the Unitary normalization provided by Apache Commons' FFT class; a smoothing function on the frequency data; and conversion from frequency values to dB to top it all off. Just for fun, I tried mimicking the WebAudio setup, but with about the same or even worse noise in the results, which suggests the issue is in the FFT step rather than any of the pre or post processing. I'm not sure how this can be the case when the FFT passes my basic calculation checks. I suppose the issue could be in the audio reading step, that I'm passing garbage into the FFT and getting garbage back, but I've experimented with reading the audio file and immediately writing a copy back to disk, and the new copy sounds just fine.
Here's a simplified version of my code that demonstrates the issue:
//Application.java
import java.io.File;
import java.io.IOException;
import javax.sound.sampled.AudioFormat;
import javax.sound.sampled.AudioInputStream;
import javax.sound.sampled.AudioSystem;
import javax.sound.sampled.UnsupportedAudioFileException;
public class Application {
public static void main(String[] args) {
sanityCheckFft();
File inputFile = new File("C:\\Aphex Twin face.mp3");
AudioProcessor audioProcessor = new AudioProcessor();
SpectrumAnalyzer debugSpectrumAnalyzer = new SpectrumAnalyzer();
try {
AudioInputStream audioStream = readAudioFile(inputFile);
byte[] bytes = audioStream.readAllBytes();
AudioFormat audioFormat = audioStream.getFormat();
FftChunk[] fft = audioProcessor.calculateFft(bytes, audioFormat, 4096);
debugSpectrumAnalyzer.debugFftSpectrum(fft);
}
catch (Exception e) {
e.printStackTrace();
}
}
//https://github.com/hendriks73/ffsampledsp#usage
private static AudioInputStream readAudioFile(File file) throws IOException, UnsupportedAudioFileException {
// compressed stream
AudioInputStream mp3InputStream = AudioSystem.getAudioInputStream(file);
// AudioFormat describing the compressed stream
AudioFormat mp3Format = mp3InputStream.getFormat();
// AudioFormat describing the desired decompressed stream
int sampleSizeInBits = 16;
int frameSize = 16 * mp3Format.getChannels() / 8;
AudioFormat pcmFormat = new AudioFormat(AudioFormat.Encoding.PCM_SIGNED,
mp3Format.getSampleRate(),
sampleSizeInBits,
mp3Format.getChannels(),
frameSize,
mp3Format.getSampleRate(),
mp3Format.isBigEndian());
// actually decompressed stream (signed PCM)
final AudioInputStream pcmInputStream = AudioSystem.getAudioInputStream(pcmFormat, mp3InputStream);
return pcmInputStream;
}
private static void sanityCheckFft() {
AudioProcessor audioProcessor = new AudioProcessor();
//pattern 1: one block
byte[] bytePattern1 = new byte[] {2, 1, -1, 5, 0, 3, 0, -4};
FftChunk[] fftResults1 = audioProcessor.calculateFft(bytePattern1, null, 8);
//expected results: [6 + 0J, -5.778 - 3.95J, 3 + -3J, 9.778 - 5.95J, -4 + 0J, 9.778 + 5.95J, 3 + 3J, -5.778 + 3.95J]
//expected results verified with https://engineering.icalculator.info/discrete-fourier-transform-calculator.html
//pattern 2: two blocks
byte[] bytePattern2 = new byte[] {2, 1, -1, 5, 0, 3, 0, -4};
FftChunk[] fftResults2 = audioProcessor.calculateFft(bytePattern1, null, 4);
//expected results: [7 + 0J, 3 + 4J, -5 + 0J, 3 - 4J], [-1 + 0J, 0 - 7J, 1 + 0J, 0 + 7J]
//expected results verified with https://engineering.icalculator.info/discrete-fourier-transform-calculator.html
/* pattern 3
* "Try a signal of alternate ones and negative ones with zeros between each. (i.e. 1,0,-1,0, 1,0,-1,0, ...) For a real FFT of length 1024, this should give you a single peak at out[255] ( the 256th frequency bin)"
* - https://stackoverflow.com/questions/8887896/why-does-my-kiss-fft-plot-show-duplicate-peaks-mirrored-on-the-y-axis#comment11127476_8887896
*/
byte[] bytePattern3 = new byte[1024];
byte[] pattern3Phases = new byte[] {1, 0, -1, 0};
for (int pattern3Index = 0; pattern3Index < bytePattern3.length; pattern3Index++) {
int pattern3PhaseIndex = pattern3Index % pattern3Phases.length;
byte pattern3Phase = pattern3Phases[pattern3PhaseIndex];
bytePattern3[pattern3Index] = pattern3Phase;
}
FftChunk[] fftResults3 = audioProcessor.calculateFft(bytePattern3, null, 1024);
//expected results: 0s except for fftResults[256]
}
}
//AudioProcessor.java
import javax.sound.sampled.AudioFormat;
import org.apache.commons.math3.complex.Complex;
import org.apache.commons.math3.transform.DftNormalization;
import org.apache.commons.math3.transform.FastFourierTransformer;
import org.apache.commons.math3.transform.TransformType;
public class AudioProcessor {
public FftChunk[] calculateFft(byte[] bytes, AudioFormat audioFormat, int debugActualChunkSize) {
//final int BITS_PER_BYTE = 8;
//final int PREFERRED_CHUNKS_PER_SECOND = 60;
/* turn the audio bytes into chunks. Each chunk represents the audio played during a certain window of time, defined by the audio's play rate (frame rate * frame size = the number of bytes processed per second)
* and the number of chunks we want to cut each second of audio into.
* frame rate * frame size = 1 second worth of bytes
* if we divide each second worth of data into chunksPerSecond chunks, that gives us:
* 1 chunk in bytes = 1 second in bytes / chunksPerSecond
* 1 chunk in bytes = frame rate * frame size / chunksPerSecond
*/
//float oneSecondByteLength = audioFormat.getChannels() * audioFormat.getSampleRate() * (audioFormat.getSampleSizeInBits() / BITS_PER_BYTE);
//int preferredChunkSize = (int)(oneSecondByteLength / PREFERRED_CHUNKS_PER_SECOND);
//int actualChunkSize = getPreviousPowerOfTwo(preferredChunkSize);
int chunkCount = bytes.length / debugActualChunkSize;
FastFourierTransformer fastFourierTransformer = new FastFourierTransformer(DftNormalization.STANDARD);
FftChunk[] fftResults = new FftChunk[chunkCount];
//set up each chunk individually for FFT processing
for (int timeIndex = 0; timeIndex < chunkCount; timeIndex++) {
//to map the input into the frequency domain, we need complex numbers (we only use the normal half of the Complex, but we need to provide & receive the entire Complex value)
Complex[] currentChunkComplexRepresentation = new Complex[debugActualChunkSize];
for (int currentChunkIndex = 0; currentChunkIndex < debugActualChunkSize; currentChunkIndex++) {
//get the next byte in the current audio chunk
int currentChunkCurrentByteIndex = (timeIndex * debugActualChunkSize) + currentChunkIndex;
byte currentChunkCurrentByte = bytes[currentChunkCurrentByteIndex];
//put the time domain data into a complex number with imaginary part as 0
currentChunkComplexRepresentation[currentChunkIndex] = new Complex(currentChunkCurrentByte, 0);
}
//perform FFT analysis on the chunk
Complex[] currentChunkFftResults = fastFourierTransformer.transform(currentChunkComplexRepresentation, TransformType.FORWARD);
FftChunk fftResult = new FftChunk(currentChunkFftResults);
fftResults[timeIndex] = fftResult;
}
return fftResults;
}
}
//FftChunk.java
import org.apache.commons.math3.complex.Complex;
import lombok.Data;
import lombok.RequiredArgsConstructor;
#Data
#RequiredArgsConstructor
public class FftChunk {
private final Complex[] fftResults;
}
//SpectrumAnalyzer.java
import java.awt.Color;
import java.awt.Dimension;
import java.awt.Graphics;
import java.awt.Graphics2D;
import java.awt.image.BufferedImage;
import javax.swing.JComponent;
import javax.swing.JFrame;
import javax.swing.JScrollPane;
import javax.swing.ScrollPaneConstants;
import javax.swing.WindowConstants;
import org.apache.commons.math3.complex.Complex;
public class SpectrumAnalyzer {
private JFrame frame;
private SpectrumAnalyzerComponent spectrumAnalyzerComponent;
public void debugFftSpectrum(FftChunk[] spectrum) {
Dimension windowSize = new Dimension(1000, 600);
spectrumAnalyzerComponent = new SpectrumAnalyzerComponent();
JScrollPane scrollPanel = new JScrollPane(spectrumAnalyzerComponent);
scrollPanel.setHorizontalScrollBarPolicy(ScrollPaneConstants.HORIZONTAL_SCROLLBAR_ALWAYS);
scrollPanel.setPreferredSize(windowSize);
frame = new JFrame();
frame.add(scrollPanel);
frame.setSize(windowSize);
frame.setVisible(true);
frame.setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE);
spectrumAnalyzerComponent.analyze(spectrum);
}
}
#SuppressWarnings("serial")
class SpectrumAnalyzerComponent extends JComponent {
private FftChunk[] spectrum;
private boolean useLogScale = true;
private int blockSizeX = 1;
private int blockSizeY = 1;
private BufferedImage cachedImage;
public void analyze(FftChunk[] spectrum) {
this.spectrum = spectrum;
if (spectrum == null) {
cachedImage = null;
}
else {
int newWidth = (spectrum.length * blockSizeX) + blockSizeX;
int newHeight = 0;
for (FftChunk audioChunk : spectrum) {
Complex[] chunkFftResults = audioChunk.getFftResults();
int chunkHeight = calculatePixelHeight(chunkFftResults);
if (chunkHeight > newHeight) {
newHeight = chunkHeight;
}
}
Dimension newSize = new Dimension(newWidth, newHeight);
this.setPreferredSize(newSize);
this.setSize(newSize);
this.revalidate();
cachedImage = new BufferedImage(newWidth, newHeight, BufferedImage.TYPE_INT_RGB);
drawSpectrum(cachedImage.createGraphics());
}
this.repaint(); //force an immediate redraw
}
#Override
public void paint(Graphics graphics) {
if (cachedImage != null) {
graphics.drawImage(cachedImage, 0, 0, null);
}
}
//based on the spectrum analyzer from https://www.royvanrijn.com/blog/2010/06/creating-shazam-in-java/
private void drawSpectrum(Graphics2D graphics) {
if (this.spectrum == null) {
return;
}
int windowHeight = this.getSize().height;
for (int timeIndex = 0; timeIndex < spectrum.length; timeIndex++) {
System.out.println(String.format("Drawing time chunk %d/%d", timeIndex + 1, spectrum.length));
FftChunk currentChunk = spectrum[timeIndex];
Complex[] currentChunkFftResults = currentChunk.getFftResults();
int fftIndex = 0;
int yIndex = 1;
/* each chunk contains N elements, where N is the size of the FFT window. The first N/2 elements are positive and the last N/2 elements are negative, but they're otherwise mirrors
* of each other. We only want the positive half.
* Additionally, because we're working with audio samples, our FFT is a "real" FFT (FFT on real numbers -
* https://stackoverflow.com/questions/8887896/why-does-my-kiss-fft-plot-show-duplicate-peaks-mirrored-on-the-y-axis/10744384#10744384 ), which produces a mirror of its own inside
* the positive elements. We need to further divide the positive elements in half. This leaves us with the first N/4 elements after all is said and done.
*/
while (fftIndex < currentChunkFftResults.length / 4) {
Complex currentChunkFftResult = currentChunkFftResults[fftIndex];
// To get the magnitude of the sound at a given frequency slice
// get the abs() from the complex number.
// In this case I use Math.log to get a more managable number (used for color)
double magnitude = Math.log10(currentChunkFftResult.abs() + 1);
// The more blue in the color the more intensity for a given frequency point:
/*int red = 0;
int green = (int) magnitude * 10;
int blue = (int) magnitude * 20;
graphics.setColor(new Color(red, green, blue));*/
float hue = (float)(magnitude / 255 * 100);
int colorValue = Color.HSBtoRGB(hue, 100, 50);
graphics.setColor(new Color(colorValue));
// Fill:
graphics.fillRect(timeIndex * blockSizeX, (windowHeight - yIndex) * blockSizeY, blockSizeX, blockSizeY);
// I used an improvised logarithmic scale and normal scale:
int normalScaleFrequencyDelta = 1;
int logScaleFrequencyDelta = (int)(Math.log10(yIndex) * Math.log10(yIndex));
if (logScaleFrequencyDelta < 1) {
logScaleFrequencyDelta = 1;
}
if (useLogScale) {
fftIndex = fftIndex + logScaleFrequencyDelta;
}
else {
fftIndex = fftIndex + normalScaleFrequencyDelta;
}
yIndex = yIndex + 1;
}
}
}
private int calculatePixelHeight(Complex[] fftResults) {
int fftIndex = 1;
int tempPixelCount = 1;
int pixelCount = 1;
while (fftIndex < fftResults.length / 4) {
pixelCount = tempPixelCount;
int normalScaleFrequencyDelta = 1;
int logScaleFrequencyDelta = (int)(Math.log10(tempPixelCount) * Math.log10(tempPixelCount));
if (logScaleFrequencyDelta < 1) {
logScaleFrequencyDelta = 1;
}
if (useLogScale) {
fftIndex = fftIndex + logScaleFrequencyDelta;
}
else {
fftIndex = fftIndex + normalScaleFrequencyDelta;
}
tempPixelCount = tempPixelCount + 1;
}
return pixelCount;
}
}
//build.gradle
plugins {
//Java application plugin
id 'application'
//Project Lombok plugin
id 'io.freefair.lombok' version '6.5.0.2'
}
repositories {
// Use Maven Central for resolving dependencies.
mavenCentral()
}
dependencies {
implementation 'com.tagtraum:ffsampledsp-complete:0.9.46'
implementation 'org.apache.commons:commons-math3:3.6.1'
}
I'm trying to test a class which purpose is taking a specification of what to do with a BufferedImage array and proccess it.
I'm concerned about how to aproach this task, it makes no sense to me just duplicate the function code on the test in order to generate a "expected bufferedImage array" and check if the generated BufferedImages are equals to the returned by the class methods.
I've read some slightly similar questions here, but the answers was "charge expected images from file and check with returned ones by class methods" but that sounds real hard to maintain if the proccess changes, new methods are added to the renderer class, it's hard or imposible pregenerate a resulting image or simply the class is refactorized and functions are splitted.
My actual question might be: Is there a little more elegant, "better to maintain" way to do that. I haven't so much experience with unit testing so i'm not sure about how to do this.
EDIT: sorry for the long code, i simplified and translated it. Excuse me if i did a bad translation. english it's not my main language.
public class Renderer extends SwingWorker<BufferedImage[], Integer>
{
private Device device;
private Main main;
private Controller controller;
public static final int FPS = 25;
public Renderer(Device device, Main main, Controller controller)
{
this.device = device;
this.main = main;
this.controller = controller;
}
#Override
protected BufferedImage[] doInBackground() throws Exception
{
// rendering image array
BufferedImage[] output = renderize(main.getActualEscene());
LOG.log.log(Level.INFO, "Rendering");
// getting how many times should repeat
String stringRepeat =
main.getActualEscene().getProperties().get(TextProperties.REPEAT.toString());
int repeat = (stringRepeat == null) ? 1 : Integer.parseInt(stringRepeat);
// getting text speed
String stringFps =
main.getActualEscene().getProperties().get(TextProperties.TEXT_SPEED.toString());
int fps = (stringFps == null) ? FPS : Integer.parseInt(stringFps);
if (!this.isCancelled()) // if this task is not cancelled
{
// we create a pre-viewer
controller.setPreviewer(
new Previewer(controller, repeat, fps, main.getActualEscene()));
SwingUtilities.invokeLater(new Runnable()
{// sincronizing this code with AWT thread
#Override
public void run()
{ // if it's not cancelled
if (controller.getPreviewer() != null
&& !controller.getPreviewer().isCancelled())
{
controller.getPreviewer().execute(); //execute the task
}
}
});
}
return output;
}
/**
* Renders a scene and transforms it in a image array, then save it to the scene
*
* #param scene -> scene
* #return an array with the scene frames
*/
public BufferedImage[] renderize(Scene scene)
{
BufferedImage[] output = null; // end result
BufferedImage[] base = new BufferedImage[1]; // base image
BufferedImage[] animationImages = null; // animation layer
BufferedImage[] textLayer = null; // text layer
BufferedImage[] overLayer = null; // overlayer
// omitted long process to retrieve image properties
// once the text properties are retrieved if it's not null it will be rendered
if (text != null)
{
String backgroundColorString =
scene.getProperties().get(TextProperties.BACKGROUND_COLOR.toString());
int backGroundcolorInt = Integer.parseInt(backgroundColorString);
if (animationImages != null) // if there is an animation layer we create a same size text layer
{
textLayer = new BufferedImage[animationImages.length];
} else
{
textLayer = new BufferedImage[1];
}
for (int i = 0; i < textLayer.length; i++)
{
textLayer[i] = new BufferedImage(device.getWidth(), device.getHeight(),
BufferedImage.TYPE_INT_ARGB);
}
String font = scene.getProperties().get(TextProperties.FONT.toString());
int lettersHeigth = device.getHeight() / 3;
int colorNumber =
Integer.parseInt(scene.getProperties().get(TextProperties.TEXT_COLOR.toString()));
textLayer = addTextToAImage(text, font, lettersHeigth, new Color(colorNumber),
new Color(backGroundcolorInt), textLayer);
}
// has it overlayer?
if (scene.getProperties().containsKey(AnimationProperties.OVER.toString())) // if it has over
{
String over = scene.getProperties().get(AnimationProperties.OVER.toString());
overLayer =
this.readingImagesFromAnimation(OverLibrary.getInstance().getOverByName(over));
}
// mixing layers
output = base; // adding base base
if (animationImages != null) // if there is animation layer we add it
{
output = mixingTwoImages(output, animationImages);
}
if (textLayer != null) // if there is text layer
{
output = mixingTwoImages(output, textLayer);
}
if (overLayer != null) //if there is overlayer
{
output = mixingTwoImages(output, overLayer);
}
// delimiting image operative zone.
output = delimite(output, device);
main.getActualEscene().setPreview(new Preview(output));
LOG.log.log(Level.INFO, "Rendering scene finished: " + scene.getNombre());
return output;
}
/**
* Apply the device mask to an image. Delimiting the scene operative zone.
*
* #param input -> BufferedImage array which need to be delimited
* #param device -> device which it mask need to be applied
* #return An BufferedImage array with a delimited image for the device.
*/
private BufferedImage[] delimite(BufferedImage[] input, Device device)
{
BufferedImage[] output = new BufferedImage[input.length];
for (int i = 0; i < output.length; i++) // copy all original frames
{
output[i] = new BufferedImage(device.getWidth(), device.getHeight(),
BufferedImage.TYPE_INT_ARGB);
Graphics2D graphics = output[i].createGraphics();
graphics.drawImage(input[i], 0, 0, null);
}
for (int i = 0; i < output.length; i++) // for each frame
{
for (int j = 0; j < output[i].getHeight(); j++) // for each row
{
for (int k = 0; k < output[i].getWidth(); k++) // for each column
{
if (!device.estaEnZonaOperativa(j, k)) // if the coordinate is not in the operative zone
{
int pixel = 0x00000000; // we create a transparent pixel
output[i].setRGB(j, k, pixel); // set the original pixel with this new one
}
}
}
}
return output;
}
/**
* method that reads an animation images and returns it into an array
*
* #param animation -> animación de la que obtendremos sus image
* #return array de BufferedImage con los frames de la animación ordenados por orden de lectura
* ascendente.
*/
private BufferedImage[] readingImagesFromAnimation(Animation animation)
{
BufferedImage[] output = new BufferedImage[animation.getData().length];
for (int i = 0; i < animation.getData().length; i++) // for each frame
{
try
{
File ruta = animation.getData()[i]; // getting its path
output[i] = ImageIO.read(ruta); // reading from the path and save it into output
} catch (IOException e)
{
LOG.log.log(Level.SEVERE, "error reading animation file: " + animation.getData()[i],
e);
}
}
return output;
}
/**
* create a text layer an add it to the image parameter center
*
* #param text -> text to include
* #param font -> font of the text
* #param size -> size
* #param color -> text color
* #param image -> image you want to add text
* #return a text layer that need to be mixed with original one.
*/
private BufferedImage[] addTextToAImage(String text, String font, int size, Color color,
Color backgroundColor, BufferedImage[] image)
{
// we set the size of the output as the same of the original image
BufferedImage[] output = new BufferedImage[image.length];
for (int i = 0; i < image.length; i++) // for each frame
{
output[i] = new BufferedImage(image[i].getWidth(), image[i].getHeight(),
BufferedImage.TYPE_INT_ARGB); // we create a single new image
Graphics2D graphics = output[i].createGraphics(); // get its graphics
// calculate for metrics
Font font = new Font(font, Font.PLAIN, size); // create a font
graphics.setFont(font); // setting it into the graphics
FontMetrics fontMetrics = graphics.getFontMetrics(font); // generating metric object
// calculate the text coordinates
int x = (image[i].getWidth() - fontMetrics.stringWidth(text)) / 2;
int y = ((image[i].getHeight() - fontMetrics.getAscent() - fontMetrics.getDescent()) / 2)
+ fontMetrics.getAscent();
int width = fontMetrics.stringWidth(text); // usefull metric to set a brackground
int height = fontMetrics.getAscent() + fontMetrics.getDescent();
// setting a background
graphics.setColor(backgroundColor);
graphics.fill(new Rectangle(x, y - fontMetrics.getAscent(), width, height));
// drawing text
graphics.setColor(color); // setting a color for text
graphics.drawString(text, x, y);
}
return output;
}
/**
* method to join two bufferedImage arrays and mix them. The shortest array will be played on loop
*
* WARNIN: the images needs to has the same resolution
*
* #param image -> image mixed as background
* #param image2 -> image mixed as foreground
* #return a mixed image array.
*/
private BufferedImage[] mixingTwoImages(BufferedImage[] image, BufferedImage[] image2)
{
// checking no empty images.
if (image.length == 0 || image2.length == 0)
{
RuntimeException exception =
new RuntimeException("empty images");
LOG.log.log(Level.SEVERE, exception.getMessage(), exception);
throw exception;
}
int width = Math.max(image[0].getWidth(), image2[0].getWidth());
int height = Math.max(image[0].getHeight(), image2[0].getHeight());
// creating a sequence of the longest size of them
BufferedImage[] output = new BufferedImage[Math.max(image.length, image2.length)];
for (int i = 0; i < output.length; i++) // for each frame
{
// we create a frame
output[i] = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB);
Graphics g = output[i].getGraphics();
// mixing the frame
if (image.length > image2.length) // if the first array is longer
{
g.drawImage(image[i], 0, 0, null);
g.drawImage(image2[i % image2.length], 0, 0, null);
} else if (image.length < image2.length) // if the second array is longer
{
g.drawImage(image[i % image.length], 0, 0, null);
g.drawImage(image2[i], 0, 0, null);
} else // bot of the same size
{
g.drawImage(image[i], 0, 0, null);
g.drawImage(image2[i], 0, 0, null);
}
}
return output;
}
I am using javax.sound to make sounds, however when you play it they have some sort of noise in background, which even overcomes the sound if you play few notes at once. Here is the code:
public final static double notes[] = new double[] {130.81, 138.59, 146.83, 155.56, 164.81, 174.61, 185,
196, 207.65, 220, 233.08, 246.94, 261.63, 277.18, 293.66,
311.13, 329.63, 349.23, 369.99, 392, 415.3, 440, 466.16,
493.88, 523.25, 554.37};
public static void playSound(int note, int type) throws LineUnavailableException { //type 0 = sin, type 1 = square
Thread t = new Thread() {
public void run() {
try {
int sound = (int) (notes[note] * 100);
byte[] buf = new byte[1];
AudioFormat af = new AudioFormat((float) sound, 8, 1, true,
false);
SourceDataLine sdl;
sdl = AudioSystem.getSourceDataLine(af);
sdl = AudioSystem.getSourceDataLine(af);
sdl.open(af);
sdl.start();
int maxi = (int) (1000 * (float) sound / 1000);
for (int i = 0; i < maxi; i++) {
double angle = i / ((float) 44100 / 440) * 2.0
* Math.PI;
double val = 0;
if (type == 0) val = Math.sin(angle)*100;
if (type == 1) val = square(angle)*50;
buf[0] = (byte) (val * (maxi - i) / maxi);
sdl.write(buf, 0, 1);
}
sdl.drain();
sdl.stop();
sdl.close();
} catch (LineUnavailableException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
};
};
t.start();
}
public static double square (double angle){
angle = angle % (Math.PI*2);
if (angle > Math.PI) return 1;
else return 0;
}
This code is from here: https://stackoverflow.com/a/1932537/3787777
In this answer I will refer to 1) your code, 2) better approach (IMHO:) and 3) playing of two notes in the same time.
Your code
First, the sample rate should not depend on note frequency. Therefore try:
AudioFormat(44100,...
Next, use 16 bit sampling (sounds better!). Here is your code that plays simple tone without noise - but I would use it bit differently (see later). Please look for the comments:
Thread t = new Thread() {
public void run() {
try {
int sound = (440 * 100); // play A
AudioFormat af = new AudioFormat(44100, 16, 1, true, false);
SourceDataLine sdl;
sdl = AudioSystem.getSourceDataLine(af);
sdl.open(af, 4096 * 2);
sdl.start();
int maxi = (int) (1000 * (float) sound / 1000); // should not depend on notes frequency!
byte[] buf = new byte[maxi * 2]; // try to find better len!
int i = 0;
while (i < maxi * 2) {
// formula is changed to be simple sine!!
double val = Math.sin(Math.PI * i * 440 / 44100);
short s = (short) (Short.MAX_VALUE * val);
buf[i++] = (byte) s;
buf[i++] = (byte) (s >> 8); // little endian
}
sdl.write(buf, 0, maxi);
sdl.drain();
sdl.stop();
sdl.close();
} catch (LineUnavailableException e) {
e.printStackTrace();
}
}
};
t.start();
Proposal for better code
Here is a simplified version of your code that plays some note (frequency) without noise. I like it better as we first create array of doubles, which are universal values. These values can be combined together, or stored or further modified. Then we convert them to (8bit or 16bit) samples values.
private static byte[] buffer = new byte[4096 * 2 / 3];
private static int bufferSize = 0;
// plays a sample in range (-1, +1).
public static void play(SourceDataLine line, double in) {
if (in < -1.0) in = -1.0; // just sanity checks
if (in > +1.0) in = +1.0;
// convert to bytes - need 2 bytes for 16 bit sample
short s = (short) (Short.MAX_VALUE * in);
buffer[bufferSize++] = (byte) s;
buffer[bufferSize++] = (byte) (s >> 8); // little Endian
// send to line when buffer is full
if (bufferSize >= buffer.length) {
line.write(buffer, 0, buffer.length);
bufferSize = 0;
}
// todo: be sure that whole buffer is sent to line!
}
// prepares array of doubles, not related with the sampling value!
private static double[] tone(double hz, double duration) {
double amplitude = 1.0;
int N = (int) (44100 * duration);
double[] a = new double[N + 1];
for (int i = 0; i <= N; i++) {
a[i] = amplitude * Math.sin(2 * Math.PI * i * hz / 44100);
}
return a;
}
// finally:
public static void main(String[] args) throws LineUnavailableException {
AudioFormat af = new AudioFormat(44100, 16, 1, true, false);
SourceDataLine sdl = AudioSystem.getSourceDataLine(af);
sdl.open(af, 4096 * 2);
sdl.start();
double[] tones = tone(440, 2.0); // play A for 2 seconds
for (double t : tones) {
play(sdl, t);
}
sdl.drain();
sdl.stop();
sdl.close();
}
Sounds nice ;)
Play two notes in the same time
Just combine two notes:
double[] a = tone(440, 1.0); // note A
double[] b = tone(523.25, 1.0); // note C (i hope:)
for (int i = 0; i < a.length; i++) {
a[i] = (a[i] + b[i]) / 2;
}
for (double t : a) {
play(sdl, t);
}
Remember that with double array you can combine and manipulate your tones - i.e. to make composition of tone sounds that are being played in the same time. Of course, if you add 3 tones, you need to normalize the value by dividing with 3 and so on.
Ding Dong :)
The answer has already been provided, but I want to provide some information that might help understanding the solution.
Why 44100?
44.1 kHz audio is widely used, due to this being the sampling rate used in CDs. Analog audio is recorded by sampling it 44,100 times per second (1 cycle per second = 1 Hz), and then these samples are used to reconstruct the audio signal when playing it back. The reason behind the selection of this frequency is rather complex; and unimportant for this explanation. That said, the suggestion of using 22000 is not very good because that frequency is too close to the human hearing range (20Hz - 20kHz). You would want to use a sampling rate higher than 40kHz for good sound quality. I think mp4 uses 96kHz.
Why 16-bit?
The standard used for CDs is 44.1kHz/16-bit. MP4 uses 96kHz/24-bit. The sample rate refers to how many X-bit samples are recorded every second. CD-quality sampling uses 44,100 16-bit samples to reproduce sound.
Why is this explanation important?
The thing to remember is that you are trying to produce digital sound (not analog). This means that these bits and bytes have to be processed by an audio CODEC. In hardware, an audio CODEC is a device that encodes analog audio as digital signals and decodes digital back into analog. For audio outputs, the digitized sound must go through a Digital-to-Analog Converter (DAC) in order for proper sound to come out of the speakers. Two of the most important characteristics of a DAC are its bandwidth and its signal-to-noise ratio and the actual bandwidth of a DAC is characterized primarily by its sampling rate.
Basically, you can't use an arbitrary sampling rate because the audio will not be reproduced well by your audio device for the reasons stated above. When in doubt, check your computer hardware and find out what your CODEC supports.
I wrote Java code that creates a PDF with the iText library, the problem is that I can't use iText to print this PDF, so I was googling and I found a Java PDF library called PDFrenderer.
The question is how I can use PDFrenderer library to write a program that helps me printing my PDF file? Assuming that the pdfwriter code is created using iText. I am developing an application (Desktop) where customers can generate PDFs and send them directly to the printer.
Any help is appreciated
Mouad
this might help: How to Integrate with the Desktop Class
The Java Desktop API allows such tasks as emailing and printing being delegated to the operating system.
http://java.sun.com/developer/technicalArticles/J2SE/Desktop/javase6/desktop_api/
I use this to print PDFs...
public class PDFPrinter {
public PDFPrinter(File file) {
try {
FileInputStream fis = new FileInputStream(file);
FileChannel fc = fis.getChannel();
ByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size());
fis.close();
fc.close();
PDFFile pdfFile = new PDFFile(bb); // Create PDF Print Page
PDFPrintPage pages = new PDFPrintPage(pdfFile);
// Create Print Job
PrinterJob pjob = PrinterJob.getPrinterJob();
PageFormat pf = PrinterJob.getPrinterJob().defaultPage();
Paper a4paper = new Paper();
double paperWidth = 8.26;
double paperHeight = 11.69;
a4paper.setSize(paperWidth * 72.0, paperHeight * 72.0);
/*
* set the margins respectively the imageable area
*/
double leftMargin = 0.3;
double rightMargin = 0.3;
double topMargin = 0.5;
double bottomMargin = 0.5;
a4paper.setImageableArea(leftMargin * 72.0, topMargin * 72.0,
(paperWidth - leftMargin - rightMargin) * 72.0,
(paperHeight - topMargin - bottomMargin) * 72.0);
pf.setPaper(a4paper);
pjob.setJobName(file.getName());
Book book = new Book();
book.append(pages, pf, pdfFile.getNumPages());
pjob.setPageable(book);
// Send print job to default printer
if (pjob.printDialog()) {
pjob.print();
}
} catch (IOException e) {
e.printStackTrace();
} catch (PrinterException e) {
JOptionPane.showMessageDialog(null, "Printing Error: "
+ e.getMessage(), "Print Aborted",
JOptionPane.ERROR_MESSAGE);
e.printStackTrace();
}
}
class PDFPrintPage implements Printable {
private PDFFile file;
PDFPrintPage(PDFFile file) {
this.file = file;
}
public int print(Graphics g, PageFormat format, int index)
throws PrinterException {
int pagenum = index + 1;
// don't bother if the page number is out of range.
if ((pagenum >= 1) && (pagenum <= file.getNumPages())) {
// fit the PDFPage into the printing area
Graphics2D g2 = (Graphics2D) g;
PDFPage page = file.getPage(pagenum);
double pwidth = format.getImageableWidth();
double pheight = format.getImageableHeight();
double aspect = page.getAspectRatio();
double paperaspect = pwidth / pheight;
Rectangle imgbounds;
if (aspect > paperaspect) {
// paper is too tall / pdfpage is too wide
int height = (int) (pwidth / aspect);
imgbounds = new Rectangle(
(int) format.getImageableX(),
(int) (format.getImageableY() + ((pheight - height) / 2)),
(int) pwidth, height);
} else {
// paper is too wide / pdfpage is too tall
int width = (int) (pheight * aspect);
imgbounds = new Rectangle(
(int) (format.getImageableX() + ((pwidth - width) / 2)),
(int) format.getImageableY(), width, (int) pheight);
}
// render the page
PDFRenderer pgs = new PDFRenderer(page, g2, imgbounds, null,
null);
try {
page.waitForFinish();
pgs.run();
} catch (InterruptedException ie) {
}
return PAGE_EXISTS;
} else {
return NO_SUCH_PAGE;
}
}
}
}
I call it with:
new PDFPrinter(file);
P.S.: You need PDFRender.jar
I'm fairly new to android programming, but I am a quick learner.
So I found an intresting piece of code here: http://code.google.com/p/camdroiduni/source/browse/trunk/code/eclipse_workspace/camdroid/src/de/aes/camdroid/CameraView.java
And it's about live streaming from your device's camera to your browser.
But I want to know how the code actually works.
These are the things I want to understand:
1) How do they stream to the webbrowser. I understand that they send a index.html file to the ip adress of the device (on wifi) and that file reloads the page every second. But how do they send the index.html file to the desired ip address with sockets?
2) http://code.google.com/p/camdroiduni/wiki/Status#save_pictures_frequently , Here they mention they are using video, but I am still convinced they take pictures and send them as I don't see the mediarecorder anywhere.
Now my question is how they keep sending AND saving those images into the SD folder (i think). I think it's done with this code, but how does it works. Like with c.takepicture, it takes long to save and start previewing again, so that's no option to livestream.
public synchronized byte[] getPicture() {
try {
while (!isPreviewOn) wait();
isDecoding = true;
mCamera.setOneShotPreviewCallback(this);
while (isDecoding) wait();
} catch (Exception e) {
return null;
}
return mCurrentFrame;
}
private LayoutParams calcResolution (int origWidth, int origHeight, int aimWidth, int aimHeight) {
double origRatio = (double)origWidth/(double)origHeight;
double aimRatio = (double)aimWidth/(double)aimHeight;
if (aimRatio>origRatio)
return new LayoutParams(origWidth,(int)(origWidth/aimRatio));
else
return new LayoutParams((int)(origHeight*aimRatio),origHeight);
}
private void raw2jpg(int[] rgb, byte[] raw, int width, int height) {
final int frameSize = width * height;
for (int j = 0, yp = 0; j < height; j++) {
int uvp = frameSize + (j >> 1) * width, u = 0, v = 0;
for (int i = 0; i < width; i++, yp++) {
int y=0;
if(yp < raw.length) {
y = (0xff & ((int) raw[yp])) - 16;
}
if (y < 0) y = 0;
if ((i & 1) == 0) {
if(uvp<raw.length) {
v = (0xff & raw[uvp++]) - 128;
u = (0xff & raw[uvp++]) - 128;
}
}
int y1192 = 1192 * y;
int r = (y1192 + 1634 * v);
int g = (y1192 - 833 * v - 400 * u);
int b = (y1192 + 2066 * u);
if (r < 0) r = 0; else if (r > 262143) r = 262143;
if (g < 0) g = 0; else if (g > 262143) g = 262143;
if (b < 0) b = 0; else if (b > 262143) b = 262143;
rgb[yp] = 0xff000000 | ((r << 6) &
0xff0000) | ((g >> 2) &
0xff00) | ((b >> 10) &
0xff);
}
}
}
#Override
public synchronized void onPreviewFrame(byte[] data, Camera camera) {
int width = mSettings.PictureW() ;
int height = mSettings.PictureH();
// API 8 and above
// YuvImage yuvi = new YuvImage(data, ImageFormat.NV21 , width, height, null);
// Rect rect = new Rect(0,0,yuvi.getWidth() ,yuvi.getHeight() );
// OutputStream out = new ByteArrayOutputStream();
// yuvi.compressToJpeg(rect, 10, out);
// byte[] ref = ((ByteArrayOutputStream)out).toByteArray();
// API 7
int[] temp = new int[width*height];
OutputStream out = new ByteArrayOutputStream();
// byte[] ref = null;
Bitmap bm = null;
raw2jpg(temp, data, width, height);
bm = Bitmap.createBitmap(temp, width, height, Bitmap.Config.RGB_565);
bm.compress(CompressFormat.JPEG, mSettings.PictureQ(), out);
/*ref*/mCurrentFrame = ((ByteArrayOutputStream)out).toByteArray();
// mCurrentFrame = new byte[ref.length];
// System.arraycopy(ref, 0, mCurrentFrame, 0, ref.length);
isDecoding = false;
notify();
}
I really hope someone can explain these things as good as possible. That would really much be appreciated.
Ok, If anyone is intrested, I have the answer.
The code repeatedly takes a snapshot from the camera preview using setOneShotPreviewCallback() to call onPreviewFrame(). The frame is delivered in YUV format so raw2jpg() converts it into 32 bit ARGB for the jpeg encoder. NV21 is a YUV planar format as described here.
getPicture() is called, presumably by the application, and produces the jpeg data for the image in the private byte array mCurrentFrame and returns that array. What happens to if afterwards is not in that code fragment. Note that getPicture() does a couple of wait()s. This is because the image acquisition code is running in a separate thread to that of the application.
In the Main activity, the public static byte CurrentJPEG get this: cameraFrame.getPicture(); in public void run(). In this webservice it is send with a socket to the desired ip.
Correct me if I'm wrong.
Now I just still wonder how the image is displayed in the browser as a picture because you send byte data to it right? Please check this out: http://code.google.com/p/camdroiduni/source/browse/trunk/code/eclipse_workspace/camdroid/src/de/aes/camdroid/WebServer.java
Nothing in that code is sending any data to any URL. The getPicture method is returning a byte array, probably being used as an outputstream in some other method/Class that is then funneling it to a web service through some sort of protocol (UDP likely).