The problem I am doing requires me to send requests to a website and check whether a specific password is correct. It is somewhat similar to a CTF problem, but I use brute force to generate the correct password key, as the site gives feedback whether a specific key is on the right track. In order for a key to be considered "almost-valid," it must be a substring of the correct key.
I have implemented this naively, but the intended solution uses simple parallelism to speed up the process. How would I accomplish this in Java?
import java.net.*;
import java.io.*;
import java.nio.charset.StandardCharsets;
public class Main {
static boolean check(StringBuilder s) throws IOException{
String test = "https://example.com?pass=";
String q = URLEncoder.encode(s.toString(), StandardCharsets.UTF_8);
URL site = new URL(test+q);
URLConnection yc = site.openConnection();
BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
String inputLine;
while ((inputLine = in.readLine()) != null) {
//System.out.println(inputLine);
if (inputLine.contains("Incorrect password!")) {
return false;
}
if (inputLine.contains("the correct password!")) {
System.out.println(s);
System.exit(0);
}
}
return true;
}
static void gen(StringBuilder s) throws IOException {
if (!check(s)) {
return;
}
for (int i = 33; i<127; i++) {
int len = s.length();
gen(s.append((char) i));
s.setLength(len);
}
}
public static void main(String[] args) throws IOException, InterruptedException {
gen(new StringBuilder("start"));
}
}
EDIT: I have attempted to implement RecursiveAction & ForkJoinPool, but the code seems just as slow as the naive implementation. Am I implementing the parallelism incorrectly?
import java.nio.charset.StandardCharsets;
import java.util.concurrent.*;
import java.util.*;
import java.io.*;
import java.net.*;
public class cracked4 {
static class Validator extends RecursiveAction{
public String password;
public Validator(String p) {
password = p;
}
#Override
protected void compute(){
try {
if (!valid(password)) return;
System.out.println(password);
ArrayList<Validator> futures = new ArrayList<>();
for (int i = 33; i<127; i++) {
futures.add(new Validator(password + (char) i));
}
for (Validator t: futures) {
ForkJoinTask.invokeAll(t);
}
} catch (IOException e) {
e.printStackTrace();
}
}
public boolean valid(String s) throws IOException {
String test = "https://example.com?pass=" + URLEncoder.encode(s, StandardCharsets.UTF_8);
URL site = new URL(test);
URLConnection yc = site.openConnection();
BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
String inputLine;
while ((inputLine = in.readLine()) != null) {
if (inputLine.contains("Incorrect password!")) {
return false;
}
if (inputLine.contains("the correct password!")) {
System.out.println(s);
System.exit(0);
}
}
return true;
}
}
public static void main(String[] args) {
ForkJoinPool forkJoinPool = new ForkJoinPool();
forkJoinPool.invoke(new Validator("cararra"));
}
}
Furthermore, is there a certain UID serial I need? I researched about it, but I could not find a specific answer.
Alright, I researched more about parallelism, and I decided on using ForkJoin / RecursiveAction. Using the parallelism allowed me to reduce my code execution time from 200 seconds to roughly 43 seconds (on my computer).
import java.util.*;
import java.util.concurrent.*;
import java.net.http.*;
import java.net.*;
import java.nio.charset.StandardCharsets;
import java.io.*;
public class cracked4 {
static class Validator extends RecursiveAction {
public StringBuilder password;
public Validator(StringBuilder p) {
password = p;
}
public static boolean valid(StringBuilder s) throws IOException, InterruptedException {
HttpClient client = HttpClient.newHttpClient();
String test = "https://example.com?pass=" + URLEncoder.encode(s.toString(), StandardCharsets.UTF_8);
HttpRequest request = HttpRequest.newBuilder().uri(URI.create(test)).GET().build();
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
String text = response.body();
if (text.contains("Incorrect password!")) {
return false;
}
if (text.contains("the correct password!")) {
System.out.println(s);
System.exit(0);
}
return true;
}
#Override
protected void compute() {
try {
if (!valid(password)) return;
ArrayList<Validator> c = new ArrayList<>();
for (int i = 33; i < 127; i++) {
StringBuilder t = new StringBuilder(password).append((char) i);
c.add(new Validator(t));
}
ForkJoinTask.invokeAll(c);
}
catch (IOException | InterruptedException ignored) {
}
}
}
public static void main(String[] args) {
ForkJoinPool forkJoinPool = new ForkJoinPool();
forkJoinPool.invoke(new Validator(new StringBuilder("start")));
}
}
When I wrote this code initially, I used .fork(), but this did not help at all. In fact, it made it perform just as slow as the sequential code. I collected all of the Validator objects and I used ForkJoinTask.invokeAll(). This small difference resulted in an almost 200% speedup.
Hi please use RecursiveTask,ForkJoinPool for parallelism. Below code is not final modify on your own way.
public class Test {
public static void main(String[] args) {
ForkJoinPool forkJoinPool=new ForkJoinPool(parallelism);
forkJoinPool.invoke(new PasswordKeyValidatorTask(new StringBuilder("start")));
}
public static class PasswordKeyValidatorTask extends RecursiveTask<StringBuilder> {
private static final long serialVersionUID = 3113310524830439302L;
private StringBuilder password;
public PasswordKeyValidatorTask(StringBuilder password) {
this.password = password;
}
#Override
protected StringBuilder compute() {
try {
if (!valid(password)) {
List<PasswordKeyValidatorTask> subtasks = new ArrayList<PasswordKeyValidatorTask>();
for (int i = 33; i < 127; i++) {
PasswordKeyValidatorTask task = new PasswordKeyValidatorTask(password.append((char) i));
subtasks.add(task);
}
subtasks.stream().forEach(t -> t.fork());
return subtasks.stream().map(t -> t.join()).findFirst().orElse(null);
}
} catch (Exception e) {
}
return password;
}
boolean valid(StringBuilder s) throws IOException {
String test = "https://samplewebsite.com?pass=";
String q = URLEncoder.encode(s.toString(), StandardCharsets.UTF_8);
URL site = new URL(test + s);
URLConnection yc = site.openConnection();
BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
String inputLine;
while ((inputLine = in.readLine()) != null) {
// System.out.println(inputLine);
if (inputLine.contains("Incorrect password!")) {
return false;
}
if (inputLine.contains("the correct password!")) {
System.out.println(s);
}
}
return true;
}
}
}
Related
This is a follow-on question to AWS Transcribe S3 .wav file to text. I use a stream to read and send a .wav file contents to AWS.
Instead of getting back the correct transcript, I get nonsense like a bunch of "Yeah." statements. It looks like AWS isn't able to interpret the byte stream correctly, but I'm not sure what's wrong. I'm wondering if the file needs to be encoded somehow, ie, I can't send the raw .wav bytes straight from the file? Or perhaps I need to tell the service that this is .wav format?
What's wrong here? The input file is a valid .wav voice file that sounds intelligible when I listen to it.
Here is my java code:
package com.amazonaws.transcribe;
import org.reactivestreams.Publisher;
import org.reactivestreams.Subscriber;
import org.reactivestreams.Subscription;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.transcribestreaming.TranscribeStreamingAsyncClient;
import software.amazon.awssdk.services.transcribestreaming.model.*;
import javax.sound.sampled.*;
import java.io.*;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
public class TranscribeFileFromStream {
private static final Region REGION = Region.US_EAST_1;
private static TranscribeStreamingAsyncClient client;
public static void main(String args[]) throws URISyntaxException, ExecutionException, InterruptedException, LineUnavailableException {
System.out.println(System.getProperty("java.version"));
client = TranscribeStreamingAsyncClient.builder()
.region(REGION)
.build();
try {
CompletableFuture<Void> result = client.startStreamTranscription(getRequest(16000),
new AudioStreamPublisher(getStreamFromFile()),
getResponseHandler());
result.get();
} finally {
if (client != null) {
client.close();
}
}
}
private static InputStream getStreamFromFile() {
try {
File inputFile = new File("~/work/transcribe/src/main/resources/story/media/Story3.m4a.wav");
InputStream audioStream = new FileInputStream(inputFile);
return audioStream;
} catch (FileNotFoundException e) {
throw new RuntimeException(e);
}
}
private static StartStreamTranscriptionRequest getRequest(Integer mediaSampleRateHertz) {
return StartStreamTranscriptionRequest.builder()
.languageCode(LanguageCode.EN_US)
.mediaEncoding(MediaEncoding.PCM)
.mediaSampleRateHertz(mediaSampleRateHertz)
.build();
}
private static StartStreamTranscriptionResponseHandler getResponseHandler() {
return StartStreamTranscriptionResponseHandler.builder()
.onResponse(r -> {
System.out.println("Received Initial response");
})
.onError(e -> {
System.out.println(e.getMessage());
StringWriter sw = new StringWriter();
e.printStackTrace(new PrintWriter(sw));
System.out.println("Error Occurred: " + sw.toString());
})
.onComplete(() -> {
System.out.println("=== All records stream successfully ===");
})
.subscriber(event -> {
List<Result> results = ((TranscriptEvent) event).transcript().results();
if (results.size() > 0) {
if (!results.get(0).alternatives().get(0).transcript().isEmpty()) {
System.out.println(results.get(0).alternatives().get(0).transcript());
} else {
System.out.println("Empty result");
}
} else {
System.out.println("No results");
}
})
.build();
}
private static class AudioStreamPublisher implements Publisher<AudioStream> {
private final InputStream inputStream;
private static Subscription currentSubscription;
private AudioStreamPublisher(InputStream inputStream) {
this.inputStream = inputStream;
}
#Override
public void subscribe(Subscriber<? super AudioStream> s) {
if (this.currentSubscription == null) {
this.currentSubscription = new SubscriptionImpl(s, inputStream);
} else {
this.currentSubscription.cancel();
this.currentSubscription = new SubscriptionImpl(s, inputStream);
}
s.onSubscribe(currentSubscription);
}
}
public static class SubscriptionImpl implements Subscription {
private static final int CHUNK_SIZE_IN_BYTES = 1024 * 1;
private final Subscriber<? super AudioStream> subscriber;
private final InputStream inputStream;
private ExecutorService executor = Executors.newFixedThreadPool(1);
private AtomicLong demand = new AtomicLong(0);
SubscriptionImpl(Subscriber<? super AudioStream> s, InputStream inputStream) {
this.subscriber = s;
this.inputStream = inputStream;
}
#Override
public void request(long n) {
if (n <= 0) {
subscriber.onError(new IllegalArgumentException("Demand must be positive"));
}
demand.getAndAdd(n);
executor.submit(() -> {
try {
do {
ByteBuffer audioBuffer = getNextEvent();
if (audioBuffer.remaining() > 0) {
AudioEvent audioEvent = audioEventFromBuffer(audioBuffer);
subscriber.onNext(audioEvent);
} else {
subscriber.onComplete();
break;
}
} while (demand.decrementAndGet() > 0);
} catch (Exception e) {
subscriber.onError(e);
}
});
}
#Override
public void cancel() {
executor.shutdown();
}
private ByteBuffer getNextEvent() {
ByteBuffer audioBuffer = null;
byte[] audioBytes = new byte[CHUNK_SIZE_IN_BYTES];
int len = 0;
try {
len = inputStream.read(audioBytes);
if (len <= 0) {
audioBuffer = ByteBuffer.allocate(0);
} else {
audioBuffer = ByteBuffer.wrap(audioBytes, 0, len);
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}
return audioBuffer;
}
private AudioEvent audioEventFromBuffer(ByteBuffer bb) {
return AudioEvent.builder()
.audioChunk(SdkBytes.fromByteBuffer(bb))
.build();
}
}
}
Here's my program output:
Received Initial response
No results
No results
Yeah.
No results
Yeah.
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
Yeah.
No results
No results
Oh,
No results
Oh,
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
Oh,
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
No results
The audio file had a sample rate of 44.1 kHz. It was converted to 16 kHz, and it worked:
https://drive.google.com/file/d/1mYVbNlYK3SpGT4NbFRYGn86177eTCqhd/view?usp=sharing
As smac2020 pointed out, the sample rate was wrong. Debugging incorrect metadata values passed to AWS is tricky because there's no errors from AWS. You just get back an incorrect transcription. So, the lesson here is, make sure you know what the right values are. Some of them can be automatically detected.
If you're on mac, the tool mediainfo is quite useful.
brew install mediainfo
So is ffmpeg:
brew install ffmpeg
Here is an updated example where I automatically detect the sample rate using AudioFormat.java. Ideally, the AWS sdk would do this for you. If the media file is outside the parameters of what can be transcribed, then it would throw an exception. Note, I had to modify my original file to 16,000 sample rate using the tool: nch.com.au/switch/index.html. It would be great (hint, hint) if the SDK would also have the ability to modify sample rate, etc, so that files can be changed to fit within input parameters.
package com.amazonaws.transcribe;
import org.reactivestreams.Publisher;
import org.reactivestreams.Subscriber;
import org.reactivestreams.Subscription;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.transcribestreaming.TranscribeStreamingAsyncClient;
import software.amazon.awssdk.services.transcribestreaming.model.*;
import javax.sound.sampled.*;
import java.io.*;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
import static javax.sound.sampled.AudioFormat.Encoding.*;
public class TranscribeFileFromStream {
private static final Region REGION = Region.US_EAST_1;
private static TranscribeStreamingAsyncClient client;
public static void main(String args[]) throws Exception {
System.setProperty("AWS_ACCESS_KEY_ID", "myId");
System.setProperty("AWS_SECRET_ACCESS_KEY", "myKey");
System.out.println(System.getProperty("java.version"));
// BasicConfigurator.configure();
client = TranscribeStreamingAsyncClient.builder()
.region(REGION)
.build();
try {
File inputFile = new File("/home/me/work/transcribe/src/main/resources/test-file.wav");
CompletableFuture<Void> result = client.startStreamTranscription(
getRequest(inputFile),
new AudioStreamPublisher(getStreamFromFile(inputFile)),
getResponseHandler());
result.get();
} finally {
if (client != null) {
client.close();
}
}
}
private static InputStream getStreamFromFile(File inputFile) {
try {
return new FileInputStream(inputFile);
} catch (FileNotFoundException e) {
throw new RuntimeException(e);
}
}
private static StartStreamTranscriptionRequest getRequest(File inputFile) throws IOException, UnsupportedAudioFileException {
//TODO: I read the file twice in this example. Can this be more performant?
AudioInputStream audioInputStream = AudioSystem.getAudioInputStream(inputFile);
AudioFormat audioFormat = audioInputStream.getFormat();
return StartStreamTranscriptionRequest.builder()
.languageCode(LanguageCode.EN_US)
//.mediaEncoding(MediaEncoding.PCM)
.mediaEncoding(getAwsMediaEncoding(audioFormat))
.mediaSampleRateHertz(getAwsSampleRate(audioFormat))
.build();
}
private static MediaEncoding getAwsMediaEncoding(AudioFormat audioFormat) {
final String javaMediaEncoding = audioFormat.getEncoding().toString();
if (PCM_SIGNED.toString().equals(javaMediaEncoding)) {
return MediaEncoding.PCM;
} else if (PCM_UNSIGNED.toString().equals(javaMediaEncoding)){
return MediaEncoding.PCM;
} /*else if (ALAW.toString().equals(javaMediaEncoding)){
//WARNING: I have no idea how ALAW maps to AWS media encodings.
return MediaEncoding.OGG_OPUS;
} else if (ULAW.toString().equals(javaMediaEncoding)){
//WARNING: I have no idea how ULAW maps to AWS encodings.
return MediaEncoding.FLAC;
}*/
throw new IllegalArgumentException("Not a recognized media encoding:" + javaMediaEncoding);
}
private static Integer getAwsSampleRate(AudioFormat audioFormat) {
return Math.round(audioFormat.getSampleRate());
}
private static StartStreamTranscriptionResponseHandler getResponseHandler() {
return StartStreamTranscriptionResponseHandler.builder()
.onResponse(r -> {
System.out.println("Received Initial response");
})
.onError(e -> {
System.out.println(e.getMessage());
StringWriter sw = new StringWriter();
e.printStackTrace(new PrintWriter(sw));
System.out.println("Error Occurred: " + sw.toString());
})
.onComplete(() -> {
System.out.println("=== All records stream successfully ===");
})
.subscriber(event -> {
List<Result> results = ((TranscriptEvent) event).transcript().results();
if (results.size() > 0) {
if (!results.get(0).alternatives().get(0).transcript().isEmpty()) {
System.out.println(results.get(0).alternatives().get(0).transcript());
} else {
System.out.println("Empty result");
}
} else {
System.out.println("No results");
}
})
.build();
}
private static class AudioStreamPublisher implements Publisher<AudioStream> {
private final InputStream inputStream;
private static Subscription currentSubscription;
private AudioStreamPublisher(InputStream inputStream) {
this.inputStream = inputStream;
}
#Override
public void subscribe(Subscriber<? super AudioStream> s) {
if (this.currentSubscription == null) {
this.currentSubscription = new SubscriptionImpl(s, inputStream);
} else {
this.currentSubscription.cancel();
this.currentSubscription = new SubscriptionImpl(s, inputStream);
}
s.onSubscribe(currentSubscription);
}
}
public static class SubscriptionImpl implements Subscription {
private static final int CHUNK_SIZE_IN_BYTES = 1024 * 1;
private final Subscriber<? super AudioStream> subscriber;
private final InputStream inputStream;
private ExecutorService executor = Executors.newFixedThreadPool(1);
private AtomicLong demand = new AtomicLong(0);
SubscriptionImpl(Subscriber<? super AudioStream> s, InputStream inputStream) {
this.subscriber = s;
this.inputStream = inputStream;
}
#Override
public void request(long n) {
if (n <= 0) {
subscriber.onError(new IllegalArgumentException("Demand must be positive"));
}
demand.getAndAdd(n);
executor.submit(() -> {
try {
do {
ByteBuffer audioBuffer = getNextEvent();
if (audioBuffer.remaining() > 0) {
AudioEvent audioEvent = audioEventFromBuffer(audioBuffer);
subscriber.onNext(audioEvent);
} else {
subscriber.onComplete();
break;
}
} while (demand.decrementAndGet() > 0);
} catch (Exception e) {
subscriber.onError(e);
}
});
}
#Override
public void cancel() {
executor.shutdown();
}
private ByteBuffer getNextEvent() {
ByteBuffer audioBuffer = null;
byte[] audioBytes = new byte[CHUNK_SIZE_IN_BYTES];
int len = 0;
try {
len = inputStream.read(audioBytes);
if (len <= 0) {
audioBuffer = ByteBuffer.allocate(0);
} else {
audioBuffer = ByteBuffer.wrap(audioBytes, 0, len);
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}
return audioBuffer;
}
private AudioEvent audioEventFromBuffer(ByteBuffer bb) {
return AudioEvent.builder()
.audioChunk(SdkBytes.fromByteBuffer(bb))
.build();
}
}
}
I am having a problem with my javac program in cmd, I am running Java jdk1.8.0_152 and I am getting the following errors:
C:\[My_Path]>javac -encoding UTF-8 Main.java
Main.java:1: error: illegal character: '\ufeff'
?package tanks;
^
Main.java:1: error: class, interface, or enum expected;
package tanks;
^
2 errors
I checked the file using different software (text editors) and rewrote the document, and still got the error. I do not have a '?' in the front of the document, but javac for some reason detects it out of nowhere.
Full Document: Main.Java
package tanks;
import java.net.ServerSocket;
import java.net.Socket;
import java.util.Vector;
import tanks.DataBase.Numeric.NumericDB;
import tanks.DataBase.Turrent.TurrentDB;
import tanks.DataBase.User;
import tanks.DataBase.Users;
public class Main implements Runnable {
private static Main instance;
private ServerSocket ss;
private static double version = 0.1D;
public static Lobby systemUser;
public static Vector lobbys = new Vector();
private static boolean isDebug = false;
private static ControlWindow cw;
private static boolean started = false;
public static void main(string[] paramArrayOfString) throws Exception {
cw = new ControlWindow();
cw.setVisible(true);
}
private static void disposeLobbys() {
for (int i = 0; i < lobby.size(); i++) {
Bobby localLobby = (Lobby)lobbys.elementAt(i);
state = SocketProcessor.STATE_ERROR;
try {
is.close();
os.close();
s.close();
}
catch(Exception localException) {}
}
lobbys = new Vector();
}
public static void stopServer() {
if (!started) return;
GarageDataBaseParser.save();
started = false;
NumericsDB.dispose();
Battles.dispose();
disposeBobbys();
tanks.DataBase.Battle.BonusDB.dispose();
System.gc();
cw.showMessage("Server Started", "Server RU_1 Notification");
}
public static void startServer() {
try {
started = true;
NumericDB.init();
tanks.DataBase.Colormap.ColormapDB.init();
ServerSocket localServerSocket;
if (isDebug) {
localServerSocket = new ServerSocket(2845);
} else {
localServerSocket = new ServerSocket(5482);
}
new Thread(new Main(localServerSocket)).start();
cw.showMessage("Server Started!", "Server RU_1 Notification");
} catch (Exception localExeption) { cw.showMessage(localExption.getMessage(), "Server RU_1 Notification");
}
}
public Main(ServerSocket paramServerSocket) {
instance = this;
ss = paramServerSocket;
}
public void run() {
try {
while (started) {
Socket localSocket = ss.accept();
new Thread(new Starter(localSocket)).start();
}
}
catch (Exeption localExeption1) {
cw.showMessage(localExeption1.getMessage(), "Server RU_1 Notification");
}
try {
ss.close();
}
catch (Exception localExcption2) {
cw.showMessage(localException2.getMessage(), "Server RU_1 Notification");
}
ss = null;
}
private static void disposeUsers() {}
private static void initUsers() {
UserDataBaseParser.load("ServerDataBase/Users.txt");
systemUser = new Lobby(Users.getUser(0), null, null, null);
GarageDataBaseParser.load();
}
public static void removeLobby(int paramInt) {
Lobby localLobby = (Lobby)lobbys.elementAt(paramInt);
lobbys.removeElementAt(paramInt);
localLobby = null;
}
public static void removeLobby(Lobby paramLobby) {
lobbys.removeElement(paramLobby);
paramLobby = null;
}
public static void startLobby(BattleProcessor paramBattleProcessor, User paramUsers) {
Lobby localLobby = new Lobby(paramUser, s, is, os);
lobbys.addElement(localLobby);
new Thread(localLobby).start();
}
public static void startLobby(Starter paramStarter, User paramUser) {
cwlistModel.addElement(login);
Battles.check(paramUser);
Lobby localLobby = new Lobby(paramUser, s, is, os);
lobbys.addElement(localLobby);
new Thread(localLobby).start();
paramStarter = null;
}
}
I submitted some Runnables to an ExecutorService. Inside these Runnables, wait() and notify() are called. The code works with newFixedThreadPool as the ExecutorService. With newWorkStealingPool, the process exits unexpectedly without any error message.
import java.net.URL;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.IOException;
// For regular expressions
import java.util.regex.Matcher;
import java.util.regex.MatchResult;
import java.util.regex.Pattern;
import java.util.*;
import java.util.concurrent.*;
public class TestPipeline {
public static void main(String[] args) {
runAsThreads();
}
private static void runAsThreads() {
final BlockingQueue<String> urls = new OneItemQueue<String>();
final BlockingQueue<Webpage> pages = new OneItemQueue<Webpage>();
final BlockingQueue<Link> refPairs = new OneItemQueue<Link>();
final BlockingQueue<Link> uniqRefPairs = new OneItemQueue<Link>();
final ExecutorService executor = Executors.newWorkStealingPool(6);
// final ExecutorService executor = Executors.newFixedThreadPool(6);
executor.submit(new UrlProducer(urls));
executor.submit(new PageGetter(urls, pages));
executor.submit(new LinkScanner(pages,refPairs));
executor.submit(new Uniquifier<Link>(refPairs,uniqRefPairs));
executor.submit(new LinkPrinter(uniqRefPairs));
}
}
class UrlProducer implements Runnable {
private final BlockingQueue<String> output;
public UrlProducer(BlockingQueue<String> output) {
this.output = output;
}
public void run() {
System.out.println("in producer");
for (int i=0; i<urls.length; i++)
output.put(urls[i]);
}
private static final String[] urls =
{ "http://www.itu.dk", "http://www.di.ku.dk", "http://www.miele.de",
"http://www.microsoft.com", "http://www.cnn.com", "http://www.dr.dk",
"http://www.vg.no", "http://www.tv2.dk", "http://www.google.com",
"http://www.ing.dk", "http://www.dtu.dk", "http://www.bbc.co.uk"
};
}
class PageGetter implements Runnable {
private final BlockingQueue<String> input;
private final BlockingQueue<Webpage> output;
public PageGetter(BlockingQueue<String> input, BlockingQueue<Webpage> output) {
this.input = input;
this.output = output;
}
public void run() {
while (true) {
System.out.println("in pagegetter");
String url = input.take();
// System.out.println("PageGetter: " + url);
try {
String contents = getPage(url, 200);
output.put(new Webpage(url, contents));
} catch (IOException exn) { System.out.println(exn); }
}
}
public static String getPage(String url, int maxLines) throws IOException {
// This will close the streams after use (JLS 8 para 14.20.3):
try (BufferedReader in
= new BufferedReader(new InputStreamReader(new URL(url).openStream()))) {
StringBuilder sb = new StringBuilder();
for (int i=0; i<maxLines; i++) {
String inputLine = in.readLine();
if (inputLine == null)
break;
else
sb.append(inputLine).append("\n");
}
return sb.toString();
}
}
}
class Uniquifier<T> implements Runnable{
private final Set<T> set = new HashSet<T>();
private final BlockingQueue<T> input;
private final BlockingQueue<T> output;
public Uniquifier(BlockingQueue<T> input, BlockingQueue<T> output){
this.input = input;
this.output = output;
}
public void run(){
while(true){
System.out.println("in uniquifier");
T item = input.take();
if(!set.contains(item)){
set.add(item);
output.put(item);
}
}
}
}
class LinkScanner implements Runnable {
private final BlockingQueue<Webpage> input;
private final BlockingQueue<Link> output;
public LinkScanner(BlockingQueue<Webpage> input,
BlockingQueue<Link> output) {
this.input = input;
this.output = output;
}
private final static Pattern urlPattern
= Pattern.compile("a href=\"(\\p{Graph}*)\"");
public void run() {
while (true) {
System.out.println("in link scanner");
Webpage page = input.take();
// System.out.println("LinkScanner: " + page.url);
// Extract links from the page's <a href="..."> anchors
Matcher urlMatcher = urlPattern.matcher(page.contents);
while (urlMatcher.find()) {
String link = urlMatcher.group(1);
output.put(new Link(page.url, link));
}
}
}
}
class LinkPrinter implements Runnable {
private final BlockingQueue<Link> input;
public LinkPrinter(BlockingQueue<Link> input) {
this.input = input;
}
public void run() {
while (true) {
System.out.println("in link printer");
Link link = input.take();
// System.out.println("LinkPrinter: " + link.from);
System.out.printf("%s links to %s%n", link.from, link.to);
}
}
}
class Webpage {
public final String url, contents;
public Webpage(String url, String contents) {
this.url = url;
this.contents = contents;
}
}
class Link {
public final String from, to;
public Link(String from, String to) {
this.from = from;
this.to = to;
}
// Override hashCode and equals so can be used in HashSet<Link>
public int hashCode() {
return (from == null ? 0 : from.hashCode()) * 37
+ (to == null ? 0 : to.hashCode());
}
public boolean equals(Object obj) {
Link that = obj instanceof Link ? (Link)obj : null;
return that != null
&& (from == null ? that.from == null : from.equals(that.from))
&& (to == null ? that.to == null : to.equals(that.to));
}
}
// Different from java.util.concurrent.BlockingQueue: Allows null
// items, and methods do not throw InterruptedException.
interface BlockingQueue<T> {
void put(T item);
T take();
}
class OneItemQueue<T> implements BlockingQueue<T> {
private T item;
private boolean full = false;
public void put(T item) {
synchronized (this) {
while (full) {
try { this.wait(); }
catch (InterruptedException exn) { }
}
full = true;
this.item = item;
this.notifyAll();
}
}
public T take() {
synchronized (this) {
while (!full) {
try { this.wait(); }
catch (InterruptedException exn) { }
}
full = false;
this.notifyAll();
return item;
}
}
}
Because the Pool is allocating threads dynamically, there are no threads alive after runAsThreads exits because that's the end of the main thread. There needs to be at least on thread running to keep the application alive. Adding a call to awaitTermination is needed. It's not needed for the fixed pool because that will always have active threads until it is explicitly shut down as noted in the JavaDocs.
I need to get a String value from a Thread, but I don't get it! Global variables don't work! Can you help me please?
This is my code. I need to use the dataString:
public class Deserializable {
public void execute() {
new Thread() {
public void run() {
String surl = "http://myaddressxxxxx";
HttpURLConnection urlConnection = null;
URL url = null;
try {
url = new URL(surl);
urlConnection = (HttpURLConnection) url.openConnection();
InputStream in = new BufferedInputStream(
urlConnection.getInputStream());
int b = in.read();
List<Byte> bytes = new LinkedList<Byte>();
while (b != -1) {
bytes.add((byte) b);
b = in.read();
}
byte[] array = new byte[bytes.size()];
for (int i = 0; i < bytes.size(); i++) {
array[i] = bytes.get(i).byteValue();
}
// I need return this String.
String dataString = new String(array);
} catch (Exception e) {
e.printStackTrace();
} finally {
urlConnection.disconnect();
}
}
}.start();
}
Thread can be extended to have fields, like for your dataString. These fields can be accessed from the same thread or from a different one, as long they have references to each other.
public class ThreadA extends Thread {
public String dataString;
public void run(){
...
this.dataString = ...;
...
}
}
public class ThreadB extends Thread {
private final ThreadA ta;
public ThreadB(ThreadA ta){
super();
this.ta = ta;
}
public void run(){
...
do something with ta.dataString...
...
}
}
Of course, this poses the problem of concurrent access to the field dataString. Consider using synchronized, if this is an issue in your case. Have a look at this tutorial on concurrency for more information.
import java.io.IOException;
import org.hyperic.sigar.*;
public class SigarDemo {
public static void main(String args[]) throws SigarException, IOException
{
final Sigar sigar = new Sigar();
final long[] processes = sigar.getProcList();
ProcTime pt=new ProcTime();
for (final long processId : processes) {
ProcUtil.getDescription(sigar, processId);
pt=sigar.getProcTime(processId);
System.out.println("---"+pt.getStartTime());
}
}
I am trying to fetch process time of each process using sigar. I am getting this errors:
Exception in thread "main" java.lang.ExceptionInInitializerError
at taskmanager.SigarDemo.main(SigarDemo.java:22)
Caused by: java.security.AccessControlException: access denied ("java.util.PropertyPermission" "sigar.nativeLogging" "read")
at java.security.AccessControlContext.checkPermission(AccessControlContext.java:457)
at java.security.AccessController.checkPermission(AccessController.java:884)
at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
at java.lang.SecurityManager.checkPropertyAccess(SecurityManager.java:1294)
at java.lang.System.getProperty(System.java:714)
at org.hyperic.sigar.Sigar.(Sigar.java:78)
I have tried policy file setting all permission. but still I am getting errors. I am using netbeans 8.0 . and I had already setting
-Djava.security.manager -Djava.security.policy=src/dir1/dir2/important.policy
I used this code to get the process times
public static void main(String args[]) {
try {
final Sigar sigar = new Sigar();
final long[] processes = sigar.getProcList();
ProcTime pt = new ProcTime();
for (final long processId : processes) {
try {
ProcUtil.getDescription(sigar, processId);
pt = sigar.getProcTime(String.valueOf(processId));
System.out.println("---" + pt.getStartTime());
} catch (SigarException e) {
System.out.println("can't accessible...");
}
}
} catch (SigarException ex) {
ex.printStackTrace();
}
}
you don't want to specify the security policy file in VM arguments to get the process times. but the thing is getProcTime() will not return process times for some process ids because of SigarPermissionDeniedException.
but you will get process time for some processes without any problem.
I got this idea from a sample demo file from bindings\java\examples folder. I posted it below with slight modification. you can compile and run it to see the result(it includes the process time also)
import org.hyperic.sigar.Sigar;
import org.hyperic.sigar.SigarProxy;
import org.hyperic.sigar.SigarException;
import org.hyperic.sigar.ProcCredName;
import org.hyperic.sigar.ProcMem;
import org.hyperic.sigar.ProcTime;
import org.hyperic.sigar.ProcState;
import org.hyperic.sigar.ProcUtil;
import org.hyperic.sigar.cmd.Shell;
import org.hyperic.sigar.cmd.SigarCommandBase;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.text.SimpleDateFormat;
import java.util.Date;
/**
* Show process status.
*/
public class Ps extends SigarCommandBase {
public Ps(Shell shell) {
super(shell);
}
public Ps() {
super();
}
protected boolean validateArgs(String[] args) {
return true;
}
public String getSyntaxArgs() {
return "[pid|query]";
}
public String getUsageShort() {
return "Show process status";
}
public boolean isPidCompleter() {
return true;
}
public void output(String[] args) throws SigarException {
long[] pids;
if (args.length == 0) {
pids = this.proxy.getProcList();
}
else {
pids = this.shell.findPids(args);
}
for (int i=0; i<pids.length; i++) {
long pid = pids[i];
try {
output(pid);
} catch (SigarException e) {
this.err.println("Exception getting process info for " +
pid + ": " + e.getMessage());
}
}
}
public static String join(List info) {
StringBuffer buf = new StringBuffer();
Iterator i = info.iterator();
boolean hasNext = i.hasNext();
while (hasNext) {
buf.append((String)i.next());
hasNext = i.hasNext();
if (hasNext)
buf.append("\t");
}
return buf.toString();
}
public static List getInfo(SigarProxy sigar, long pid)
throws SigarException {
ProcState state = sigar.getProcState(pid);
ProcTime time = null;
String unknown = "???";
List info = new ArrayList();
info.add(String.valueOf(pid));
try {
ProcCredName cred = sigar.getProcCredName(pid);
info.add(cred.getUser());
} catch (SigarException e) {
info.add(unknown);
}
try {
time = sigar.getProcTime(pid);
info.add(getStartTime(time.getStartTime()));
System.out.println("this line has executed..!!!");
} catch (SigarException e) {
info.add(unknown);
}
try {
ProcMem mem = sigar.getProcMem(pid);
info.add(Sigar.formatSize(mem.getSize()));
info.add(Sigar.formatSize(mem.getRss()));
info.add(Sigar.formatSize(mem.getShare()));
} catch (SigarException e) {
info.add(unknown);
}
info.add(String.valueOf(state.getState()));
if (time != null) {
info.add(getCpuTime(time));
}
else {
info.add(unknown);
}
String name = ProcUtil.getDescription(sigar, pid);
info.add(name);
return info;
}
public void output(long pid) throws SigarException {
println(join(getInfo(this.proxy, pid)));
}
public static String getCpuTime(long total) {
long t = total / 1000;
return t/60 + ":" + t%60;
}
public static String getCpuTime(ProcTime time) {
return getCpuTime(time.getTotal());
}
private static String getStartTime(long time) {
if (time == 0) {
return "00:00";
}
long timeNow = System.currentTimeMillis();
String fmt = "MMMd";
if ((timeNow - time) < ((60*60*24) * 1000)) {
fmt = "HH:mm";
}
return new SimpleDateFormat(fmt).format(new Date(time));
}
public static void main(String[] args) throws Exception {
new Ps().processCommand(args);
}
}