We Keep Coding

SMPP java server connect to SMSC or SMPP Client can receive SMS via SMPP?

i have 3rd party smpp connection.
so i created java smpp client and its working fine.
how i can create smpp server connect to 3rd party smpp connection and need to send sms via that created smpp server. my problem is i doesn't know how to connect smpp server to smsc/smpp 3rd party connection
please help to me connect this smpp server to smpp 3rd party server for send sms..
public class SMPPServerSimulator extends ServerResponseDeliveryAdapter implements Runnable, ServerMessageReceiverListener {
private static final Logger log = LoggerFactory.getLogger(SMPPServerSimulator.class);
private static final String QUERYSM_NOT_IMPLEMENTED = "query_sm not implemented";
private static final String CANCELSM_NOT_IMPLEMENTED = "cancel_sm not implemented";
private static final String DATASM_NOT_IMPLEMENTED = "data_sm not implemented";
private static final String REPLACESM_NOT_IMPLEMENTED = "replace_sm not implemented";
private static final String BROADCASTSM_NOT_IMPLEMENTED = "broadcast_sm not implemented";
private static final String CANCELBROADCASTSM_NOT_IMPLEMENTED = "cancel_broadcast_sm not implemented";
private static final String QUERYBROADCASTSM_NOT_IMPLEMENTED = "query_broadcast_sm not implemented";
private static final Integer DEFAULT_PORT = 2775;
private static final String DEFAULT_SYSID = "twtest";
private static final String DEFAULT_PASSWORD = "test123";
private static final String SMSC_SYSTEMID = "sys";
private final ExecutorService execService = Executors.newFixedThreadPool(5);
private final ExecutorService execServiceDelReceipt = Executors.newFixedThreadPool(100);
private final MessageIDGenerator messageIDGenerator = new RandomMessageIDGenerator();
private final boolean useSsl;
private final int port;
private final String systemId;
private final String password;
public SMPPServerSimulator(boolean useSsl, int port, String systemId, String password) {
this.useSsl = useSsl;
this.port = port;
this.systemId = systemId;
this.password = password;
}
#Override
public void run() {
boolean running = true;
/*
* for SSL use the SSLServerSocketConnectionFactory() or DefaultSSLServerSocketConnectionFactory()
*/
try (SMPPServerSessionListener sessionListener = useSsl ?
new SMPPServerSessionListener(port, new KeyStoreSSLServerSocketConnectionFactory())
: new SMPPServerSessionListener(port)) {
/*
* for SSL use the SSLServerSocketConnectionFactory() or DefaultSSLServerSocketConnectionFactory()
*/
log.info("Listening on port {}{}", port, useSsl ? " (SSL)" : "");
while (running) {
System.out.println("running");
SMPPServerSession serverSession = sessionListener.accept();
log.info("Accepted connection with session {}", serverSession.getSessionId());
serverSession.setMessageReceiverListener(this);
serverSession.setResponseDeliveryListener(this);
Future<Boolean> bindResult = execService.submit(new WaitBindTask(serverSession, 30000, systemId, password));
try {
boolean bound = bindResult.get();
System.out.println(bound+" sss");
if (bound) {
// Could start deliver_sm to ESME
log.info("The session is now in state {}", serverSession.getSessionState());
serverSession.deliverShortMessage("CMT",
TypeOfNumber.INTERNATIONAL, NumberingPlanIndicator.ISDN, "94712320529",
TypeOfNumber.INTERNATIONAL, NumberingPlanIndicator.ISDN, "TWTEST",
new ESMClass(0), (byte) 0x00, (byte) 0x00, new RegisteredDelivery(), DataCodings.ZERO,
"Hello Worldxxxxxxxxxxx".getBytes());
}
} catch (InterruptedException e){
log.info("Interrupted WaitBind task: {}", e.getMessage());
Thread.currentThread().interrupt();
running = false;
} catch (ExecutionException e){
log.info("Exception on execute WaitBind task: {}", e.getMessage());
running = false;
} catch (NegativeResponseException | ResponseTimeoutException | PDUException | InvalidResponseException e){
log.info("Could not send deliver_sm: {}", e.getMessage());
}
}
} catch (IOException e) {
log.error("IO error occurred", e);
}
}
#Override
public QuerySmResult onAcceptQuerySm(QuerySm querySm, SMPPServerSession source) throws ProcessRequestException {
log.info("QuerySm not implemented");
throw new ProcessRequestException(QUERYSM_NOT_IMPLEMENTED, SMPPConstant.STAT_ESME_RINVCMDID);
}
#Override
public SubmitSmResult onAcceptSubmitSm(SubmitSm submitSm,
SMPPServerSession source) throws ProcessRequestException {
MessageId messageId = messageIDGenerator.newMessageId();
log.info("Receiving submit_sm '{}', and return message id {}", new String(submitSm.getShortMessage()), messageId);
if (SMSCDeliveryReceipt.FAILURE.containedIn(submitSm.getRegisteredDelivery()) || SMSCDeliveryReceipt.SUCCESS_FAILURE.containedIn(submitSm.getRegisteredDelivery())) {
execServiceDelReceipt.execute(new DeliveryReceiptTask(source, submitSm, messageId));
}
/*
* SMPP 5.0 allows the following optional parameters (SMPP 5.0 paragraph 4.2.5):
* additional_status_info_text, delivery_failure_reason, dpf_result, network_error_code
* Add the congestionState for SMPP 5.0 connections.
*/
if (source.getInterfaceVersion().value() >= InterfaceVersion.IF_50.value()) {
final int congestionRatio = source.getCongestionRatio();
OptionalParameter.Congestion_state congestionState = new OptionalParameter.Congestion_state((byte) congestionRatio);
return new SubmitSmResult(messageId, new OptionalParameter[]{ congestionState });
}
return new SubmitSmResult(messageId, new OptionalParameter[0]);
}
#Override
public void onSubmitSmRespSent(SubmitSmResult submitSmResult,
SMPPServerSession source) {
log.debug("submit_sm_resp with message_id {} has been sent", submitSmResult.getMessageId());
}
#Override
public SubmitMultiResult onAcceptSubmitMulti(SubmitMulti submitMulti, SMPPServerSession source)
throws ProcessRequestException {
MessageId messageId = messageIDGenerator.newMessageId();
log.debug("Receiving submit_multi_sm '{}', and return message id {}",
new String(submitMulti.getShortMessage()), messageId);
if (SMSCDeliveryReceipt.FAILURE.containedIn(submitMulti.getRegisteredDelivery())
|| SMSCDeliveryReceipt.SUCCESS_FAILURE.containedIn(submitMulti.getRegisteredDelivery())) {
execServiceDelReceipt.execute(new DeliveryReceiptTask(source, submitMulti, messageId));
}
/*
* SMPP 5.0 allows the following optional parameters (SMPP 5.0 paragraph 4.2.5):
* additional_status_info_text, delivery_failure_reason, dpf_result, network_error_code
* Add the congestionState for SMPP 5.0 connections.
*/
if (source.getInterfaceVersion().value() >= InterfaceVersion.IF_50.value()) {
final int congestionRatio = source.getCongestionRatio();
OptionalParameter.Congestion_state congestionState = new OptionalParameter.Congestion_state((byte) congestionRatio);
return new SubmitMultiResult(messageId.getValue(), new UnsuccessDelivery[0], new OptionalParameter[]{ congestionState });
}
return new SubmitMultiResult(messageId.getValue(), new UnsuccessDelivery[0], new OptionalParameter[0]);
}
#Override
public DataSmResult onAcceptDataSm(DataSm dataSm, Session source)
throws ProcessRequestException {
log.info("Accepting data_sm, but not implemented");
throw new ProcessRequestException(DATASM_NOT_IMPLEMENTED, SMPPConstant.STAT_ESME_RSYSERR);
}
#Override
public void onAcceptCancelSm(CancelSm cancelSm, SMPPServerSession source)
throws ProcessRequestException {
log.info("Accepting cancel_sm, but not implemented");
throw new ProcessRequestException(CANCELSM_NOT_IMPLEMENTED, SMPPConstant.STAT_ESME_RCANCELFAIL);
}
#Override
public void onAcceptReplaceSm(ReplaceSm replaceSm, SMPPServerSession source)
throws ProcessRequestException {
log.info("Accepting replace_sm, but not implemented");
throw new ProcessRequestException(REPLACESM_NOT_IMPLEMENTED, SMPPConstant.STAT_ESME_RREPLACEFAIL);
}
#Override
public BroadcastSmResult onAcceptBroadcastSm(final BroadcastSm broadcastSm, final SMPPServerSession source)
throws ProcessRequestException {
MessageId messageId = messageIDGenerator.newMessageId();
log.debug("Receiving broadcast_sm '{}', and return message id {}",
new String(broadcastSm.getOptionalParameter(OptionalParameter.Tag.MESSAGE_PAYLOAD).serialize()), messageId);
return new BroadcastSmResult(messageId, new OptionalParameter[0]);
}
#Override
public void onAcceptCancelBroadcastSm(final CancelBroadcastSm cancelBroadcastSm, final SMPPServerSession source)
throws ProcessRequestException {
log.info("Accepting cancel_broadcast_sm, but not implemented");
throw new ProcessRequestException(CANCELBROADCASTSM_NOT_IMPLEMENTED, SMPPConstant.STAT_ESME_RBCASTCANCELFAIL);
}
#Override
public QueryBroadcastSmResult onAcceptQueryBroadcastSm(final QueryBroadcastSm queryBroadcastSm,
final SMPPServerSession source) throws ProcessRequestException {
log.info("Accepting query_broadcast_sm, but not implemented");
throw new ProcessRequestException(QUERYBROADCASTSM_NOT_IMPLEMENTED, SMPPConstant.STAT_ESME_RBCASTQUERYFAIL);
}
private static class WaitBindTask implements Callable<Boolean> {
private final SMPPServerSession serverSession;
private final long timeout;
private final String systemId;
private final String password;
public WaitBindTask(SMPPServerSession serverSession, long timeout, String systemId, String password) {
this.serverSession = serverSession;
this.timeout = timeout;
this.systemId = systemId;
this.password = password;
}
#Override
public Boolean call() {
try {
System.out.println("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
BindRequest bindRequest = serverSession.waitForBind(timeout);
try {
if (BindType.BIND_TRX.equals(bindRequest.getBindType())) {
if (systemId.equals(bindRequest.getSystemId())) {
if (password.equals(bindRequest.getPassword())) {
log.info("Accepting bind for session {}, interface version {}", serverSession.getSessionId(), bindRequest.getInterfaceVersion());
serverSession.setInterfaceVersion(InterfaceVersion.IF_50.min(bindRequest.getInterfaceVersion()));
// The systemId identifies the SMSC to the ESME.
bindRequest.accept(SMSC_SYSTEMID, InterfaceVersion.IF_50);
return true;
} else {
log.info("Rejecting bind for session {}, interface version {}, invalid password", serverSession.getSessionId(), bindRequest.getInterfaceVersion());
bindRequest.reject(SMPPConstant.STAT_ESME_RINVPASWD);
}
} else {
log.info("Rejecting bind for session {}, interface version {}, invalid system id", serverSession.getSessionId(), bindRequest.getInterfaceVersion());
bindRequest.reject(SMPPConstant.STAT_ESME_RINVSYSID);
}
} else {
log.info("Rejecting bind for session {}, interface version {}, only accept transceiver", serverSession.getSessionId(), bindRequest.getInterfaceVersion());
bindRequest.reject(SMPPConstant.STAT_ESME_RBINDFAIL);
}
} catch (PDUStringException e) {
log.error("Invalid system id: " + SMSC_SYSTEMID, e);
bindRequest.reject(SMPPConstant.STAT_ESME_RSYSERR);
}
} catch (IllegalStateException e) {
log.error("System error", e);
} catch (TimeoutException e) {
log.warn("Wait for bind has reach timeout", e);
} catch (IOException e) {
log.error("Failed accepting bind request for session {}", serverSession.getSessionId());
}
return false;
}
}
private static class DeliveryReceiptTask implements Runnable {
private final SMPPServerSession session;
private final MessageId messageId;
private final TypeOfNumber sourceAddrTon;
private final NumberingPlanIndicator sourceAddrNpi;
private final String sourceAddress;
private final TypeOfNumber destAddrTon;
private final NumberingPlanIndicator destAddrNpi;
private final String destAddress;
private final int totalSubmitted;
private final int totalDelivered;
private final byte[] shortMessage;
public DeliveryReceiptTask(SMPPServerSession session,
SubmitSm submitSm, MessageId messageId) {
this.session = session;
this.messageId = messageId;
// reversing destination to source
sourceAddrTon = TypeOfNumber.valueOf(submitSm.getDestAddrTon());
sourceAddrNpi = NumberingPlanIndicator.valueOf(submitSm.getDestAddrNpi());
sourceAddress = submitSm.getDestAddress();
// reversing source to destination
destAddrTon = TypeOfNumber.valueOf(submitSm.getSourceAddrTon());
destAddrNpi = NumberingPlanIndicator.valueOf(submitSm.getSourceAddrNpi());
destAddress = submitSm.getSourceAddr();
totalSubmitted = totalDelivered = 1;
shortMessage = submitSm.getShortMessage();
}
public DeliveryReceiptTask(SMPPServerSession session,
SubmitMulti submitMulti, MessageId messageId) {
this.session = session;
this.messageId = messageId;
// set to unknown and null, since it was submit_multi
sourceAddrTon = TypeOfNumber.UNKNOWN;
sourceAddrNpi = NumberingPlanIndicator.UNKNOWN;
sourceAddress = null;
// reversing source to destination
destAddrTon = TypeOfNumber.valueOf(submitMulti.getSourceAddrTon());
destAddrNpi = NumberingPlanIndicator.valueOf(submitMulti.getSourceAddrNpi());
destAddress = submitMulti.getSourceAddr();
// distribution list assumed only contains single address
totalSubmitted = totalDelivered = submitMulti.getDestAddresses().length;
shortMessage = submitMulti.getShortMessage();
}
#Override
public void run() {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
log.error("Interrupted", e);
//re-interrupt the current thread
Thread.currentThread().interrupt();
}
SessionState state = session.getSessionState();
if (!state.isReceivable()) {
log.debug("Not sending delivery receipt for message id {} since session state is {}", messageId, state);
return;
}
String stringValue = Integer.valueOf(messageId.getValue(), 16).toString();
try {
DeliveryReceipt delRec = new DeliveryReceipt(stringValue, totalSubmitted, totalDelivered, new Date(), new Date(), DeliveryReceiptState.DELIVRD, "000", new String(shortMessage));
session.deliverShortMessage(
"mc",
sourceAddrTon, sourceAddrNpi, sourceAddress,
destAddrTon, destAddrNpi, destAddress,
new ESMClass(MessageMode.DEFAULT, MessageType.SMSC_DEL_RECEIPT, GSMSpecificFeature.DEFAULT),
(byte)0,
(byte)0,
new RegisteredDelivery(0),
DataCodings.ZERO,
delRec.toString().getBytes());
log.debug("Sending delivery receipt for message id {}: {}", messageId, stringValue);
} catch (Exception e) {
log.error("Failed sending delivery_receipt for message id " + messageId + ":" + stringValue, e);
}
}
}
public static void main(String[] args) {
// System.setProperty("javax.net.debug", "ssl");
/*
* To use SSL, add -Djsmpp.simulator.ssl=true
* To debug SSL, add -Djavax.net.debug=ssl
*/
String systemId = System.getProperty("jsmpp.client.systemId", DEFAULT_SYSID);
String password = System.getProperty("jsmpp.client.password", DEFAULT_PASSWORD);
int port;
try {
port = Integer.parseInt(System.getProperty("jsmpp.simulator.port", DEFAULT_PORT.toString()));
} catch (NumberFormatException e) {
port = DEFAULT_PORT;
}
boolean useSsl = Boolean.parseBoolean(System.getProperty("jsmpp.simulator.ssl", "false"));
SMPPServerSimulator smppServerSim = new SMPPServerSimulator(useSsl, port, systemId, password);
smppServerSim.run();
}
}

UDP Socket: java.net.SocketException: socket closed

MessageCreator: Encapsulate and resolve ports and device unique identifiers.
public class MessageCreator {
private static final String HEADER_PORT = "to port:";
private static final String HEADER_SN = "My sn:";
public static String buildWithPort(int port) {
return HEADER_PORT + port;
}
public static int parsePort(String data) {
if (data.startsWith(HEADER_PORT)) {
return Integer.parseInt(data.substring(HEADER_PORT.length()));
}
return -1;
}
public static String buildWithSn(String sn) {
return HEADER_SN + sn;
}
public static String parseSn(String data) {
if (data.startsWith(HEADER_SN)) {
return data.substring(HEADER_SN.length());
}
return null;
}
}
UdpProvider: Loop to listen to a specific port, then parse the received data, determine whether the data conforms to the predetermined format, get the sender's response port from it, and respond with the uniquely identified UUID value to the UDP searcher.
public class UdpProvider {
public static void main(String[] args) throws IOException {
String sn = UUID.randomUUID().toString();
Provider provider = new Provider(sn);
provider.start();
// Warning: Result of 'InputStream.read()' is ignored
System.in.read();
provider.exit();
}
private static class Provider extends Thread {
private DatagramSocket ds = null;
private boolean done = false;
private final String sn;
public Provider(String sn) {
super();
this.sn = sn;
}
#Override
public void run() {
super.run();
try {
ds = new DatagramSocket(20000);
while (!done) {
final byte[] buf = new byte[512];
DatagramPacket receivePak = new DatagramPacket(buf, buf.length);
ds.receive(receivePak);
String ip = receivePak.getAddress().getHostAddress();
int port = receivePak.getPort();
byte[] receivePakData = receivePak.getData();
String receiveData = new String(receivePakData, 0, /*receivePakData.length*/receivePak.getLength());
System.out.println("received from -> ip: " + ip + ", port: " + port + ", data: " + receiveData);
int responsePort = MessageCreator.parsePort(receiveData.trim());
if (responsePort != -1) {
String responseData = MessageCreator.buildWithSn(sn);
byte[] bytes = responseData.getBytes(StandardCharsets.UTF_8);
DatagramPacket responsePak = new DatagramPacket(bytes, bytes.length,
/*InetAddress.getLocalHost()*/
receivePak.getAddress(),
responsePort);
ds.send(responsePak);
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
close();
}
}
public void exit() {
done = true;
close();
}
public void close() {
if (ds != null) {
ds.close();
ds = null;
}
}
}
}
UdpSearcher: Listening to a specific port and sending a LAN broadcast, sending a broadcast sets the listening port in the data, so you need to turn on listening first to finish before you can send a broadcast, and once you receive the response data, you can parse the device information
public class UdpSearcher {
private static final int LISTENER_PORT = 30000;
public static void main(String[] args) throws IOException, InterruptedException {
Listener listener = listen();
sendBroadcast();
// Warning: Result of 'InputStream.read()' is ignored
System.in.read();
List<Device> deviceList = listener.getDevicesAndClose();
for (Device device : deviceList) {
System.out.println(device);
}
}
public static void sendBroadcast() throws IOException {
DatagramSocket ds = new DatagramSocket();
String sendData = MessageCreator.buildWithPort(LISTENER_PORT);
byte[] sendDataBytes = sendData.getBytes(StandardCharsets.UTF_8);
DatagramPacket sendPak = new DatagramPacket(sendDataBytes, sendDataBytes.length);
sendPak.setAddress(InetAddress.getByName("255.255.255.255"));
sendPak.setPort(20000);
ds.send(sendPak);
ds.close();
}
public static Listener listen() throws InterruptedException {
CountDownLatch countDownLatch = new CountDownLatch(1);
Listener listener = new Listener(LISTENER_PORT, countDownLatch);
listener.start();
countDownLatch.await();
return listener;
}
private static class Listener extends Thread {
private final int listenPort;
private DatagramSocket ds = null;
private boolean done = false;
private final CountDownLatch countDownLatch;
private List<Device> devices = new ArrayList<>();
public Listener(int listenPort, CountDownLatch countDownLatch) {
super();
this.listenPort = listenPort;
this.countDownLatch = countDownLatch;
}
#Override
public void run() {
super.run();
countDownLatch.countDown();
try {
ds = new DatagramSocket(listenPort);
while (!done) {
final byte[] buf = new byte[512];
DatagramPacket receivePak = new DatagramPacket(buf, buf.length);
ds.receive(receivePak);
String ip = receivePak.getAddress().getHostAddress();
int port = receivePak.getPort();
byte[] data = receivePak.getData();
String receiveData = new String(data, 0, /*data.length*/receivePak.getLength());
String sn = MessageCreator.parseSn(receiveData);
System.out.println("received from -> ip: " + ip + ", port: " + port + ", data: " + receiveData);
if (sn != null) {
Device device = new Device(ip, port, sn);
devices.add(device);
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
close();
}
}
public void close() {
if (ds != null) {
ds.close();
ds = null;
}
}
public List<Device> getDevicesAndClose() {
done = true;
close();
return devices;
}
}
private static class Device {
private String ip;
private int port;
private String sn;
public Device(String ip, int port, String sn) {
this.ip = ip;
this.port = port;
this.sn = sn;
}
#Override
public String toString() {
return "Device{" +
"ip='" + ip + '\'' +
", port=" + port +
", sn='" + sn + '\'' +
'}';
}
}
}
UdpProvider and UdpSearcher worked fine and printed corrresponding data until I input a char sequence from keyboard follwed by pressing Enter key on each console window, both threw an exception on this line ds.receive(receivePak); :

Apache Hbase MapReduce job take too much time while reading the datastore

I have setup Apache Hbase, Nutch and Hadoop cluster. I have crawled few documents i.e., about 30 Million. There are 3 workers in the cluster and 1 master. I have write my own Hbase mapreduce job to read crawled data and change some score little bit based on some logic.
For this purpose, I have combined the documents of same domain and found their effective bytes and found some score. Later, in reducer, I have assigned that score to each URL of that domain (via cache). This portion of job takes took much time i.e., 16 hours. Following is the code snippet
for ( int index = 0; index < Cache.size(); index++) {
String Orig_key = Cache.get(index);
float doc_score = log10;
WebPage page = datastore.get(Orig_key);
if ( page == null ) {
continue;
}
page.setScore(doc_score);
if (mark) {
page.getMarkers().put( Queue, Q1);
}
context.write(Orig_key, page);
}
If I remove that document read statement from datastore then job is finished in 2 to 3 hours only. That why, I think the statement WebPage page = datastore.get(Orig_key); is causing this problem. Is'nt it ?
If that is the case then what is best approach. The Cache object is simply a list that contains URLs of same domain.
DomainAnalysisJob.java
...
...
public class DomainAnalysisJob implements Tool {
public static final Logger LOG = LoggerFactory
.getLogger(DomainAnalysisJob.class);
private static final Collection<WebPage.Field> FIELDS = new HashSet<WebPage.Field>();
private Configuration conf;
protected static final Utf8 URL_ORIG_KEY = new Utf8("doc_orig_id");
protected static final Utf8 DOC_DUMMY_MARKER = new Utf8("doc_marker");
protected static final Utf8 DUMMY_KEY = new Utf8("doc_id");
protected static final Utf8 DOMAIN_DUMMY_MARKER = new Utf8("domain_marker");
protected static final Utf8 LINK_MARKER = new Utf8("link");
protected static final Utf8 Queue = new Utf8("q");
private static URLNormalizers urlNormalizers;
private static URLFilters filters;
private static int maxURL_Length;
static {
FIELDS.add(WebPage.Field.STATUS);
FIELDS.add(WebPage.Field.LANG_INFO);
FIELDS.add(WebPage.Field.URDU_SCORE);
FIELDS.add(WebPage.Field.MARKERS);
FIELDS.add(WebPage.Field.INLINKS);
}
/**
* Maps each WebPage to a host key.
*/
public static class Mapper extends GoraMapper<String, WebPage, Text, WebPage> {
#Override
protected void setup(Context context) throws IOException ,InterruptedException {
Configuration conf = context.getConfiguration();
urlNormalizers = new URLNormalizers(context.getConfiguration(), URLNormalizers.SCOPE_DEFAULT);
filters = new URLFilters(context.getConfiguration());
maxURL_Length = conf.getInt("url.characters.max.length", 2000);
}
#Override
protected void map(String key, WebPage page, Context context)
throws IOException, InterruptedException {
String reversedHost = null;
if (page == null) {
return;
}
if ( key.length() > maxURL_Length ) {
return;
}
String url = null;
try {
url = TableUtil.unreverseUrl(key);
url = urlNormalizers.normalize(url, URLNormalizers.SCOPE_DEFAULT);
url = filters.filter(url); // filter the url
} catch (Exception e) {
LOG.warn("Skipping " + key + ":" + e);
return;
}
if ( url == null) {
context.getCounter("DomainAnalysis", "FilteredURL").increment(1);
return;
}
try {
reversedHost = TableUtil.getReversedHost(key.toString());
}
catch (Exception e) {
return;
}
page.getMarkers().put( URL_ORIG_KEY, new Utf8(key) );
context.write( new Text(reversedHost), page );
}
}
public DomainAnalysisJob() {
}
public DomainAnalysisJob(Configuration conf) {
setConf(conf);
}
#Override
public Configuration getConf() {
return conf;
}
#Override
public void setConf(Configuration conf) {
this.conf = conf;
}
public void updateDomains(boolean buildLinkDb, int numTasks) throws Exception {
NutchJob job = NutchJob.getInstance(getConf(), "rankDomain-update");
job.getConfiguration().setInt("mapreduce.task.timeout", 1800000);
if ( numTasks < 1) {
job.setNumReduceTasks(job.getConfiguration().getInt(
"mapred.map.tasks", job.getNumReduceTasks()));
} else {
job.setNumReduceTasks(numTasks);
}
ScoringFilters scoringFilters = new ScoringFilters(getConf());
HashSet<WebPage.Field> fields = new HashSet<WebPage.Field>(FIELDS);
fields.addAll(scoringFilters.getFields());
StorageUtils.initMapperJob(job, fields, Text.class, WebPage.class,
Mapper.class);
StorageUtils.initReducerJob(job, DomainAnalysisReducer.class);
job.waitForCompletion(true);
}
#Override
public int run(String[] args) throws Exception {
boolean linkDb = false;
int numTasks = -1;
for (int i = 0; i < args.length; i++) {
if ("-rankDomain".equals(args[i])) {
linkDb = true;
} else if ("-crawlId".equals(args[i])) {
getConf().set(Nutch.CRAWL_ID_KEY, args[++i]);
} else if ("-numTasks".equals(args[i]) ) {
numTasks = Integer.parseInt(args[++i]);
}
else {
throw new IllegalArgumentException("unrecognized arg " + args[i]
+ " usage: updatedomain -crawlId <crawlId> [-numTasks N]" );
}
}
LOG.info("Updating DomainRank:");
updateDomains(linkDb, numTasks);
return 0;
}
public static void main(String[] args) throws Exception {
final int res = ToolRunner.run(NutchConfiguration.create(),
new DomainAnalysisJob(), args);
System.exit(res);
}
}
DomainAnalysisReducer.java
...
...
public class DomainAnalysisReducer extends
GoraReducer<Text, WebPage, String, WebPage> {
public static final Logger LOG = DomainAnalysisJob.LOG;
public DataStore<String, WebPage> datastore;
protected static float q1_ur_threshold = 500.0f;
protected static float q1_ur_docCount = 50;
public static final Utf8 Queue = new Utf8("q"); // Markers for Q1 and Q2
public static final Utf8 Q1 = new Utf8("q1");
public static final Utf8 Q2 = new Utf8("q2");
#Override
protected void setup(Context context) throws IOException,
InterruptedException {
Configuration conf = context.getConfiguration();
try {
datastore = StorageUtils.createWebStore(conf, String.class, WebPage.class);
}
catch (ClassNotFoundException e) {
throw new IOException(e);
}
q1_ur_threshold = conf.getFloat("domain.queue.threshold.bytes", 500.0f);
q1_ur_docCount = conf.getInt("domain.queue.doc.count", 50);
LOG.info("Conf updated: Queue-bytes-threshold = " + q1_ur_threshold + " Queue-doc-threshold: " + q1_ur_docCount);
}
#Override
protected void cleanup(Context context) throws IOException, InterruptedException {
datastore.close();
}
#Override
protected void reduce(Text key, Iterable<WebPage> values, Context context)
throws IOException, InterruptedException {
ArrayList<String> Cache = new ArrayList<String>();
int doc_counter = 0;
int total_ur_bytes = 0;
for ( WebPage page : values ) {
// cache
String orig_key = page.getMarkers().get( DomainAnalysisJob.URL_ORIG_KEY ).toString();
Cache.add(orig_key);
// do not consider those doc's that are not fetched or link URLs
if ( page.getStatus() == CrawlStatus.STATUS_UNFETCHED ) {
continue;
}
doc_counter++;
int ur_score_int = 0;
int doc_ur_bytes = 0;
int doc_total_bytes = 0;
String ur_score_str = "0";
String langInfo_str = null;
// read page and find its Urdu score
langInfo_str = TableUtil.toString(page.getLangInfo());
if (langInfo_str == null) {
continue;
}
ur_score_str = TableUtil.toString(page.getUrduScore());
ur_score_int = Integer.parseInt(ur_score_str);
doc_total_bytes = Integer.parseInt( langInfo_str.split("&")[0] );
doc_ur_bytes = ( doc_total_bytes * ur_score_int) / 100; //Formula to find ur percentage
total_ur_bytes += doc_ur_bytes;
}
float avg_bytes = 0;
float log10 = 0;
if ( doc_counter > 0 && total_ur_bytes > 0) {
avg_bytes = (float) total_ur_bytes/doc_counter;
log10 = (float) Math.log10(avg_bytes);
log10 = (Math.round(log10 * 100000f)/100000f);
}
context.getCounter("DomainAnalysis", "DomainCount").increment(1);
// if average bytes and doc count, are more than threshold then mark as q1
boolean mark = false;
if ( avg_bytes >= q1_ur_threshold && doc_counter >= q1_ur_docCount ) {
mark = true;
for ( int index = 0; index < Cache.size(); index++) {
String Orig_key = Cache.get(index);
float doc_score = log10;
WebPage page = datastore.get(Orig_key);
if ( page == null ) {
continue;
}
page.setScore(doc_score);
if (mark) {
page.getMarkers().put( Queue, Q1);
}
context.write(Orig_key, page);
}
}
}
In my testing and debugging, I have found that the statement WebPage page = datastore.get(Orig_key); is major cause of too much time. It took about 16 hours to complete the job but when I replaced this statement with WebPage page = WebPage.newBuilder().build(); the time was reduced to 6 hours. Is this due to IO ?

Android SSDP Service Discovery

I am implementing an java program that scans some of the smart devices that I have, using SSDP discovery. The code that I am using detects only 2 devices among 10 that are connected to my local network. I could not understand the reason is.
My question is, why I am not able to get all the devices that are in my local network.
This is the code that I am using
static final String HOST = "Host:" + SSDP.ADDRESS + ":" + SSDP.PORT;
static final String MAN = "Man: \"ssdp:discover\"";
static final String NEWLINE = "\r\n";
int mMX = 10; /* seconds to delay response */
String mST; /* Search target */
public SSDPSearchMsg(String ST) {
mST = ST;
}
public int getmMX() {
return mMX;
}
public void setmMX(int mMX) {
this.mMX = mMX;
}
public String getmST() {
return mST;
}
public void setmST(String mST) {
this.mST = mST;
}
#Override
public String toString() {
StringBuilder content = new StringBuilder();
content.append(SSDP.SL_MSEARCH).append(NEWLINE);
content.append(HOST).append(NEWLINE);
content.append(MAN).append(NEWLINE);
content.append(mST).append(NEWLINE);
content.append("MX:" + mMX).append(NEWLINE);
content.append(NEWLINE);
return content.toString();
}
Below is few constants
/* New line definition */
public static final String NEWLINE = "\r\n";
public static final String ADDRESS = "239.255.255.250";
public static final int PORT = 1900;
/* Definitions of start line */
public static final String SL_NOTIFY = "NOTIFY * HTTP/1.1";
public static final String SL_MSEARCH = "M-SEARCH * HTTP/1.1";
public static final String SL_OK = "HTTP/1.1 200 OK";
/* Definitions of search targets */
public static final String ST_RootDevice = "ST: upnp:rootdevice";
public static final String ST_ContentDirectory = "ST: urn:schemas-upnp-org:service:ContentDirectory:1";
public static final String ST_ALL="ST: ssdp:all";
public static final String ST_Media="ST:urn:schemas-upnp-org:device:MediaRenderer:1";
/* Definitions of notification sub type */
public static final String NTS_ALIVE = "NTS:ssdp:alive";
public static final String NTS_BYE = "NTS:ssdp:byebye";
public static final String NTS_UPDATE = "NTS:ssdp:update";
Main class
public class SSDPSocket {
SocketAddress mSSDPMulticastGroup;
MulticastSocket mSSDPSocket;
public SSDPSocket() throws IOException {
InetAddress localInAddress = InetAddress.getLocalHost();
System.out.println("Local address: " + localInAddress.getHostAddress());
mSSDPMulticastGroup = new InetSocketAddress(SSDP.ADDRESS, SSDP.PORT);
mSSDPSocket = new MulticastSocket(new InetSocketAddress(localInAddress,
0));
NetworkInterface netIf = NetworkInterface
.getByInetAddress(localInAddress);
mSSDPSocket.joinGroup(mSSDPMulticastGroup, netIf);
}
/* Used to send SSDP packet */
public void send(String data) throws IOException {
DatagramPacket dp = new DatagramPacket(data.getBytes(), data.length(),
mSSDPMulticastGroup);
mSSDPSocket.send(dp);
}
/* Used to receive SSDP packet */
public DatagramPacket receive() throws IOException {
byte[] buf = new byte[1024];
DatagramPacket dp = new DatagramPacket(buf, buf.length);
mSSDPSocket.receive(dp);
return dp;
}
public void close() {
if (mSSDPSocket != null) {
mSSDPSocket.close();
}
}
/* For test purpose */
public static void main(String[] args) {
SSDPSearchMsg search = new SSDPSearchMsg(SSDP.ST_Media);
System.out.println(search.toString());
SSDPSocket sock;
try {
sock = new SSDPSocket();
sock.send(search.toString());
while (true) {
DatagramPacket dp = sock.receive();
String c = new String(dp.getData());
System.out.println(c);
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
Can anyone please suggest a solution for this problem?

I just worked on an SSDP Java client and studied the spec carefully.
You are missing an "MX" header. If the header MX is not set, the SSDP Service "SHOULD" not respond to your request.
Therefore, try adding an "MX: 1" header and see where this goes.
The Schneider database was of good help for this issue.
Also, you can take a look at the ssdp client if you still are trying to use it.

Unable to receive proper UDP packets using SSDP

I'm trying to implement a very simple SSDP functionality into my android app taken from here.
My application sends some UDP packets containing a relevant M-SEARCH message to the broadcasting address without any issue. The problem is, I should be getting a proper response back from other devices that's running a UPNP server. For some reason, I'm only receiving the exact same packets back I sent from my android device.
MainActivity.java
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
WifiManager wm = (WifiManager)getSystemService(Context.WIFI_SERVICE);
WifiManager.MulticastLock multicastLock = wm.createMulticastLock("multicastLock");
multicastLock.setReferenceCounted(true);
multicastLock.acquire();
setContentView(R.layout.activity_main);
((Button)this.findViewById(R.id.btnSendSSDPSearch)).setOnClickListener(this);
}
#Override
public void onClick(View v) {
switch (v.getId()) {
case R.id.btnSendSSDPSearch:
new Thread(new Runnable() {
#Override
public void run() {
SendMSearchMessage();
}
}).start();
default:
break;
}
}
private void SendMSearchMessage() {
SSDPSearchMsg searchContentDirectory = new SSDPSearchMsg(SSDPConstants.ST_ContentDirectory);
SSDPSearchMsg searchAVTransport = new SSDPSearchMsg(SSDPConstants.ST_AVTransport);
SSDPSearchMsg searchProduct = new SSDPSearchMsg(SSDPConstants.ST_Product);
SSDPSocket sock;
try {
sock = new SSDPSocket();
for (int i = 0; i < 2; i++) {
sock.send(searchContentDirectory.toString());
sock.send(searchAVTransport.toString());
sock.send(searchProduct.toString());
}
while (true) {
DatagramPacket dp = sock.receive(); //Here, I only receive the same packets I initially sent above
String c = new String(dp.getData());
System.out.println(c);
}
} catch (IOException e) {
// TODO Auto-generated catch block
Log.e("M-SEARCH", e.getMessage());
}
SSDPSocket.java where the UDP packet transmission is actually done
public class SSDPSocket {
SocketAddress mSSDPMulticastGroup;
MulticastSocket mSSDPSocket;
InetAddress broadcastAddress;
public SSDPSocket() throws IOException {
mSSDPSocket = new MulticastSocket(55325); //Bind some random port for receiving datagram
broadcastAddress = InetAddress.getByName(SSDPConstants.ADDRESS);
mSSDPSocket.joinGroup(broadcastAddress);
}
/* Used to send SSDP packet */
public void send(String data) throws IOException {
DatagramPacket dp = new DatagramPacket(data.getBytes(), data.length(),
broadcastAddress,SSDPConstants.PORT);
mSSDPSocket.send(dp);
}
/* Used to receive SSDP packet */
public DatagramPacket receive() throws IOException {
byte[] buf = new byte[1024];
DatagramPacket dp = new DatagramPacket(buf, buf.length);
mSSDPSocket.receive(dp);
return dp;
}
public void close() {
if (mSSDPSocket != null) {
mSSDPSocket.close();
}
}
}
SSDPSearchMsg.java for constructing SSDP Broadcast string
(Probably unrelated to the problem I'm experiencing but just in case)
public class SSDPSearchMsg {
static final String HOST = "Host:" + SSDP.ADDRESS + ":" + SSDP.PORT;
static final String MAN = "Man:ssdp:discover";
static final String NEWLINE = System.getProperty("line.separator");
int mMX = 3; /* seconds to delay response */
String mST; /* Search target */
public SSDPSearchMsg(String ST) {
mST = ST;
}
public int getmMX() {
return mMX;
}
public void setmMX(int mMX) {
this.mMX = mMX;
}
public String getmST() {
return mST;
}
public void setmST(String mST) {
this.mST = mST;
}
#Override
public String toString() {
StringBuilder content = new StringBuilder();
content.append(SSDP.SL_MSEARCH).append(NEWLINE);
content.append(HOST).append(NEWLINE);
content.append(MAN).append(NEWLINE);
content.append(mST).append(NEWLINE);
content.append("MX:" + mMX).append(NEWLINE);
content.append(NEWLINE);
return content.toString();
}
}
SSDPConstants.java
public class SSDPConstants {
/* New line definition */
public static final String NEWLINE = "\r\n";
public static final String ADDRESS = "239.255.255.250";
public static final int PORT = 1900;
/* Definitions of start line */
public static final String SL_NOTIFY = "NOTIFY * HTTP/1.1";
public static final String SL_MSEARCH = "M-SEARCH * HTTP/1.1";
public static final String SL_OK = "HTTP/1.1 200 OK";
/* Definitions of search targets */
public static final String ST_RootDevice = "St: rootdevice";
public static final String ST_ContentDirectory = "St: urn:schemas-upnp-org:service:ContentDirectory:1";
public static final String ST_AVTransport = "St: urn:schemas-upnp-org:service:AVTransport:1";
public static final String ST_Product = "St: urn:av-openhome-org:service:Product:1";
/* Definitions of notification sub type */
public static final String NTS_ALIVE = "NTS:ssdp:alive";
public static final String NTS_BYE = "NTS:ssdp:byebye";
public static final String NTS_UPDATE = "NTS:ssdp:update";
}
I also made sure that the manifest includes relevant permissions:
<uses-permission android:name="android.permission.INTERNET"/>
<uses-permission android:name="android.permission.CHANGE_WIFI_MULTICAST_STATE" />
<uses-permission android:name="android.permission.ACCESS_WIFI_STATE" />
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
I'm testing the app on an actual device, not on emulator.
Any assistance would be appreciated.
Edit upon comment:
Multicast itself should work without issue. I've downloaded an app called BubbleUPNP to test the SSDP functionality. Sure enough, wireshark properly captures all messages sent from the phone to the broadcasting address in SSDP Protocol:
M-SEARCH * HTTP/1.1
Man: "ssdp:discover"
Mx: 3
Host: 239.255.255.250:1900
St: urn:schemas-upnp-org:service:AVTransport:1
And the response
HTTP/1.1 200 OK
ST:urn:schemas-upnp-org:service:ContentDirectory:1
USN:uuid:d5829e90-73ce-4213-9ad1-4e75dbdd0232::urn:schemas-upnp-org:service:ContentDirectory:1
Location:http://10.175.95.4:2869/upnphost/udhisapi.dll?content=uuid:d5829e90-73ce-4213-9ad1-4e75dbdd0232
OPT:"http://schemas.upnp.org/upnp/1/0/"; ns=01
01-NLS:05f3dd08b4b4b5aafa1fe983fa447f49
Cache-Control:max-age=900
Server:Microsoft-Windows-NT/5.1 UPnP/1.0 UPnP-Device-Host/1.0
So yeah, this is without a doubt, an implementation issue. Nothing wrong with the device.

Weird. I fixed the issue but I'm genuinely not sure what made it work.
Here are some changes that I made:
Instead of assigning a fixed port, I made it dynamically allocate an available port.
public class SSDPSocket {
SocketAddress mSSDPMulticastGroup;
MulticastSocket mSSDPSocket;
InetAddress broadcastAddress;
public SSDPSocket() throws IOException {
mSSDPSocket = new MulticastSocket();
broadcastAddress = InetAddress.getByName(SSDPConstants.ADDRESS);
mSSDPSocket.joinGroup(broadcastAddress);
}
...
}
I also changed the M-Search message structure, including its order.
public class SSDPSearchMsg {
static final String HOST = "Host: " + SSDPConstants.ADDRESS + ":" + SSDPConstants.PORT;
static final String MAN = "Man: \"ssdp:discover\"";
static final String NEWLINE = "\r\n";
int mMX = 3; /* seconds to delay response */
String mST; /* Search target */
public SSDPSearchMsg(String ST) {
mST = ST;
}
public int getmMX() {
return mMX;
}
public void setmMX(int mMX) {
this.mMX = mMX;
}
public String getmST() {
return mST;
}
public void setmST(String mST) {
this.mST = mST;
}
#Override
public String toString() {
StringBuilder content = new StringBuilder();
content.append(SSDPConstants.SL_MSEARCH).append(NEWLINE);
content.append(MAN).append(NEWLINE);
content.append("Mx: " + mMX).append(NEWLINE);
content.append(HOST).append(NEWLINE);
content.append(mST).append(NEWLINE);
content.append(NEWLINE);
return content.toString();
}
}
And everything suddenly works. Why it works is beyond me. My previous implementation follows the SSDP protocol as far as I can tell.

On possible answer is that you may have an "old" device. Apparently multicast (from Java) is broken before Android 2.3.7
Reference: https://stackoverflow.com/a/9836464/139985
Another possibility is that it is a device-specific problem; e.g. like this: https://stackoverflow.com/a/3714848/139985. (I'm not saying it is that specific problem ...)
Another is that multicast is disabled in the kernel configs: http://code.google.com/p/android/issues/detail?id=51195
It seems like there are a range of different causes for multicast not working on various Android devices ...

I followed your code and change some words to upper case. It works.
static final String HOST = "HOST: " + SSDPConstants.ADDRESS + ":" + SSDPConstants.PORT;
static final String MAN = "MAN: \"ssdp:discover\"";
Don't need to change the order in MSEARCH message.