I’ve a use case where I need to fetch all the records from Cassandra for a given time range and divide it into 30 chunks then further aggregate each chunk, for example let us suppose I’m fetching 60 records for a time range of 30 minutes. Now I need to divide into 30 chunk which will be 2 records per minute. If I’m fetching 600 records for a time range of 1 hour, then 30 chunk will be 20 records per 2 minutes. If I’m fetching 600 records for a time range of 1 week, then 30 chunk will be 20 records per 5.6 hours and so on.
For implementing the same I have written a java code which is giving result in 3 seconds for 100k records. I thought implementing the same in Cassandra UDF will have performance benefit, but the UDF is taking 6-7 seconds(double the time taken by java code) which is shocking for me. Somebody please guide, where I’m off the track, below is my table structure and java as well as UDF code.
Cassandra table schema
CREATE TABLE transactions_data (
app_name text,
api_name text,
app_id text,
start_time timestamp,
duration int,
end_time timestamp,
node_id text,
request_body text,
request_parameter_name1 text,
request_parameter_name2 text,
request_parameter_name3 text,
request_parameter_name4 text,
request_parameter_name5 text,
request_parameter_value1 text,
request_parameter_value2 text,
request_parameter_value3 text,
request_parameter_value4 text,
request_parameter_value5 text,
response_body text,
response_parameter_name1 text,
response_parameter_name2 text,
response_parameter_name3 text,
response_parameter_name4 text,
response_parameter_name5 text,
response_parameter_value1 text,
response_parameter_value2 text,
response_parameter_value3 text,
response_parameter_value4 text,
response_parameter_value5 text,
responsestatus text,
responsestatuscode text,
transaction_id text,
PRIMARY KEY ((app_name, api_name, app_id), start_time)
);
Java code
public class SamplingDataJava {
private static Logger logger = LoggerFactory.getLogger(SamplingDataJava.class);
private static String startTime = "2017-03-21 00:00:00.000";
private static String endTime = "2017-04-25 00:00:00.000";
private final String SELECT_STATEMENT = "select start_time,duration from transactions_data "
+ " where app_name='app_name-abc' and api_name='api_name-1' "
+ " and app_id='app_id-xyz' " + " AND start_time>='"
+ startTime + "' AND start_time<='" + endTime + "' ";
private Cluster cluster;
private Session session;
private String Host = "localhost";
public SamplingDataJava() throws IOException {
// this.query=query;
logger.info("Using CQL3 Writer");
cluster = Cluster.builder().addContactPoints(Host)
.withSocketOptions(new SocketOptions().setConnectTimeoutMillis(2000000)).build();
session = cluster.connect();
}
private class Result {
double duration;
int count;
Result(double duration, int count) {
this.duration = duration;
this.count = count;
}
#Override
public String toString() {
return "Result [duration=" + duration + ", count=" + count + "]";
}
}
public void hashSampling(long interval,long initTime) throws IOException {
HashMap<Long, Result> agg = new HashMap<>();
ResultSet rs = session.execute(SELECT_STATEMENT);
int i = 0;
for (com.datastax.driver.core.Row row : rs) {
i++;
Long hashcode = Math.abs((row.getTimestamp("start_time").getTime() - initTime) / interval);
Result hasResult = agg.get(hashcode);
if (hasResult == null) {
hasResult = new Result(row.getInt("duration"), 1);
} else {
hasResult.duration = (hasResult.duration + row.getInt("duration"));
hasResult.count++;
}
agg.put(hashcode, hasResult);
}
System.out.println("total number of records " + i);
Long code=0L;
while (code<30) {
System.out.println(" code "+agg.get(code));
code++;
}
}
public void close() {
cluster.close();
session.close();
}
public static void main(String[] args) throws IOException {
long beginTime = System.currentTimeMillis();
SamplingDataJava cqp = new SamplingDataJava();
long onlyQueryTime = System.currentTimeMillis();
DateTimeFormatter readPattern = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
DateTime sTime = readPattern.parseDateTime(startTime);
DateTime eTime = readPattern.parseDateTime(endTime);
long interval = (eTime.getMillis() - sTime.getMillis()) / 30;
System.out.println("start end time :" + eTime.getMillis() + " " + sTime.getMillis());
cqp.hashSampling(interval,sTime.getMillis());
System.out.println("total time without open close " + (System.currentTimeMillis() - onlyQueryTime));
cqp.close();
System.out.println("total time " + (System.currentTimeMillis() - beginTime));
}
}
UDF code
CREATE OR REPLACE FUNCTION txn_group_count_and_sum( txn map<bigint,
frozen<tuple<int,int>>>, start_time bigint, duration int , sample_size
bigint, begin_time bigint )
RETURNS NULL ON NULL INPUT
RETURNS map<bigint, frozen<tuple<int,int>>>
LANGUAGE java AS '
Long hashcode = (start_time - begin_time)/sample_size;
TupleValue tupleValue = txn.get(hashcode);
if (tupleValue == null) {
com.datastax.driver.core.TupleType tupleType =
com.datastax.driver.core.TupleType.of(
com.datastax.driver.core.ProtocolVersion.
NEWEST_SUPPORTED, com.datastax.driver.core.CodecRegistry.DEFAULT_INSTANCE,
com.datastax.driver.core.DataType.cint(),
com.datastax.driver.core.DataType.cint());
tupleValue = tupleType.newValue(1, duration );
}else{
tupleValue.setInt(0, tupleValue.getInt(0) + 1);
tupleValue.setInt(1, tupleValue.getInt(1) + duration);
}
txn.put(hashcode, tupleValue);
return txn; ' ;
CREATE OR REPLACE AGGREGATE group_count_and_sum(bigint, int ,bigint, bigint)
SFUNC txn_group_count_and_sum
STYPE map<bigint, frozen<tuple<int,int>>>
INITCOND {};
query
select group_count_and_sum(toUnixTimestamp(start_time),duration,100800000,1490054400000) from transactions_data
where app_name='app_name-abc' and api_name='api_name-1'
and app_id='app_id-xyz'
AND start_time>='2017-03-21 00:00:00.000' AND start_time<='2017-04-25 00:00:00.000';
Note:-
100800000 = (end_time - start_time)/30
1490054400000 = millisecond of 2017-03-21 00:00:00.000
Related
I'm currently making a shopping app with a shopping cart. I got the listview to display with the proper quantity and price but now have problems with trying to show the total price for checkout. There are 2 problems I'm facing :
How to get data from a whole column, right now I am only able to get data from the first index of the intended column.
When I clear the database/clear the cart, the app crashes, saying there is a null. Probably due to the cursor not able to get the read anything from the database.
CartDatabaseHelper.java getCartPrice() method
public Double getCartPrice() {
String query = "SELECT * FROM " + CartContract.CartEntry.TABLE_NAME + " WHERE Cart_Meal_Price";
SQLiteDatabase dbCart;
dbCart = this.getWritableDatabase();
Cursor cursor = dbCart.rawQuery(query, null);
cursor.moveToFirst();
do{if (cursor!=null) {
Double totalPrice = cursor.getDouble(2);
Double sum =+ totalPrice;
return sum;
} else {
return 0.00;
}}while(cursor.moveToNext());
}
CartDataBaseHelper.java Declaration and Initialization of cart database
public static final int DATABASE_VERSION = 1;
public static final String DATABASE_NAME = "CartDatabase";
public static final String CART_TABLE = "CartTable";
public static final String CREATE_CART_TABLE = "CREATE TABLE " + CartContract.CartEntry.TABLE_NAME + " ( " +
CartContract.CartEntry._ID + " INTEGER PRIMARY KEY AUTOINCREMENT, " +
CartContract.CartEntry.COLUMN_NAME + " TEXT ," +
CartContract.CartEntry.COLUMN_PRICE + " REAL , " +
CartContract.CartEntry.COLUMN_QUANTITY + " REAL ) " ;
SummaryActivity.java getTotalPrice() method
public void getTotalPrice() {
cartDB = new CartDatabaseHelper(this);
double total = cartDB.getCartPrice();
totalprice.setText("Total Price : RM" + total);
}
SummaryActivity.java Clear database onClickListener
clearthedata.setOnClickListener(new View.OnClickListener() {
#Override
public void onClick(View view) {
int deletethedata = getContentResolver().delete(CartContract.CartEntry.CONTENT_URI, null, null);
cartDB = new CartDatabaseHelper(context);
double total = cartDB.getCartPrice();
if (cartDB != null) {
totalprice.setText("Total Price : RM" + total);
} else {
totalprice.setText("Total Price : RM0.00");
}
}
});
If you want to get the sum of all the values of the column CartContract.CartEntry.COLUMN_PRICE in the table, then you need a query like:
SELECT COALESCE(SUM(price), 0) FROM tablename
Here COALESCE() is used to return 0 just in case the table is empty.
So your code should be:
public double getCartPrice() {
String query = "SELECT COALESCE(SUM(" + CartContract.CartEntry.COLUMN_PRICE + "), 0) FROM " + CartContract.CartEntry.TABLE_NAME;
SQLiteDatabase dbCart = this.getWritableDatabase();
Cursor cursor = dbCart.rawQuery(query, null);
cursor.moveToFirst();
double total = cursor.getDouble(0);
cursor.close();
dbCart.close();
return total;
}
If you want to delete all the rows of the table, then:
SQLiteDatabase dbCart = this.getWritableDatabase();
dbCart.delete(CartContract.CartEntry.TABLE_NAME, null, null);
This question already has answers here:
"Integer number too large" error message for 600851475143
(8 answers)
Closed 2 years ago.
I'm trying to store a long number (i.e. 9999999999) on a SQLite INTEGER column, and when I try to use ".put" from ContentValues, it doesn't accept, saying it's larger than an integer number.
The interesting thing is that when I tried to create a long variable, I had the same warning.
Bellow there is the code, witch on AndroiStudio gives me the erros:
PS.: sorry for posting it, because there are so many of these kind of question, but I couldn't find a solution looking them.
public class FeedHelperGamer extends SQLiteOpenHelper {
String TAG = "HelperGamer";
//string to create gamerDB
private static final String SQL_CREATE_ENTRIES_Gamer =
"CREATE TABLE " + FeedReaderContract.FeedEntry.TABLE_GAMER + " (" +
_ID + " INTEGER PRIMARY KEY," +
COLUMN_GAMER_NAME+ " TEXT," +
COLUMN_GAMER_GAMES + " INTEGER," +
COLUMN_GAMER_HITS + " INTEGER," +
COLUMN_GAMER_POINTS + " INTEGER," +
COLUMN_GAMER_PICTURE + " BLOB," +
COLUMN_GAMEMODE1 + " TEXT," +
COLUMN_GAMEMODE2 + " TEXT," +
COLUMN_GAMEMODE3 + " TEXT," +
COLUMN_GAMEMODE4 + " TEXT," +
COLUMN_GAMEMODE5 + " TEXT)";
private static final String SQL_DELETE_ENTRIES_GAMER =
"DROP TABLE IF EXISTS " + FeedReaderContract.FeedEntry.TABLE_GAMER;
// If you change the database schema, you must increment the database version.
public static final int GAMER_DATABASE_VERSION = 1;
public static final String DATABASE_NAME_GAMER = "Gamer.db";
public FeedHelperGamer(Context context) {
super(context, DATABASE_NAME_GAMER, null, GAMER_DATABASE_VERSION);
}
public void onCreate(SQLiteDatabase db) {
db.execSQL(SQL_CREATE_ENTRIES_Gamer);
}
public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
// This database is only a cache for online data, so its upgrade policy is
// to simply to discard the data and start over
db.execSQL(SQL_DELETE_ENTRIES_GAMER);
onCreate(db);
}
public void onDowngrade(SQLiteDatabase db, int oldVersion, int newVersion) {
onUpgrade(db, oldVersion, newVersion);
}
//insere novo jogador
public long addNewGamer(String tmpGamerName, byte[] imageBytes) {
final SQLiteDatabase db = this.getWritableDatabase();
long number = 999999999999999; //it's useless just created to test the long variable
// Create a new map of values, where column names are the keys
ContentValues values = new ContentValues();
values.put( COLUMN_GAMER_NAME, tmpGamerName );
values.put( COLUMN_GAMER_GAMES, 0 );
values.put( COLUMN_GAMER_HITS, 0 );
values.put( COLUMN_GAMER_POINTS, 9999999999);
values.put( COLUMN_GAMER_PICTURE, imageBytes );
values.put( COLUMN_GAMEMODE1, modeStateFalse );
values.put( COLUMN_GAMEMODE2, modeStateFalse );
values.put( COLUMN_GAMEMODE3, modeStateFalse );
values.put( COLUMN_GAMEMODE4, modeStateFalse );
long newRoId = db.insert( FeedReaderContract.FeedEntry.TABLE_GAMER, null, values );
return newRoId;
}
You should change this line
long number = 999999999999999;
like below
long number = 999999999999999L;
I've written what seems to be a prohibitively slow program where it is taking many minutes rather than seconds to read and write data from a database. It appears to be slowing down a great deal in my runComparisons method below although im unsure what steps I should take to correct this.
full program
public static void runComparisons(String industryTitle, List<String> tickerName, TradingInterval tradingInterval) throws SQLException {
// Get primary performance values ticker return
for (String ticker : tickerName) {
String startDate = tradingInterval.beginDate;
//System.out.println("start date " + startDate);
String endDate = tradingInterval.endDate;
//System.out.println("end date " + endDate);
PreparedStatement statement = conn.prepareStatement
("select P.TransDate, P.openPrice, P.closePrice" +
" from PriceVolume P" +
" where Ticker = ? and TransDate>= ? and TransDate <= ?" +
"order by TransDate DESC");
statement.setString(1, ticker);
statement.setString(2, startDate);
statement.setString(3, endDate);
ResultSet result = statement.executeQuery();
String date;
double nextOpenPrice = 0.0;
double currentClosePrice;
//int numberDays = 1;
//int splitCounter = 0;
double divideAmount = 1;
String tickerSymbol = "";
// create list to contain dates for use in investment strategy
ArrayList<holdSplitVals> ascendingList = new ArrayList<>();
// process result set
while (result.next()) {
//tickerSymbol = result.getString("Ticker");
// reset split amount for next iteration
date = result.getString("TransDate");
double openPrice = result.getDouble("OpenPrice");
double closePrice = result.getDouble("ClosePrice");
currentClosePrice = closePrice;
// get closing/opening ratio
double splitAmount = getRatios(date, nextOpenPrice, currentClosePrice);
if (splitAmount > 0) {
divideAmount = divideAmount * splitAmount;
//splitCounter++;
}
// hold data in helper class
holdSplitVals data = new holdSplitVals();
data.date = date;
data.openPrice = openPrice / divideAmount;
data.closePrice = closePrice / divideAmount;
ascendingList.add(0, data);
nextOpenPrice = openPrice;
if (ascendingList.size() >= 1) {
outputTable(closePrice, openPrice, industryTitle, tickerSymbol, startDate, endDate);
}
}
}
}
private static void outputTable(double closePrice, double openPrice, String industryTitle, String tickerSymbol, String startDate, String endDate) throws SQLException {
double tickerNumber = (closePrice / openPrice) - 1;
DecimalFormat df = new DecimalFormat("#.####");
df.setRoundingMode(RoundingMode.CEILING);
String tickerNum = df.format(tickerNumber);
PreparedStatement writeStatement = writerconn.prepareStatement(
"insert into Performance(Industry, Ticker, StartDate, EndDate, TickerReturn)"
+ "values(?, ?, ?, ?, ?)");
writeStatement.setString(1, industryTitle);
writeStatement.setString(2, tickerSymbol);
writeStatement.setString(3, startDate);
writeStatement.setString(4, endDate);
writeStatement.setString(5, tickerNum);
writeStatement.executeUpdate();
}
private static double getRatios(String date, double nextOpenPrice, double currentClosePrice) {
double coRatio = currentClosePrice / nextOpenPrice;
// calculate splits
// for 2:1 split
if (Math.abs(coRatio - 2.0) < 0.20) {
//splitCounter++;
//splitDate = ("2:1 split on " + date);
//System.out.println("2:1 split on " + date + " " + currentClosePrice + "--> " + nextOpenPrice);
return 2.0;
// for 3:1 split
} else if (Math.abs(coRatio - 3.0) < 0.20) {
//splitCounter++;
//splitDate = ("3:1 split on "+ date);
//System.out.println("3:1 split on " + date + " " + currentClosePrice + "--> " + nextOpenPrice);
return 3.0;
// for 3:2 split
} else if (Math.abs(coRatio - 1.5) < 0.15) {
//splitCounter++;
//splitDate = ("3:2 split on " + date);
//System.out.println("3:2 split on " + date + " " + currentClosePrice + "--> " + nextOpenPrice);
return 1.5;
}
return 0;
}
}
I have a constructor below. How do I write an array of Event like
Event[] September = new Event[30];
Event[] October = new Event[31];
How do i produce an output such as this
class Event
{
private String TimeofEvent;
private String startingTime;
private int priority;
public Event(String TimeofEvent, String startingTime, int priority)
{
this.TimeofEvent = TimeofEvent;
this.startingTime = startingTime;
this.priority = (int)(Math.random() *3) + 1;
}
public String toString()
{
return "TimeofEvent: " + TimeofEvent + " " + "Starting Time: " + startingTime + " " + "Priority: " + " " +priority;
}
You could attempt to use a HashMap to store your Objects if that is the issue.
To get the output right I would play around with the GridLayout
I am running Apache storm topology and I am trying to read the values from properties file. My main function is able to read the values while the bolts are throwing the following error.
Also , I am able to read all the values from a normal class in the same project.
Error
java.lang.NullPointerException at
java.util.Properties$LineReader.readLine(Properties.java:434) at
java.util.Properties.load0(Properties.java:353) at
java.util.Properties.load(Properties.java:341) at
com.StormConsumer.commons.FilePropertyManager.loadPrope
rtyFile(FilePropertyManager.java:54) at
com.StormConsumer.commons.FilePropertyManager.getProper
ty(FilePropertyManager.java:34) at
com.StormConsumer.bolt.SessionIdCounter.prepare(SessionIdCounter.java:42) at
backtype.storm.topology.BasicBoltExecutor.prepare(BasicBoltExecutor.java:43)
at backtype.storm.daemon.executor$fn__4722$fn__4734.invoke(executor.clj:692)
at backtype.storm.util$async_loop$fn__458.invoke(util.clj:461) at
clojure.lang.AFn.run(AFn.java:24) at java.lang.Thread.run(Thread.java:745)
How to solve it ?
This is the class :
public class SessionIdCounter extends BaseBasicBolt {
private static final long serialVersionUID = 1L;
Cluster cluster;
Session session;
/*private static final String KEYSPACE = FilePropertyManager.getProperty(
ApplicationConstants.CASSANDRA_CONSTANTS_FILE,
ApplicationConstants.KEYSPACE);
private static final String CREATE_TABLE = FilePropertyManager.getProperty(
ApplicationConstants.CASSANDRA_QUERY_FILE,
ApplicationConstants.TABLE_SessionID);*/
/**
* creates a table SessionID with fields sessionid, year, month, day, hour,
* dayofyear, weekofyear, count.
*/
public void cleanup() {
}
public void prepare(Map stormConf, TopologyContext context) {
final String KEYSPACE = FilePropertyManager.getProperty(
ApplicationConstants.CASSANDRA_CONSTANTS_FILE,
ApplicationConstants.KEYSPACE);
final String CREATE_TABLE = FilePropertyManager.getProperty(
ApplicationConstants.CASSANDRA_QUERY_FILE,
ApplicationConstants.TABLE_SessionID);
cluster = CassandraConnection.setupCassandraClient();
session = CassandraConnection.getSessionWithRetry(cluster, KEYSPACE);
session.executeAsync(CREATE_TABLE);
}
public void declareOutputFields(OutputFieldsDeclarer declarer) {
}
public void execute(Tuple input, BasicOutputCollector collector) {
String sessionid = input.getString(0);
int year = input.getInteger(1);
int month = input.getInteger(2);
int day = input.getInteger(3);
int hour = input.getInteger(4);
int dayofyear = input.getInteger(5);
int weekofyear = input.getInteger(6);
/*
* Inserting Values In Cassandra
*/
String insertUpdateTable = "UPDATE SessionID SET count = count + 1 "
+ "where sessionid = \'" + sessionid + "\' AND year = " + year
+ " AND month = " + month + " AND day = " + day
+ " AND hour = " + hour + " AND dayofyear = " + dayofyear
+ " AND weekofyear = " + weekofyear + " ;";
session.executeAsync(insertUpdateTable);
}
}
The problem is in your property file. If the ApplicationConstants.CASSANDRA_CONSTANTS_FILE points to a local file not avalaible in all your server, then you've got a problem. Also, check the format of your file.