I am iterating 500,000 items using for loop , after 300, 000 item i am getting Out of memory , I have also tried to split loop in 100,000 still, but that also didn't work. When I increased memory in runConfig-xms10g, its working fine. Can anyone tell me how to split and free memory or any other way to iterate large number of recods.
protected Map<Long, Map<String, String>> getExistingItems()
{
Map<Long, Map<String, String>> items = new HashMap<Long, Map<String, String>>();
for (Item item : itemMaster.getItems()) {
if (item.getExpirationDate() == null && !items.containsValue(item.getItemId())) {
Map<String, String> item_hash = new HashMap<String, String>();
if (item.getEffectiveDate() != null)
item_hash.put("effectiveDate", sdf.format(item.getEffectiveDate().getTime()));
for (ItemMasterAttributeValue attrVal : item.getItemMasterAttributeValues()) {
item_hash.put(attrVal.getId().getItemMasterAttribute().getCode(), attrVal.getValue());
}
items.put(item.getItemId(), item_hash);
}
}
after splitting below is code:
protected Map<Long, Map<String, String>> getExistingItems()
{
Map<Long, Map<String, String>> items = new HashMap<Long, Map<String, String>>();
java.util.Date date = new java.util.Date();
System.out.println("before getItems : " + new Timestamp(date.getTime()));
Collection<Item> allItems = itemMaster.getItems();
System.out.println("after getItems : " + new Timestamp(date.getTime()));
int split = (allItems.size() / 100000);
Map<Integer, Collection<Item>> splitMap = new HashMap<Integer, Collection<Item>>();
Collection<Item> tempCollection = new ArrayList<Item>();
int splitKey = 1;
int key = 1;
for(Item item : allItems)
{
tempCollection.add(item);
if(splitKey / 100000 >= key && splitKey % 100000 == 0)
{
splitMap.put(key, tempCollection);
tempCollection = new ArrayList<Item>();
key++;
}
splitKey++;
}
splitMap.put(key, tempCollection);
System.out.println("map size " + splitMap.size());
for(int i = 1; i <= split + 1; i++)
{
System.out.println("i is :" + i + " " + Calendar.getInstance().getTime());
for(Item item : splitMap.get(i))
{
if(!items.containsKey(item.getItemId()))
{
Map<String, String> item_hash = new HashMap<String, String>();
if (item.getEffectiveDate() != null)
item_hash.put("effectiveDate", sdf.format(item.getEffectiveDate().getTime()));
for (ItemMasterAttributeValue attrVal : item.getItemMasterAttributeValues()) {
item_hash.put(attrVal.getId().getItemMasterAttribute().getCode(), attrVal.getValue());
}
items.put(item.getItemId(), item_hash);
}
}
System.gc();
}
Related
I have recently given interview with major MNC and faced a coding question. It was an inventory based question. Got a file which has inputs as:
Date, name of fruit, no of fruits sold, total fruits.
**2018-06-01,apple,15,25 ;
2018-06-02,apple,13,25 ;
2018-06-03,apple,20,25 ;
2018-06-01,mango,12,25 ;
2018-06-02,mango,19,25 ;
2018-06-03,mango,20,25 ;
2018-06-01,orange,15,25 ;
2018-06-02,mango,16,25 ;
2018-06-02,orange,14,25 ;
2018-06-03,orange,24,25 ;
2018-06-02,apple,18,25**
I have to iterate file and in the end print out the results how many fruits sold on 2018-06-02. Below is the code which i have writen
enter code here
public class Inventory {
public static void main(String[] args) throws IOException {
FileInputStream geek = new FileInputStream("/Users/preet/Desktop/test2.txt");
InputStreamReader reader = new InputStreamReader(geek, StandardCharsets.UTF_8);
BufferedReader in = new BufferedReader(reader);
List<String> lines = new ArrayList<String>();
Map<String, Map<String, List<Integer>>> map = new HashMap<String, Map<String, List<Integer>>>();
Map<String, List<Integer>> map2 = new HashMap<String, List<Integer>>();
List<Integer> apple = new ArrayList<Integer>();
List<Integer> mango = new ArrayList<Integer>();
List<Integer> orange = new ArrayList<Integer>();
String result = "";
String line = null;
String[] input = new String[10];
int i = 0;
while ((line = in.readLine()) != null) {
input[i] = line;
System.out.println("value at " + i + ":" + input[i]);
i++;
}
for (String a : input) {
String b[] = a.split(",");
if (b[1].equalsIgnoreCase("apple")) {
apple.add(Integer.parseInt(b[2]));
map2.put(b[1], apple);
map.put(b[0], map2);
} else if (b[1].equalsIgnoreCase("orange")) {
orange.add(Integer.parseInt(b[2]));
map2.put(b[1], orange);
map.put(b[0], map2);
} else if (b[1].equalsIgnoreCase("mango")) {
mango.add(Integer.parseInt(b[2]));
map2.put(b[1], mango);
map.put(b[0], map2);
}
if (map.containsKey(b[0])) {
Object value1 = map.get(b[0]);
map.put(b[0], map2);
} else
map.put(b[0], map2);
}
System.out.println("Fetching Keys and corresponding [Multiple] Values n");
List<String> results = new ArrayList<String>();
for (Map.Entry<String, Map<String, List<Integer>>> entry : map.entrySet()) {
String orange1 = null;
String apple1 = null;
String mango1 = null;
int a1 = 0;
int a2 = 0;
int a3 = 0;
Map<String, List<Integer>> values = entry.getValue();
for (Map.Entry<String, List<Integer>> entry1 : map2.entrySet()) {
String key = entry1.getKey();
if (key.equalsIgnoreCase("apple")) {
List l1 = entry1.getValue();
for (int j = 0; j < l1.size(); j++) {
a1 = (int) l1.get(j);
}
apple1 = entry.getKey() + " " + a1 + " apples";
results.add(apple1);
} else if (key.equalsIgnoreCase("mango")) {
List l1 = entry1.getValue();
for (int j = 0; j < l1.size(); j++) {
a2 = (int) l1.get(j);
}
mango1 = entry.getKey() + " " + a2 + " mangoes";
results.add(mango1);
} else if (key.equalsIgnoreCase("orange")) {
List l1 = entry1.getValue();
for (int j = 0; j < l1.size(); j++) {
a3 = (int) l1.get(j);
}
orange1 = entry.getKey() + " " + a3 + " oranges";
results.add(orange1);
}
}
System.out.println("Values = " + (values));
}
System.out.println("****" + results);
}
}
For an interview I would suggest the following as a first shot: iterate over the file. Split each line at commas using String.split(). If element 0 of the resulting array is identical to 2018-06-02, parse the integer in element 2 (the no of fruits sold) using Integer.parseInt() and add to the total count.
An obvious thing missing from such a first shot is input validation. May first check that there is at least one line in the file. Check that each line ends with a semicolon. Remove it and any space before it before splitting. Check that the length of the array is 4. Use the one-arg LocalDate.parse() for parsing the date on each line and Integer.parseInt() to parse both the number sold and the total number. Validate that the number sold is less than or equal to the total number. May also check that the name is that of a known fruit.
Another thing is data modelling. Design a fruit class with a name field and a fruit sales class with a date, a total number and a number sold, for example.
This question already has answers here:
Why does my ArrayList contain N copies of the last item added to the list?
(5 answers)
Closed 5 years ago.
I am trying to create a HashMap in a HashMap so it will be easier for me to access elements of it in the future as shown below.
The problem is it only repeating the last elements of the while loop and not the rest of it.
HashMap<String,String> result = new HashMap<>();
HashMap<Integer, HashMap<String,String>> fr = new HashMap<>();
int i = 0;
try {
ResultSet rq = qexec.execSelect();
// ResultSetFormatter.out(System.out, rq, query);
// get result from SPARQL query
while (rq.hasNext()) {
QuerySolution soln = rq.next();
id = soln.getLiteral("?id").getLexicalForm();
//...
result.put("id",id);
//...
if (soln.getLiteral("?wateruse") != null) {
wateruse = soln.getLiteral("?wateruse").getLexicalForm();
//...
result.put("wateruse",wateruse);
} else {
System.out.println("NO");
}
fr.put(i, result);
i++;
}
} finally {
qexec.close();
}
This is how the result should be:
John001
High usage
John002
John003
Smith001
Moderate
Smith002
Smith003
...
Kevin001
Low usage
But fr only repeats Kevin001 and Low usage without the rest.
I've tried to put fr.put(i,result) outside the loop but that still does not give the correct result.
EDIT
I tried to print all elements from fr that shows the repeating elements.
finally {
qexec.close();
}
for (int index : fr.keySet()) {
for(Map.Entry<String, String> entry :result.entrySet()) {
System.out.println(index + " = " + entry.getKey() + " : " + entry.getValue());
}
}
UPDATE - SOLUTION
Declare HashMap inside the loop as mentioned in comments below.
To print nested HashMap, no need to use result.
I did as shown below and it prints both outermap and innermap as well.
for (int k=0; k < fr.size(); k++) {
HashMap<String,String> innermap = fr.get(k);
for(Map.Entry<String, String> e : innermap.entrySet()) {
System.out.println(k + " = " + e.getKey() + " : " + e.getValue());
}
}
You're adding the same result map to your parent map each time through the loop. Create a new instance of result each time through the loop:
Map<String, String> result = new HashMap<>();
Map<Integer, Map<String, String>> fr = new HashMap<>();
int i = 0;
try {
ResultSet rq = qexec.execSelect();
while (rq.hasNext()) {
// Create your new HashMap inside the loop:
result = new HashMap<>();
QuerySolution soln = rq.next();
id = soln.getLiteral("?id").getLexicalForm();
//...
result.put("id",id);
//...
if (soln.getLiteral("?wateruse") != null) {
wateruse = soln.getLiteral("?wateruse").getLexicalForm();
//...
result.put("wateruse",wateruse);
}
else {
System.out.println("NO");
}
fr.put(i,result);
i++;
}
}
To print the results from fr an its nested map, you can do something like this:
for (Map<String, String> map : fr.values()) {
for(Map.Entry<String, String> e : map.entrySet()) {
System.out.println(index + " = " + e.getKey()
+ " : " + e.getValue());
}
}
Try this a small change here, place the "result" map creation in while loop
Map<Integer, Map<String, String>> fr = new HashMap<>();
int i = 0;
try {
ResultSet rq = qexec.execSelect();
while (rq.hasNext()) {
Map<String, String> result = new HashMap<>();
QuerySolution soln = rq.next();
id = soln.getLiteral("?id").getLexicalForm();
//...
result.put("id",id);
//...
if (soln.getLiteral("?wateruse") != null) {
wateruse = soln.getLiteral("?wateruse").getLexicalForm();
//...
result.put("wateruse",wateruse);
}
else {
System.out.println("NO");
}
fr.put(i,result);
i++;
}
}
This for loop to print elemenets:
for (int i=0;i< fr.size();i++){
Map<String,String> element= fr.get(i);
// use the element here.
}
I have a java program and it produces the output as follows :
termname :docname : termcount
Forexample termname is hello and docname is :2 and termcount is :4
hello:doc1:4
.....
......
I stored all the values in a map. here is the following program
public class tuple {
public static void main(String[]args) throws FileNotFoundException, UnsupportedEncodingException, SQLException, ClassNotFoundException, Exception{
File file2 = new File("D:\\logs\\tuple.txt");
PrintWriter tupled = new PrintWriter(file2, "UTF-8");
List<Map<String, Integer>> list = new ArrayList<>();
Map<String, Integer>map= new HashMap<>();;
String word;
//Iterate over documents
for (int i = 1; i <= 2; i++) {
//map = new HashMap<>();
Scanner tdsc = new Scanner(new File("D:\\logs\\AfterStem" + i + ".txt"));
//Iterate over words
while (tdsc.hasNext()) {
word = tdsc.next();
final Integer freq = map.get(word);
if (freq == null) {
map.put(word, 1);
} else {
map.put(word, map.get(word) + 1);
}
}
list.add(map);
}
// tupled.println(list);
//tupled.close();
//Print result
int documentNumber = 0;
for (Map<String, Integer> document : list) {
for (Map.Entry<String, Integer> entry : document.entrySet()) {
documentNumber++;
//System.out.println(entry.getKey() + ":doc"+documentNumber+":" + entry.getValue());
tupled.print(entry.getKey());
tupled.print(":doc:");
tupled.print(Integer.toString(documentNumber));
tupled.print(",");
tupled.println(entry.getValue());
}
//documentNumber++;
}
tupled.close();
Now I want to store this values into derby database of neatbeans.
How I would be able to do that ?
Nested HashMap:
HashMap<String,HashMap<String,String>> outerMap=new HashMap<String, HashMap<String, String>>();
HashMap<String,String> innerhashMap=new HashMap<String, String>();
innerhashMap.put("aaa","AAA");
outerMap.put("111",innerhashMap);
innerhashMap.put("aaa","AAA");
outerMap.put("222",innerhashMap);
I want outer map keys list,inner map keys list and innermap values list
for ( String outerkey : outerMap.keySet() ) {
HashMap<String,String> innerHashMap = (HashMap<String,String>) outerMap.get(outerKey)
for ( String innerKey : innerHashMap.keySet() ) {
String innerValue = (String) innerHashMap.get(innerKey);
//... Process them
}
}
HashMap<String,HashMap<String,String>> outerMap = new HashMap<>();
HashMap<String,String> innerhashMap = new HashMap<>();
innerhashMap.put("aaa","AAA");
outerMap.put("111",innerhashMap);
innerhashMap.put("aaa","AAA");
outerMap.put("222",innerhashMap);
outerMap.forEach((k, v) -> {
System.out.println("OUTER KEY: " +k);
v.forEach((kk, vv) -> {
System.out.println("INNER KEY: " +kk+ " INNER VALUE: " +vv);
});
});
Need to write the java code (algorithm) equivalent to the below sql query -
select u.browsers, a.app_name, count(*)
as no_of_apps from users u inner join apps a
where u.user_id = a.user_id group by u.browsers, a.app_name order by no_of_apps limit 0,2.
There are two different .tsv files (user.tsv and app.tsv) and the format of data they contains is -
+1.users.tsv
user_id browsers
1 Mozilla
2 Mozilla
3 Chrome
+2. apps.tsv
app_id user_id app_name
1 1 HelloWorld
2 3 Sonar
3 1 Sonar
4 2 HelloWorld
5 1 HelloWorld
Here user_id is acting like a Foreign key in apps.tsv file. What I really need is the algorithm in java to get data from the two files something like this -
browsers app_name no_of_apps
Mozilla HelloWorld 3
Mozilla Sonar 1
Chrome Sonar 1
Code I wrote for this is -
import java.util.*;
import java.util.Map.*;
import java.util.HashMap;
import java.io.*;
import java.nio.file.*;
public class Task1 {
public static void main(String... args) {
//cloumn places
int state_col = 2;
int user_col = 0;
int jobid_col = 2;
List<String> userLines = null;
List<String> appLines = null;
String[][] userResultArray = null;
String[][] appResultArray = null;
Map<String, String> userMap = new HashMap<String,String>();
Map<String, String> appMap = new HashMap<String, String>();
Map<String, String> userJobMap = new HashMap<String,String>();
List<Integer> app = new ArrayList<Integer>();
try {
userLines = Files.readAllLines(Paths.get("/home/indg/java-spring/spring/idea_workspace/Task1/tabFiles/users.tsv"));
appLines = Files.readAllLines(Paths.get("/home/indg/java-spring/spring/idea_workspace/Task1/tabFiles/apps.tsv"));
} catch(Exception ex) {
ex.printStackTrace();
}
if(userLines != null) {
userResultArray = new String[userLines.size()][];
for(int i =0; i<userLines.size()-1; i++){
userResultArray[i] = userLines.get(i).split("\t"); //tab-separated
if(userResultArray[i][state_col] != "" && userResultArray[i][state_col] != null) {
userMap.put(userResultArray[i][user_col],userResultArray[i][state_col]);
}
}
} else {
System.out.println("Error in reading line");
}
if(appLines != null) {
appResultArray = new String[appLines.size()][];
for(int i =0; i<appLines.size()-1; i++){
appResultArray[i] = appLines.get(i).split("\t"); //tab-separated
if(appResultArray[i][jobid_col] != "" && appResultArray[i][jobid_col] != null) {
appMap.put(appResultArray[i][user_col], appResultArray[i][jobid_col]);
}
}
} else {
System.out.println("Error in reading line");
}
for(String userKey: userMap.keySet()) {
//System.out.println(userEntry.getKey() + "**");
String jobId = appMap.get(userKey);
if(jobId != null && userMap.get(userKey) != null && userMap.get(userKey) != "" ) {
userJobMap.put(userMap.get(userKey), jobId);
}
}
System.out.println(userJobMap + " --");
}
}
I want to apply order by and limit in the query too for example - order by no_of_apps limit 0,2 how to do that in the code
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
public class Task1 {
public static void main(String... args) {
List<String> userLines = Arrays.asList(new String[] { "1\tMozilla", "2\tMozilla", "3\tChrome" });
List<String> appLines = Arrays
.asList(new String[] { "1\t1\tHelloWorld", "2\t3\tSonar", "3\t1\tSonar", "4\t2\tHelloWorld", "5\t1\tHelloWorld" });
Map<String, String> userMap = new HashMap<String, String>();
Map<String, Map<String, Integer>> resultMap = new HashMap<String, Map<String, Integer>>();
if (userLines != null) {
for (int i = 0; i < userLines.size(); i++) {
String[] userResultArray = userLines.get(i).split("\t"); // tab-separated
userMap.put(userResultArray[0], userResultArray[1]);
}
} else {
System.out.println("Error in reading line");
}
if (appLines != null) {
for (int i = 0; i < appLines.size(); i++) {
String[] appResultArray = appLines.get(i).split("\t"); // tab-separated
if (userMap.containsKey(appResultArray[1])) {
String userName = userMap.get(appResultArray[1]);
if (resultMap.containsKey(userName)) {
Map<String, Integer> map = resultMap.get(userName);
Integer n = map.get(appResultArray[2]) == null ? 0 : map.get(appResultArray[2]);
map.put(appResultArray[2], ++n);
} else {
Map<String, Integer> map = new HashMap<String, Integer>();
map.put(appResultArray[2], 1);
resultMap.put(userName, map);
}
}
}
} else {
System.out.println("Error in reading line");
}
// sort
Map<Integer, List<String>> sortedMap = new TreeMap<Integer, List<String>>();
for (String userName : resultMap.keySet()) {
for (String app : resultMap.get(userName).keySet()) {
Integer n = resultMap.get(userName).get(app);
List<String> list = sortedMap.get(n) == null ? new ArrayList<String>() : sortedMap.get(n);
list.add(userName + "\t" + app);
sortedMap.put(n, list);
}
}
int limit = 0;
outer:
for (Integer n : sortedMap.keySet()) {
for (String s : sortedMap.get(n)) {
System.out.print(s);
System.out.println("\t" + n);
if (++limit == 2) break outer;
}
}
}
}